创建目录
mkdir /data/airflow
cd /data/airflow
下载并安装python
wget https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz
tar -zxvf Python-3.7.7.tgz
cd Python-3.7.7
./configure --with-zlib #重要
make && make install
export AIRFLOW_HOME=/data/airflow
pip3 install apache-airflow
airflow initdb
修改 /data/airflow/airflow.cfg
sql_alchemy_conn = mysql+pymysql://root:xxx@127.0.0.1:3306/airflow
executor = CeleryExecutor
需安装Celery pip3 install celery
broker_url = redis://127.0.0.1:6379/3
celery_result_backend = redis://127.0.0.1:6379/4
result_backend = db+mysql+pymysql://root:xxx@127.0.0.1:3306/airflow
#启用web验证
[webserver]
authenticate = True
auth_backend = airflow.contrib.auth.backends.password_auth
需先部署mysql,启动会报错 需安装pymysql
pip3 install pymysql
需安装redis服务器,redis客户端
pip3 install redis
启动
1
2
3
4
5
6airflow initdb cd /data/airflow nohup airflow webserver > webserver.out 2>&1 & nohup airflow scheduler > scheduler.out 2>&1 & su airflow #重要 nohup airflow worker > worker.out 2>&1 &
添加airflow用户
python3
1
2
3
4
5
6
7
8
9
10
11
12
13>>> import airflow >>> from airflow import models, settings >>> from airflow.contrib.auth.backends.password_auth import PasswordUser >>> user = PasswordUser(models.User()) >>> user.username = 'new_user_name' >>> user.email = 'new_user_email@example.com' >>> user.password = 'set_the_password' >>> session = settings.Session() >>> session.add(user) >>> session.commit() >>> session.close() >>> exit()
打开浏览器输入192.168.1.101:8080 用设置的用户登录
最后
以上就是妩媚火车最近收集整理的关于airflow从零开始部署的全部内容,更多相关airflow从零开始部署内容请搜索靠谱客的其他文章。
发表评论 取消回复