DAG: remove_history_files

schedule: 0 3 * * *

remove_history_files

Toggle wrap 
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
from airflow.contrib.operators.spark_submit_operator import SparkSubmitOperator
from datetime import timedelta,datetime

default_args = {
    'owner': 'Terry',
    'depends_on_past': False,
    'start_date': "2021-08-02",
    'email': ['tian23.li@tcl.com'],
    'email_on_failure': True,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=30),
}



dag = DAG(
    dag_id='remove_history_files',
    description='This dag is use to remove history files',
    default_args=default_args,
    schedule_interval='0 3 * * *',
)

# remove_history_files = BashOperator(
#     task_id='remove_history_files',
#     bash_command="sh /data/gangyanyuan/rm_history_files.sh ",
#     dag=dag,
# )

remove_history_files_2024 = BashOperator(
    task_id='remove_history_files_2024',
    bash_command="sh /data/gangyanyuan/rm_history_files_2024.sh ",
    dag=dag,
)

# remove_history_files_gary = BashOperator(
#     task_id='remove_history_files_gary',
#     bash_command="cd /data/gangyanyuan/gary && bash remove_unused_files.sh ",
#     dag=dag,
# )




remove_history_files_2024