|
186 | 186 | "age_check_column": ImportError.timestamp,
|
187 | 187 | "keep_last": False,
|
188 | 188 | "keep_last_filters": None,
|
189 |
| - "keep_last_group_by": None |
| 189 | + "keep_last_group_by": None, |
| 190 | + "do_not_delete_by_dag_id": True |
190 | 191 | })
|
191 | 192 |
|
192 | 193 | except Exception as e:
|
|
204 | 205 | "age_check_column": Task.date_done,
|
205 | 206 | "keep_last": False,
|
206 | 207 | "keep_last_filters": None,
|
207 |
| - "keep_last_group_by": None |
| 208 | + "keep_last_group_by": None, |
| 209 | + "do_not_delete_by_dag_id": True |
208 | 210 | }, {
|
209 | 211 | "airflow_db_model": TaskSet,
|
210 | 212 | "age_check_column": TaskSet.date_done,
|
211 | 213 | "keep_last": False,
|
212 | 214 | "keep_last_filters": None,
|
213 |
| - "keep_last_group_by": None |
| 215 | + "keep_last_group_by": None, |
| 216 | + "do_not_delete_by_dag_id": True |
214 | 217 | }))
|
215 | 218 |
|
216 | 219 | except Exception as e:
|
@@ -358,10 +361,15 @@ def cleanup_function(**context):
|
358 | 361 |
|
359 | 362 | if ENABLE_DELETE:
|
360 | 363 | logging.info("Performing Delete...")
|
361 |
| - # using bulk delete |
362 |
| - query.delete(synchronize_session=False) |
363 |
| - session.commit() |
364 |
| - logging.info("Finished Performing Delete") |
| 364 | + if context["params"].get("do_not_delete_by_dag_id"): |
| 365 | + query.filter(age_check_column <= max_date).delete(synchronize_session=False) |
| 366 | + session.commit() |
| 367 | + else: |
| 368 | + dags = session.query(airflow_db_model.dag_id).distinct() |
| 369 | + list_dags = [str(list(dag)[0]) for dag in dags] |
| 370 | + for dag in list_dags: |
| 371 | + query.filter(age_check_column <= max_date).filter(airflow_db_model.dag_id == dag).delete(synchronize_session=False) |
| 372 | + session.commit() |
365 | 373 | else:
|
366 | 374 | logging.warn("You've opted to skip deleting the db entries. "
|
367 | 375 | "Set ENABLE_DELETE to True to delete entries!!!")
|
|
0 commit comments