From 3d3490290cbe707d6057c45f2f47f57299784ed9 Mon Sep 17 00:00:00 2001 From: PCaldeira-Claranet Date: Mon, 7 Apr 2025 16:38:30 +0100 Subject: [PATCH 1/5] My first commit!! --- hopster.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 hopster.txt diff --git a/hopster.txt b/hopster.txt new file mode 100644 index 00000000000..c60ef7b0b8e --- /dev/null +++ b/hopster.txt @@ -0,0 +1 @@ +I'm baby jean shorts godard selfies tacos. Leggings chicharrones banh mi, neutral milk hotel narwhal pabst asymmetrical gatekeep whatever raclette chia. Migas gorpcore banjo letterpress organic, vexillologist freegan flexitarian occupy selfies mustache. Messenger bag tumeric leggings biodiesel, letterpress succulents celiac fam humblebrag drinking vinegar. 8-bit franzen occupy, asymmetrical typewriter iPhone semiotics crucifix kombucha helvetica. Williamsburg stumptown butcher, bicycle rights raclette intelligentsia YOLO gorpcore vibecession shaman disrupt. Grailed disrupt hexagon, photo booth portland occupy live-edge messenger bag everyday carry hell of narwhal fashion axe distillery lumbersexual PBR&B. \ No newline at end of file From a7afabd52986e6c770003cb49c58916ed970c561 Mon Sep 17 00:00:00 2001 From: PCaldeira-Claranet Date: Tue, 8 Apr 2025 15:03:40 +0100 Subject: [PATCH 2/5] Adding git ignore to exclude files --- .gitignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..e69de29bb2d From 2d04529faea7e6e9c8f865ada33207a4c4a739cb Mon Sep 17 00:00:00 2001 From: PCaldeira-Claranet Date: Tue, 8 Apr 2025 15:12:52 +0100 Subject: [PATCH 3/5] Excluding log files directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e69de29bb2d..bf0824e596e 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +*.log \ No newline at end of file From 3f2a48470e63de9c362a54119747586f060fc260 Mon Sep 17 00:00:00 2001 From: PCaldeira-Claranet Date: Wed, 9 Apr 2025 15:58:23 +0100 Subject: [PATCH 4/5] Notebook to see --- cred.ipynb | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 cred.ipynb diff --git a/cred.ipynb b/cred.ipynb new file mode 100644 index 00000000000..7d8cbb79d1c --- /dev/null +++ b/cred.ipynb @@ -0,0 +1,104 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "dd78701c", + "metadata": {}, + "outputs": [], + "source": [ + "from airflow import DAG\n", + "from airflow.providers.postgres.operators.postgres import PostgresOperator\n", + "from airflow.providers.http.sensors.http import HttpSensor\n", + "from airflow.providers.http.operators.http import SimpleHttpOperator\n", + "from airflow.operators.python import PythonOperator\n", + "from airflow.providers.postgres.hooks.postgres import PostgresHook\n", + "\n", + "from datetime import datetime\n", + "from pandas import json_normalize # type:ignore\n", + "import json\n", + "\n", + "# The python function to call\n", + "def _process_user(ti): # ti = task instance\n", + " user = ti.xcom_pull(task_ids=\"extract_user\") # fetch data pushed by the previous task extract_user\n", + " user = user['results'][0]\n", + " processed_user = json_normalize({\n", + " 'firstname': user['name']['first'],\n", + " 'lastname': user['name']['last'],\n", + " 'country': user['location']['country'],\n", + " 'username': user['login']['username'],\n", + " 'email': user['email']\n", + " })\n", + " processed_user.to_csv('/tmp/processed_user.csv', index=None, header=False)\n", + "\n", + "# The task to run\n", + "def _store_user():\n", + " hook = PostgresHook(postgres_conn_id='postgres')\n", + " hook.copy_expert(\n", + " sql=\"COPY users FROM stdin WITH DELIMITER as ','\",\n", + " filename='/tmp/processed_user.csv'\n", + " )\n", + "\n", + "\n", + "\n", + "\n", + "with DAG(dag_id=\"user_processing\", start_date=datetime(2023, 1, 1), schedule_interval=\"@daily\", catchup=False) as dag:\n", + " \n", + " create_table = PostgresOperator(\n", + " task_id='create_table',\n", + " postgres_conn_id='postgres',\n", + " sql='''\n", + " CREATE TABLE IF NOT EXISTS users (\n", + " firstname TEXT NOT NULL,\n", + " lastname TEXT NOT NULL,\n", + " country TEXT NOT NULL,\n", + " username TEXT NOT NULL,\n", + " password TEXT NOT NULL,\n", + " email TEXT NOT NULL\n", + " );\n", + " '''\n", + " )\n", + " \n", + " is_api_available = HttpSensor(\n", + " task_id='is_api_available',\n", + " http_conn_id='user_api',\n", + " endpoint='api/'\n", + " )\n", + " \n", + " extract_user = SimpleHttpOperator(\n", + " task_id='extract_user',\n", + " http_conn_id='user_api',\n", + " endpoint='api/',\n", + " method='GET',\n", + " response_filter=lambda response: json.loads(response.text),\n", + " log_response=True\n", + " )\n", + " \n", + " process_user = PythonOperator(\n", + " task_id ='process_user',\n", + " python_callable=_process_user\n", + " )\n", + "\n", + " store_user = PythonOperator(\n", + " task_id='store_user',\n", + " python_callable=_store_user\n", + " )\n", + " \n", + " create_table >> is_api_available >> extract_user >> process_user >> store_user" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From e82bd4a84dc04c9bb6dac69b2c4132e11e19e7fe Mon Sep 17 00:00:00 2001 From: PCaldeira-Claranet Date: Wed, 9 Apr 2025 16:32:04 +0100 Subject: [PATCH 5/5] feature branch commit --- newFileFeatureBranch.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 newFileFeatureBranch.txt diff --git a/newFileFeatureBranch.txt b/newFileFeatureBranch.txt new file mode 100644 index 00000000000..4447d7425aa --- /dev/null +++ b/newFileFeatureBranch.txt @@ -0,0 +1 @@ +other branch