diff --git a/.github/workflows/get-metrics.py b/.github/workflows/get-metrics.py
new file mode 100644
index 00000000..4accaed9
--- /dev/null
+++ b/.github/workflows/get-metrics.py
@@ -0,0 +1,256 @@
+import datetime
+import json
+import os
+
+import cartopy
+import matplotlib
+import matplotlib.cm as cm
+import matplotlib.colors as colors
+import matplotlib.pyplot as plt
+import numpy as np
+from google.analytics.data_v1beta import BetaAnalyticsDataClient
+from google.analytics.data_v1beta.types import DateRange, Dimension, Metric, RunReportRequest
+
+PORTAL_ID = '266784902'
+FOUNDATIONS_ID = '281776420'
+COOKBOOKS_ID = '324070631'
+
+PRIVATE_KEY_ID = os.environ.get('PRIVATE_KEY_ID')
+PRIVATE_KEY = os.environ.get('PRIVATE_KEY')
+
+credentials_dict = {
+ 'type': 'service_account',
+ 'project_id': 'cisl-vast-pythia',
+ 'private_key_id': PRIVATE_KEY_ID,
+ 'private_key': PRIVATE_KEY,
+ 'client_email': 'pythia-metrics-api@cisl-vast-pythia.iam.gserviceaccount.com',
+ 'client_id': '113402578114110723940',
+ 'auth_uri': 'https://accounts.google.com/o/oauth2/auth',
+ 'token_uri': 'https://oauth2.googleapis.com/token',
+ 'auth_provider_x509_cert_url': 'https://www.googleapis.com/oauth2/v1/certs',
+ 'client_x509_cert_url': 'https://www.googleapis.com/robot/v1/metadata/x509/pythia-metrics-api%40cisl-vast-pythia.iam.gserviceaccount.com',
+ 'universe_domain': 'googleapis.com',
+}
+
+client = BetaAnalyticsDataClient.from_service_account_info(credentials_dict)
+
+pre_project_date = '2020-03-31' # random date before project start
+
+
+def _format_rounding(value):
+ return f'{round(value / 1000, 1):.1f}K'
+
+
+def _run_total_users_report(property_id):
+ request = RunReportRequest(
+ property=f'properties/{property_id}',
+ dimensions=[],
+ metrics=[Metric(name='activeUsers')],
+ date_ranges=[DateRange(start_date=pre_project_date, end_date='today')],
+ )
+ response = client.run_report(request)
+
+ total_users = 0
+ for row in response.rows:
+ total_users += int(row.metric_values[0].value)
+
+ return _format_rounding(total_users)
+
+
+def get_total_users(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID):
+ metrics_dict = {}
+ metrics_dict['Now'] = str(datetime.datetime.now())
+ metrics_dict['Portal'] = _run_total_users_report(PORTAL_ID)
+ metrics_dict['Foundations'] = _run_total_users_report(FOUNDATIONS_ID)
+ metrics_dict['Cookbooks'] = _run_total_users_report(COOKBOOKS_ID)
+ with open('portal/metrics/user_metrics.json', 'w') as outfile:
+ json.dump(metrics_dict, outfile)
+
+
+def _run_active_users_this_year(property_id):
+ current_year = datetime.datetime.now().year
+ start_date = f'{current_year}-01-01'
+
+ request = RunReportRequest(
+ property=f'properties/{property_id}',
+ dimensions=[Dimension(name='date')],
+ metrics=[Metric(name='activeUsers')],
+ date_ranges=[DateRange(start_date=start_date, end_date='today')],
+ )
+ response = client.run_report(request)
+
+ dates = []
+ user_counts = []
+ for row in response.rows:
+ date_str = row.dimension_values[0].value
+ date = datetime.datetime.strptime(date_str, '%Y%m%d')
+ dates.append(date)
+ user_counts.append(int(row.metric_values[0].value))
+
+ return zip(*sorted(zip(dates, user_counts), key=lambda x: x[0]))
+
+
+def plot_projects_this_year(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID):
+ portal_dates, portal_users = _run_active_users_this_year(PORTAL_ID)
+ foundations_dates, foundations_users = _run_active_users_this_year(FOUNDATIONS_ID)
+ cookbooks_dates, cookbooks_users = _run_active_users_this_year(COOKBOOKS_ID)
+
+ plt.figure(figsize=(10, 5.5))
+ plt.title('Year-to-Date Pythia Active Users', fontsize=15)
+
+ plt.plot(portal_dates, portal_users, color='purple', label='Portal')
+ plt.plot(foundations_dates, foundations_users, color='royalblue', label='Foundations')
+ plt.plot(cookbooks_dates, cookbooks_users, color='indianred', label='Cookbooks')
+
+ plt.legend(fontsize=12, loc='upper right')
+
+ plt.xlabel('Date', fontsize=12)
+ plt.savefig('portal/metrics/thisyear.png', bbox_inches='tight')
+
+
+def _run_top_pages_report(property_id):
+ request = RunReportRequest(
+ property=f'properties/{property_id}',
+ dimensions=[Dimension(name='pageTitle')],
+ date_ranges=[DateRange(start_date=pre_project_date, end_date='today')],
+ metrics=[Metric(name='screenPageViews')],
+ )
+ response = client.run_report(request)
+
+ views_dict = {}
+ for row in response.rows:
+ page = row.dimension_values[0].value
+ views = int(row.metric_values[0].value)
+ views_dict[page] = views
+
+ top_pages = sorted(views_dict.items(), key=lambda item: item[1], reverse=True)[:5]
+ pages = [page.split('β')[0] for page, _ in top_pages]
+ views = [views for _, views in top_pages]
+
+ return pages[::-1], views[::-1]
+
+
+def plot_top_pages(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID):
+ portal_pages, portal_views = _run_top_pages_report(PORTAL_ID)
+ foundations_pages, foundations_views = _run_top_pages_report(FOUNDATIONS_ID)
+ cookbooks_pages, cookbooks_views = _run_top_pages_report(COOKBOOKS_ID)
+
+ pages = cookbooks_pages + foundations_pages + portal_pages
+
+ fig, ax = plt.subplots(figsize=(10, 5.5))
+ plt.title('All-Time Top Pages', fontsize=15)
+
+ y = np.arange(5)
+ y2 = np.arange(6, 11)
+ y3 = np.arange(12, 17)
+ y4 = np.append(y, y2)
+ y4 = np.append(y4, y3)
+
+ bar1 = ax.barh(y3, portal_views, align='center', label='Portal', color='purple')
+ bar2 = ax.barh(y2, foundations_views, align='center', label='Foundations', color='royalblue')
+ bar3 = ax.barh(y, cookbooks_views, align='center', label='Cookbooks', color='indianred')
+
+ ax.set_yticks(y4, labels=pages, fontsize=12)
+
+ ax.bar_label(bar1, fmt=_format_rounding, padding=5, fontsize=10)
+ ax.bar_label(bar2, fmt=_format_rounding, padding=5, fontsize=10)
+ ax.bar_label(bar3, fmt=_format_rounding, padding=5, fontsize=10)
+
+ ax.set_xscale('log')
+ ax.set_xlim([10, 10**5])
+ ax.set_xlabel('Page Views', fontsize=12)
+
+ plt.legend(fontsize=12, loc='lower right')
+ plt.savefig('portal/metrics/toppages.png', bbox_inches='tight')
+
+
+def _run_usersXcountry_report(property_id):
+ request = RunReportRequest(
+ property=f'properties/{property_id}',
+ dimensions=[Dimension(name='country')],
+ metrics=[Metric(name='activeUsers')],
+ date_ranges=[DateRange(start_date=pre_project_date, end_date='today')],
+ )
+ response = client.run_report(request)
+
+ user_by_country = {}
+ for row in response.rows:
+ country = row.dimension_values[0].value
+ users = int(row.metric_values[0].value)
+ user_by_country[country] = user_by_country.get(country, 0) + users
+
+ return user_by_country
+
+
+def plot_usersXcountry(FOUNDATIONS_ID):
+ users_by_country = _run_usersXcountry_report(FOUNDATIONS_ID)
+
+ # Google API Country names do not match Cartopy Country Shapefile names
+ dict_api2cartopy = {
+ 'Tanzania': 'United Republic of Tanzania',
+ 'United States': 'United States of America',
+ 'Congo - Kinshasa': 'Democratic Republic of the Congo',
+ 'Bahamas': 'The Bahamas',
+ 'Timor-Leste': 'East Timor',
+ 'C\u00f4te d\u2019Ivoire': 'Ivory Coast',
+ 'Bosnia & Herzegovina': 'Bosnia and Herzegovina',
+ 'Serbia': 'Republic of Serbia',
+ 'Trinidad & Tobago': 'Trinidad and Tobago',
+ }
+
+ for key in dict_api2cartopy:
+ users_by_country[dict_api2cartopy[key]] = users_by_country.pop(key)
+
+ top_10_countries = sorted(users_by_country.items(), key=lambda item: item[1], reverse=True)[:10]
+ top_10_text = '\n'.join(
+ f'{country}: {_format_rounding(value)}' for i, (country, value) in enumerate(top_10_countries)
+ )
+
+ fig = plt.figure(figsize=(10, 4))
+ ax = plt.axes(projection=cartopy.crs.PlateCarree(), frameon=False)
+ ax.set_title('Pythia Foundations Users by Country', fontsize=15)
+
+ shapefile = cartopy.io.shapereader.natural_earth(category='cultural', resolution='110m', name='admin_0_countries')
+ reader = cartopy.io.shapereader.Reader(shapefile)
+ countries = reader.records()
+
+ colormap = plt.get_cmap('Blues')
+ newcmp = colors.ListedColormap(colormap(np.linspace(0.2, 1, 128)))
+ newcmp.set_extremes(under='grey')
+
+ norm = colors.LogNorm(vmin=1, vmax=max(users_by_country.values()))
+ mappable = cm.ScalarMappable(norm=norm, cmap=newcmp)
+
+ for country in countries:
+ country_name = country.attributes['SOVEREIGNT']
+ if country_name in users_by_country.keys():
+ facecolor = newcmp(norm(users_by_country[country_name]))
+ ax.add_geometries(
+ [country.geometry],
+ cartopy.crs.PlateCarree(),
+ facecolor=facecolor,
+ edgecolor='white',
+ linewidth=0.7,
+ norm=matplotlib.colors.LogNorm(),
+ )
+ else:
+ ax.add_geometries(
+ [country.geometry], cartopy.crs.PlateCarree(), facecolor='grey', edgecolor='white', linewidth=0.7
+ )
+
+ cax = fig.add_axes([0.1, -0.015, 0.67, 0.03])
+ cbar = fig.colorbar(mappable=mappable, cax=cax, spacing='uniform', orientation='horizontal', extend='min')
+ cbar.set_label('Unique Users')
+
+ props = dict(boxstyle='round', facecolor='white', edgecolor='white')
+ ax.text(1.01, 0.5, top_10_text, transform=ax.transAxes, fontsize=12, verticalalignment='center', bbox=props)
+
+ plt.tight_layout()
+ plt.savefig('portal/metrics/bycountry.png', bbox_inches='tight')
+
+
+if __name__ == '__main__':
+ get_total_users(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID)
+ plot_projects_this_year(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID)
+ plot_top_pages(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID)
+ plot_usersXcountry(FOUNDATIONS_ID)
diff --git a/.github/workflows/nightly-build.yaml b/.github/workflows/nightly-build.yaml
index f2ab7753..620234a6 100644
--- a/.github/workflows/nightly-build.yaml
+++ b/.github/workflows/nightly-build.yaml
@@ -6,7 +6,24 @@ on:
- cron: '0 0 * * *' # Daily βAt 00:00β
jobs:
+ automate-metrics:
+ runs-on: macos-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Automate Metrics
+ env:
+ PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
+ PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}
+ run: |
+ python -m venv analytics-api
+ source analytics-api/bin/activate
+ pip install google-analytics-data cartopy matplotlib
+
+ python .github/workflows/get-metrics.py
+ python .github/workflows/write-metrics-md.py
+
build:
+ needs: automate-metrics
if: ${{ github.repository_owner == 'ProjectPythia' }}
uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main
with:
@@ -20,3 +37,10 @@ jobs:
uses: ./.github/workflows/sphinx-link-checker.yaml
with:
path_to_source: 'portal'
+
+ deploy:
+ needs: build
+ uses: ProjectPythia/cookbook-actions/.github/workflows/deploy-book.yaml@main
+ with:
+ cname: projectpythia.org
+ publish_dir: 'portal/_build/html'
diff --git a/.github/workflows/publish-site.yaml b/.github/workflows/publish-site.yaml
index a8cd22f3..d342d19f 100644
--- a/.github/workflows/publish-site.yaml
+++ b/.github/workflows/publish-site.yaml
@@ -8,7 +8,24 @@ on:
workflow_dispatch:
jobs:
+ automate-metrics:
+ runs-on: macos-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Automate Metrics
+ env:
+ PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
+ PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}
+ run: |
+ python -m venv analytics-api
+ source analytics-api/bin/activate
+ pip install google-analytics-data cartopy matplotlib
+
+ python .github/workflows/get-metrics.py
+ python .github/workflows/write-metrics-md.py
+
build:
+ needs: automate-metrics
uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main
with:
environment_file: 'environment.yml'
diff --git a/.github/workflows/trigger-preview.yaml b/.github/workflows/trigger-preview.yaml
index 05463fcf..fcd9167b 100644
--- a/.github/workflows/trigger-preview.yaml
+++ b/.github/workflows/trigger-preview.yaml
@@ -8,7 +8,24 @@ on:
- completed
jobs:
+ automate-metrics:
+ runs-on: macos-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Automate Metrics
+ env:
+ PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
+ PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}
+ run: |
+ python -m venv analytics-api
+ source analytics-api/bin/activate
+ pip install google-analytics-data cartopy matplotlib
+
+ python .github/workflows/get-metrics.py
+ python .github/workflows/write-metrics-md.py
+
find-pull-request:
+ needs: automate-metrics
uses: ProjectPythia/cookbook-actions/.github/workflows/find-pull-request.yaml@main
deploy-preview:
needs: find-pull-request
diff --git a/.github/workflows/write-metrics-md.py b/.github/workflows/write-metrics-md.py
new file mode 100644
index 00000000..3de8abc9
--- /dev/null
+++ b/.github/workflows/write-metrics-md.py
@@ -0,0 +1,37 @@
+import json
+
+
+def process_user_data(json_file, top_pages, this_year, map, md_file):
+ with open(json_file, 'r') as f:
+ user_data = json.load(f)
+
+ with open(md_file, 'w') as f:
+ f.write('# Metrics \n\n')
+ now = user_data['Now']
+ f.write(f'Last Updated: {now}')
+ user_data.pop('Now')
+ f.write('\n\n')
+
+ headers = '| Project | Users |'
+ separator = '| ' + ' | '.join(['-----'] * 2) + ' |'
+ rows = []
+ for key in user_data.keys():
+ rows.append('| ' + key + ' | ' + user_data[key] + ' |')
+ table = '\n'.join([headers, separator] + rows)
+ f.write(table)
+ f.write('\n\n')
+
+ f.write(f'\n\n')
+ f.write(f'\n\n')
+ f.write(f'\n\n')
+
+ f.close()
+
+
+if __name__ == '__main__':
+ json_file = 'portal/metrics/user_metrics.json'
+ top_pages = 'metrics/toppages.png'
+ this_year = 'metrics/thisyear.png'
+ map = 'metrics/bycountry.png'
+ md_file = 'portal/metrics.md'
+ process_user_data(json_file, top_pages, this_year, map, md_file)
diff --git a/.gitignore b/.gitignore
index 1babb552..4fb65167 100644
--- a/.gitignore
+++ b/.gitignore
@@ -138,3 +138,8 @@ portal/code_of_conduct.md
portal/resource-gallery.md
portal/resource-gallery/*.md
resource-gallery-submission-input.json
+
+# Analytics
+.github/workflows/analytics-api/
+.github/workflows/*.json
+.github/workflows/*.png
diff --git a/portal/_templates/footer-menu.html b/portal/_templates/footer-menu.html
index 3a17455b..919cc9c3 100644
--- a/portal/_templates/footer-menu.html
+++ b/portal/_templates/footer-menu.html
@@ -8,6 +8,7 @@ {{ _("About") }}
{{ _("About Project Pythia") }}
{{ _("How to use Pythia Foundations") }}
{{ _("How to Cite") }}
+ {{ _("Metrics") }}