diff --git a/.gitea/workflows/create-weekly-analytics-stats.yaml b/.gitea/workflows/create-weekly-analytics-stats.yaml new file mode 100644 index 0000000..a797146 --- /dev/null +++ b/.gitea/workflows/create-weekly-analytics-stats.yaml @@ -0,0 +1,62 @@ +name: Create Weekly Analytics Stats + +on: + schedule: + # 03:00 UTC = 04:00 CET + - cron: "0 3 * * 1" + workflow_dispatch: + +jobs: + run-analytics: + runs-on: ubuntu + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.PUSH_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests otc-metadata + + - name: Run analytics for eu_de + env: + UMAMI_USERNAME: ${{ secrets.UMAMI_USERNAME }} + UMAMI_PASSWORD: ${{ secrets.UMAMI_PASSWORD }} + run: | + python ./tools/collect_statistics.py \ + --website-id "${{ secrets.UMAMI_WEBSITE_ID }}" \ + --cloud-environment "eu_de" \ + --environment "public" \ + --limit "10" + + - name: Run analytics for swiss + env: + UMAMI_USERNAME: ${{ secrets.UMAMI_USERNAME }} + UMAMI_PASSWORD: ${{ secrets.UMAMI_PASSWORD }} + run: | + python ./tools/collect_statistics.py \ + --website-id "${{ secrets.UMAMI_WEBSITE_ID }}" \ + --cloud-environment "swiss" \ + --environment "public" \ + --limit "10" + + - name: Commit and push results + run: | + git config --global user.name "gitea-actions[bot]" + git config --global user.email "actions@users.noreply.local" + git checkout -B analytics-update + git add otc_metadata/analytics/ + if git diff --cached --quiet; then + echo "No changes to commit" + else + git commit -m "chore: update analytics data [skip ci]" + git push origin analytics-update --force + fi diff --git a/.gitea/workflows/python-tox.yaml b/.gitea/workflows/python-tox.yaml new file mode 100644 index 0000000..e08c5d3 --- /dev/null +++ b/.gitea/workflows/python-tox.yaml @@ -0,0 +1,18 @@ +name: Run Tox Check + +on: + pull_request: + types: [opened, reopened, synchronize, edited] + +jobs: + tox-py312: + runs-on: ubuntu + steps: + - uses: https://github.com/opentelekomcloud-infra/github-actions/.github/actions/tox-py-test@v1 + + tox-pep8: + runs-on: ubuntu + steps: + - uses: https://github.com/opentelekomcloud-infra/github-actions/.github/actions/tox-py-test@v1 + with: + tox-env: pep8 diff --git a/.gitea/workflows/update_opensearch_filters.yaml b/.gitea/workflows/update_opensearch_filters.yaml new file mode 100644 index 0000000..13dfd68 --- /dev/null +++ b/.gitea/workflows/update_opensearch_filters.yaml @@ -0,0 +1,32 @@ +name: Updates Opensearch filters + +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + update-opensearch-filters: + if: github.event.pull_request.merged == true + runs-on: ubuntu + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies and local package otc-metadata package + run: | + python -m pip install --upgrade pip + pip install . -r tools-requirements.txt + + - name: Update swiss and eu_de Opensearch indizies + run: | + python tools/index_metadata.py --hosts ${{ secrets.OPENSEARCH_HOST1 }} --target-environment public --index search_index_de --cloud-environment eu_de --username ${{ secrets.OPENSEARCH_USER }} --password ${{ secrets.OPENSEARCH_PW }} --delete-index + python tools/index_metadata.py --hosts ${{ secrets.OPENSEARCH_HOST1 }} --target-environment public --index search_index_swiss --cloud-environment swiss --username ${{ secrets.OPENSEARCH_USER }} --password ${{ secrets.OPENSEARCH_PW }} --delete-index diff --git a/otc_metadata/analytics/__init__.py b/otc_metadata/analytics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/otc_metadata/analytics/data.py b/otc_metadata/analytics/data.py new file mode 100644 index 0000000..131f6ff --- /dev/null +++ b/otc_metadata/analytics/data.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import json + +analytics_path = Path("otc_metadata/analytics/public") + +cloud_environments = [ + 'eu_de', + 'swiss' +] +analytics_data = {k: [] for k in cloud_environments} + +# Open and read the json data files +for env in cloud_environments: + file_path = analytics_path.joinpath(f"{env}.json") + with open(file_path, 'r') as file: + analytics_data[env] = json.load(file) + + +class AnalyticsData(object): + """Encapsulate OTC Analystics data""" + + def __init__(self): + self._analytics_data = analytics_data + + def all_analytics_data(self): + """returns all analytics data""" + return self._analytics_data + + def analytics_data_by_cloud_environment(self, cloud_environment): + """returns all analytics data""" + if cloud_environment and cloud_environment in self._analytics_data: + return self._analytics_data[cloud_environment] + else: + raise ValueError(f"cloud_environment '{cloud_environment}' does not exist.") diff --git a/otc_metadata/analytics/public/eu_de.json b/otc_metadata/analytics/public/eu_de.json new file mode 100644 index 0000000..9c2ac3c --- /dev/null +++ b/otc_metadata/analytics/public/eu_de.json @@ -0,0 +1,12 @@ +[ + "evs", + "ims", + "ecs", + "cce", + "obs", + "rds", + "sfs", + "iam", + "elb", + "apig" +] \ No newline at end of file diff --git a/otc_metadata/analytics/public/swiss.json b/otc_metadata/analytics/public/swiss.json new file mode 100644 index 0000000..59ca59b --- /dev/null +++ b/otc_metadata/analytics/public/swiss.json @@ -0,0 +1,12 @@ +[ + "evs", + "ims", + "ecs", + "cce", + "obs", + "rds", + "iam", + "elb", + "cbr", + "vpc" +] \ No newline at end of file diff --git a/otc_metadata/data/documents/bcc-api-ref.yaml b/otc_metadata/data/documents/bcc-api-ref.yaml index 90eaccf..134e7a1 100644 --- a/otc_metadata/data/documents/bcc-api-ref.yaml +++ b/otc_metadata/data/documents/bcc-api-ref.yaml @@ -6,6 +6,7 @@ rst_location: api-ref/source service_type: bcc title: API Reference type: api-ref +disable_import: true cloud_environments: - name: eu_de visibility: internal diff --git a/otc_metadata/data/documents/bcc-umn.yaml b/otc_metadata/data/documents/bcc-umn.yaml index 45a06d4..2b53210 100644 --- a/otc_metadata/data/documents/bcc-umn.yaml +++ b/otc_metadata/data/documents/bcc-umn.yaml @@ -6,6 +6,7 @@ rst_location: umn/source service_type: bcc title: User Guide type: umn +disable_import: true cloud_environments: - name: eu_de visibility: internal diff --git a/otc_metadata/data/documents/bd-umn.yaml b/otc_metadata/data/documents/bd-umn.yaml deleted file mode 100644 index fec0e90..0000000 --- a/otc_metadata/data/documents/bd-umn.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -hc_location: usermanual/bd -html_location: docs/bd/umn -link: /business-dashboard/umn/ -rst_location: umn/source -service_type: bd -title: User Guide -type: umn -cloud_environments: - - name: eu_de - visibility: public - pdf_visibility: hidden - pdf_enabled: false diff --git a/otc_metadata/data/documents/sd-umn.yaml b/otc_metadata/data/documents/sd-umn.yaml index 578e30c..18e010d 100644 --- a/otc_metadata/data/documents/sd-umn.yaml +++ b/otc_metadata/data/documents/sd-umn.yaml @@ -6,6 +6,7 @@ rst_location: umn/source service_type: sd title: User Guide type: umn +disable_import: true cloud_environments: - name: swiss visibility: public diff --git a/otc_metadata/data/repositories/bd.yaml b/otc_metadata/data/repositories/bd.yaml deleted file mode 100644 index 80a45aa..0000000 --- a/otc_metadata/data/repositories/bd.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -service_type: bd -repositories: - - environment: internal - repo: docs/business-dashboard - type: gitea - cloud_environments: - - eu_de diff --git a/otc_metadata/data/repositories/dms-rocketmq.yaml b/otc_metadata/data/repositories/dms_rocketmq.yaml similarity index 100% rename from otc_metadata/data/repositories/dms-rocketmq.yaml rename to otc_metadata/data/repositories/dms_rocketmq.yaml diff --git a/otc_metadata/data/services/bd.yaml b/otc_metadata/data/services/bd.yaml deleted file mode 100644 index 325e61b..0000000 --- a/otc_metadata/data/services/bd.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -service_category: other -service_title: Business Dashboard -service_type: bd -service_uri: business-dashboard -is_global: false -teams: - - name: docs-dashboard-rw - permission: write -cloud_environments: - - name: eu_de - visibility: internal diff --git a/otc_metadata/data/services/dms-rocketmq.yaml b/otc_metadata/data/services/dms_rocketmq.yaml similarity index 100% rename from otc_metadata/data/services/dms-rocketmq.yaml rename to otc_metadata/data/services/dms_rocketmq.yaml diff --git a/otc_metadata/services.py b/otc_metadata/services.py index b35dacf..851e57f 100644 --- a/otc_metadata/services.py +++ b/otc_metadata/services.py @@ -463,3 +463,25 @@ class Services(object): res.sort(key=lambda x: x.get("name", "").lower()) return res + + def all_services_by_cloud_environment(self, cloud_environment, environments): + """Retrieve all services filtered by cloud_environment + """ + res = [] + for srv in self.all_services: + if environments and cloud_environment: + for srv_cloud_environment in srv["cloud_environments"]: + if srv_cloud_environment["name"] == cloud_environment: + for environment in environments: + if srv_cloud_environment["visibility"] == environment: + res.append(srv) + else: + continue + else: + raise Exception("No cloud_environment or environments " + "specified in function all_services_by_cloud_environment.") + + # Sort services + res.sort(key=lambda x: x.get("service_title", "").lower()) + + return res diff --git a/tools/collect_statistics.py b/tools/collect_statistics.py new file mode 100755 index 0000000..4b1405d --- /dev/null +++ b/tools/collect_statistics.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests +import json +from datetime import datetime, timedelta +import os +import otc_metadata.services +import argparse +import logging + +# ===== Configuration ===== +USERNAME = os.getenv("UMAMI_USERNAME") +PASSWORD = os.getenv("UMAMI_PASSWORD") +OUTPUT_FILE = "stats.json" + +# ===== Data ===== +blacklist = [ + "ed", + "sd" +] + +# ===== Logger ===== + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + + +def parse_args(): + """ + Command-line arguments + """ + parser = argparse.ArgumentParser(description="Analytics Script") + parser.add_argument( + "--base-url", + default="https://analytics.otc-service.com", + help="Base_Url of analytics server" + ) + parser.add_argument( + "--cloud-environment", + default="eu_de", + choices=['eu_de', 'swiss'], + help="Cloud Environments (default: eu_de)" + ) + parser.add_argument( + "--environment", + default=['public'], + nargs='+', + choices=['public', 'internal', 'hidden'], + help="Environments (default: ['public'])" + ) + parser.add_argument( + "--limit", + type=int, + default=10, + help="Result count" + ) + parser.add_argument( + "--website-id", + required=True, + help="Umami Website ID" + ) + + return parser.parse_args() + + +def get_umami_token(base_url): + """Get Bearer-Token from Umami-API.""" + url = f"{base_url}/api/auth/login" + response = requests.post(url, json={"username": USERNAME, "password": PASSWORD}) + response.raise_for_status() + return response.json().get("token") + + +def get_4_weeks_range(): + """Calculates start and end of 4 weeks range in UNIX timestamp format.""" + end_date = datetime.utcnow() + start_date = end_date - timedelta(weeks=4) + + start_ts = int(start_date.timestamp() * 1000) + end_ts = int(end_date.timestamp() * 1000) + return start_ts, end_ts + + +def fetch_pageviews(token, start_ts, end_ts, website_id, base_url): + """Retrieves statistics from API server.""" + headers = {"Authorization": f"Bearer {token}"} + url = f"{base_url}/api/websites/{website_id}/metrics" + params = { + "type": "url", + "startAt": start_ts, + "endAt": end_ts + } + + response = requests.get(url, headers=headers, params=params) + response.raise_for_status() + return response + + +def filter_unique_service_types(stats, cloud_environment, environment, limit): + """ + Filter stats and return unique service_type values that exist in metadata, + skip blacklisted service types, and limit results to `limit` entries. + """ + services = otc_metadata.services.Services().all_services_by_cloud_environment( + cloud_environment=cloud_environment, + environments=environment + ) + + # Map service_uri -> service_type + uri_to_type = {s["service_uri"]: s["service_type"] for s in services} + + seen_types = set() + filtered = [] + + for entry in stats.json(): + url_path = entry["x"] + + for service_uri, service_type in uri_to_type.items(): + if f"/{service_uri}" in url_path and service_type not in seen_types: + if service_type in blacklist: + continue + + filtered.append(service_type) + seen_types.add(service_type) + + if len(filtered) >= limit: + return filtered + break + + return filtered + + +def save_to_file(data, environment, cloud_environment): + """ + Saves data in the folder ./analytics//.json + """ + folder = os.path.join("otc_metadata", "analytics", environment) + os.makedirs(folder, exist_ok=True) + + filename = os.path.join(folder, f"{cloud_environment}.json") + abs_path = os.path.abspath(filename) + + with open(filename, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + logger.info(f"✅ Data saved in: {abs_path}") + + +def main(): + try: + args = parse_args() + token = get_umami_token(base_url=args.base_url) + start_ts, end_ts = get_4_weeks_range() + stats = fetch_pageviews(token, start_ts, end_ts, website_id=args.website_id, base_url=args.base_url) + filtered_stats = filter_unique_service_types( + stats=stats, + cloud_environment=args.cloud_environment, + environment=args.environment, + limit=args.limit + ) + save_to_file( + data=filtered_stats, + environment=args.environment[0], + cloud_environment=args.cloud_environment + ) + except Exception as e: + logger.error(f"Error: {e}") + raise + + +if __name__ == "__main__": + main()