diff --git a/analytics/public/eu_de.json b/analytics/public/eu_de.json new file mode 100644 index 0000000..649d4c0 --- /dev/null +++ b/analytics/public/eu_de.json @@ -0,0 +1,12 @@ +[ + "evs", + "ecs", + "ims", + "cce", + "obs", + "rds", + "iam", + "sfs", + "elb", + "config" +] \ No newline at end of file diff --git a/analytics/public/swiss.json b/analytics/public/swiss.json new file mode 100644 index 0000000..885f1da --- /dev/null +++ b/analytics/public/swiss.json @@ -0,0 +1,12 @@ +[ + "evs", + "ecs", + "ims", + "cce", + "obs", + "rds", + "iam", + "elb", + "cbr", + "vpn" +] \ No newline at end of file diff --git a/stats.json b/stats.json new file mode 100644 index 0000000..649d4c0 --- /dev/null +++ b/stats.json @@ -0,0 +1,12 @@ +[ + "evs", + "ecs", + "ims", + "cce", + "obs", + "rds", + "iam", + "sfs", + "elb", + "config" +] \ No newline at end of file diff --git a/tools/collect_statistics.py b/tools/collect_statistics.py index 7106b7a..ca93948 100644 --- a/tools/collect_statistics.py +++ b/tools/collect_statistics.py @@ -15,17 +15,32 @@ import requests import json -import time from datetime import datetime, timedelta import os import otc_metadata.services import argparse +import logging # ===== Configuration ===== USERNAME = os.getenv("UMAMI_USERNAME") PASSWORD = os.getenv("UMAMI_PASSWORD") OUTPUT_FILE = "stats.json" +# ===== Data ===== +blacklist = [ + "ed", + "sd" +] + +# ===== Logger ===== + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + + def parse_args(): """ Command-line arguments @@ -36,6 +51,12 @@ def parse_args(): default="https://analytics.otc-service.com", help="Base_Url of analytics server" ) + parser.add_argument( + "--cloud-environment", + default="eu_de", + choices=['eu_de', 'swiss'], + help="Cloud Environments (default: eu_de)" + ) parser.add_argument( "--environment", default=['public'], @@ -44,10 +65,10 @@ def parse_args(): help="Environments (default: ['public'])" ) parser.add_argument( - "--cloud-environment", - default="eu_de", - choices=['eu_de', 'swiss'], - help="Cloud Environments (default: eu_de)" + "--limit", + type=int, + default=10, + help="Result count" ) parser.add_argument( "--website-id", @@ -57,6 +78,7 @@ def parse_args(): return parser.parse_args() + def get_umami_token(base_url): """Get Bearer-Token from Umami-API.""" url = f"{base_url}/api/auth/login" @@ -64,6 +86,7 @@ def get_umami_token(base_url): response.raise_for_status() return response.json().get("token") + def get_4_weeks_range(): """Calculates start and end of 4 weeks range in UNIX timestamp format.""" end_date = datetime.utcnow() @@ -73,6 +96,7 @@ def get_4_weeks_range(): end_ts = int(end_date.timestamp() * 1000) return start_ts, end_ts + def fetch_pageviews(token, start_ts, end_ts, website_id, base_url): """Retrieves statistics from API server.""" headers = {"Authorization": f"Bearer {token}"} @@ -87,38 +111,78 @@ def fetch_pageviews(token, start_ts, end_ts, website_id, base_url): response.raise_for_status() return response -def filter_stats_by_existing_services( - stats, cloud_environment, environment): - """Filter the stats to services which are existing""" + +def filter_unique_service_types(stats, cloud_environment, environment, limit): + """ + Filter stats and return unique service_type values that exist in metadata, + skip blacklisted service types, and limit results to `limit` entries. + """ services = otc_metadata.services.Services().all_services_by_cloud_environment( cloud_environment=cloud_environment, - environments=environment) - - print(json.dumps(services)) + environments=environment + ) - res = services - return res + # Map service_uri -> service_type + uri_to_type = {s["service_uri"]: s["service_type"] for s in services} + + seen_types = set() + filtered = [] + + for entry in stats.json(): + url_path = entry["x"] + + for service_uri, service_type in uri_to_type.items(): + if f"/{service_uri}" in url_path and service_type not in seen_types: + if service_type in blacklist: + continue + + filtered.append(service_type) + seen_types.add(service_type) + + if len(filtered) >= limit: + return filtered + break + + return filtered + + +def save_to_file(data, environment, cloud_environment): + """ + Saves data in the folder ./analytics//.json + """ + folder = os.path.join("analytics", environment) + os.makedirs(folder, exist_ok=True) + + filename = os.path.join(folder, f"{cloud_environment}.json") + abs_path = os.path.abspath(filename) + + with open(filename, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + logger.info(f"✅ Data saved in: {abs_path}") -def save_to_file(data, filename): - """Saves collected data in a file.""" - print(data) - # with open(filename, "w", encoding="utf-8") as f: - # json.dump(data, f, indent=2, ensure_ascii=False) def main(): - args = parse_args() - token = get_umami_token(base_url=args.base_url) - start_ts, end_ts = get_4_weeks_range() - stats = fetch_pageviews(token, start_ts, end_ts, website_id=args.website_id, base_url=args.base_url) - filtered_stats = filter_stats_by_existing_services( - stats=stats, - cloud_environment=args.cloud_environment, - environment=args.environment + try: + args = parse_args() + token = get_umami_token(base_url=args.base_url) + start_ts, end_ts = get_4_weeks_range() + stats = fetch_pageviews(token, start_ts, end_ts, website_id=args.website_id, base_url=args.base_url) + filtered_stats = filter_unique_service_types( + stats=stats, + cloud_environment=args.cloud_environment, + environment=args.environment, + limit=args.limit + ) + save_to_file( + data=filtered_stats, + environment=args.environment[0], + cloud_environment=args.cloud_environment + ) + except Exception as e: + logger.error(f"Error: {e}") + raise - ) - - save_to_file(filtered_stats, OUTPUT_FILE) - print(f"Statistics saved in {OUTPUT_FILE}") if __name__ == "__main__": - main() \ No newline at end of file + main()