From a4dd45706222ee3b8a4abcfd46838d3a33947035 Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Wed, 9 Nov 2022 17:14:49 +0100 Subject: [PATCH] update scripts for latest templates --- otc_metadata/data/services.yaml | 27 +++---- otc_metadata/services.py | 3 +- otc_metadata/templates/conf.py.j2 | 20 +++-- .../templates/doc_requirements.txt.j2 | 4 + otc_metadata/templates/tox.ini.j2 | 77 ++++++++++++++++++ tools/generate_doc_confpy.py | 74 ++++++++++++++--- tools/update_zuul_project_configs.py | 79 ++++++++++++++++--- 7 files changed, 241 insertions(+), 43 deletions(-) create mode 100644 otc_metadata/templates/doc_requirements.txt.j2 create mode 100644 otc_metadata/templates/tox.ini.j2 diff --git a/otc_metadata/data/services.yaml b/otc_metadata/data/services.yaml index a10dc14..c9aa12f 100644 --- a/otc_metadata/data/services.yaml +++ b/otc_metadata/data/services.yaml @@ -74,9 +74,10 @@ documents: - hc_location: bpicg/bms html_location: docs/bms/image-creation-guide pdf_name: bms-image-creation-guide - rst_location: doc/image-creation-guide + rst_location: doc/image-creation-guide/source service_type: bms title: Private Image Creation Guide + type: image-creation-guide - hc_location: usermanual/bms html_location: docs/bms/umn link: /umn/bare-metal-server/ @@ -112,7 +113,7 @@ documents: - hc_location: usermanual2/cce html_location: docs/cce/umn link: /umn/cloud-container-engine/ - pdf_name: cce-umn + # pdf_name: cce-umn rst_location: umn/source service_type: cce title: User Guide @@ -355,6 +356,7 @@ documents: rst_location: doc/tool/source service_type: dws title: Tool Guide + type: tool-guide - hc_location: usermanual/dws html_location: docs/dws/umn link: /umn/data-warehouse-service/ @@ -392,6 +394,7 @@ documents: rst_location: umn/source service_type: edb title: User Guide + type: manual - hc_location: api/eip html_location: docs/eip/api-ref link: /api-ref/elastic-ip/ @@ -509,9 +512,10 @@ documents: type: dev - html_location: docs/ims/public-images pdf_name: ims-public-images - rst_location: docs/ims/public-images + rst_location: doc/public-images/source service_type: ims title: Public Image Introduction + type: public-images - hc_location: usermanual/ims html_location: docs/ims/umn link: /umn/image-management-service/ @@ -579,9 +583,10 @@ documents: - hc_location: cmpntguide/mrs html_location: docs/mrs/component-operation-guide pdf_name: mrs-component-operation-guide - rst_location: docs/mrs/component-operation-guide + rst_location: doc/component-operation-guide/source service_type: mrs title: Component Operation Guide + type: operation-guide - hc_location: usermanual/mrs html_location: docs/mrs/umn link: /umn/mapreduce-service/ @@ -628,13 +633,7 @@ documents: rst_location: doc/s3api/source service_type: obs title: S3 API Reference - - doc_type: s3cmd - hc_location: ugs3cmd/obs - html_location: docs/obs/s3cmd - pdf_name: obs-s3cmd - rst_location: doc/s3cmd/source - service_type: obs - title: Third-Party Client (s3cmd) + type: s3api - hc_location: usermanual/obs html_location: docs/obs/umn link: /umn/object-storage-service/ @@ -757,12 +756,6 @@ documents: service_type: sfs title: API Reference type: api-ref - - hc_location: api_cli/sfs - html_location: docs/sfs/cli - pdf_name: sfs-cli - rst_location: doc/cli/source - service_type: sfs - title: Manila Client Reference - hc_location: usermanual/sfs html_location: docs/sfs/umn link: /umn/scalable-file-service/ diff --git a/otc_metadata/services.py b/otc_metadata/services.py index 9fa4ad7..fbb7c6f 100644 --- a/otc_metadata/services.py +++ b/otc_metadata/services.py @@ -195,8 +195,9 @@ class Services(object): rst_location=doc["rst_location"], title=doc["title"], type=doc.get("type", "dummy"), - pdf_name=doc["pdf_name"], ) + if "pdf_name" in doc: + doc_res["pdf_name"] = doc["pdf_name"] if "hc_location" in doc: doc_res["hc_location"] = doc["hc_location"] srv_res["docs"].append(doc_res) diff --git a/otc_metadata/templates/conf.py.j2 b/otc_metadata/templates/conf.py.j2 index bd7f8b6..1e04e55 100644 --- a/otc_metadata/templates/conf.py.j2 +++ b/otc_metadata/templates/conf.py.j2 @@ -27,12 +27,12 @@ otcdocs_auto_version = False project = '{{ project }}' otcdocs_repo_name = '{{ repo_name }}' # Those variables are required for edit/bug links -{% if git_fqdn -%} +{%- if git_fqdn %} otcdocs_git_fqdn = '{{ git_fqdn }}' -{% endif %} -{% if git_type -%} +{%- endif %} +{%- if git_type %} otcdocs_git_type = '{{ git_type }}' -{% endif %} +{%- endif %} # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -82,6 +82,15 @@ html_theme = 'otcdocs' # further. For a list of options available for each theme, see the # documentation. html_theme_options = { +{%- if html_options is defined -%} +{%- for (k, v) in html_options.items() %} +{%- if v is boolean %} + "{{ k }}": {{ v }}, +{%- else %} + "{{ k }}": "{{ v }}", +{%- endif %} +{%- endfor %} +{%- endif %} } # The name for this set of Sphinx documents. If None, it defaults to @@ -97,10 +106,11 @@ html_static_path = ['_static'] # -- Options for PDF output -------------------------------------------------- latex_documents = [ -{% if pdf_name is defined -%} +{%- if pdf_name is defined %} ('index', '{{ pdf_name }}.tex', u'{{ title }}', u'OpenTelekomCloud', 'manual'), {% endif -%} ] + diff --git a/otc_metadata/templates/doc_requirements.txt.j2 b/otc_metadata/templates/doc_requirements.txt.j2 new file mode 100644 index 0000000..a61c706 --- /dev/null +++ b/otc_metadata/templates/doc_requirements.txt.j2 @@ -0,0 +1,4 @@ +sphinx>=2.0.0,!=2.1.0 # BSD +otcdocstheme # Apache-2.0 +# releasenotes +reno>=3.1.0 # Apache-2.0 diff --git a/otc_metadata/templates/tox.ini.j2 b/otc_metadata/templates/tox.ini.j2 new file mode 100644 index 0000000..9e02e70 --- /dev/null +++ b/otc_metadata/templates/tox.ini.j2 @@ -0,0 +1,77 @@ +[tox] +minversion = 3.1 +envlist = py39,pep8 +skipsdist = True +ignore_basepython_conflict = True + +[testenv] +usedevelop = True +install_command = pip install {opts} {packages} +deps = + -r{toxinidir}/requirements.txt +commands = stestr run {posargs} + stestr slowest + +[testenv:pep8] +commands = + doc8 doc/source README.rst + +[testenv:venv] +deps = + -r{toxinidir}/requirements.txt +commands = {posargs} + +[testenv:docs] +commands = +{%- for doc in docs %} + {[testenv:{{ doc.type }}]commands} +{%- endfor %} + +[testenv:docs-pdf] +whitelist_externals = + mkdir +commands = + mkdir -p doc/build/pdf +{%- for doc in docs %} +{%- if doc.pdf_name %} + {[testenv:{{ doc.type }}-pdf-docs]commands} +{%- endif %} +{%- endfor %} + +{% for doc in docs -%} +{% set loc = doc.rst_location | replace('/source', '') %} +[testenv:{{ doc.type }}] +whitelist_externals = + cp + mkdir +commands = + sphinx-build -W --keep-going -b html {{ loc }}/source doc/build/html/{{ doc.type }} +{%- if doc.type == 'api-ref' %} + mkdir -p api-ref/build/html + cp -av doc/build/html/api-ref api-ref/build/html +{%- elif doc.type == 'umn' %} + mkdir -p umn/build/html + cp -av doc/build/html/umn umn/build/html +{%- elif doc.type == 'dev-guide' %} + mkdir -p dev_guide/build/html + cp -av doc/build/html/dev-guide dev_guide/build/html +{%- endif %} + +{% if doc.pdf_name -%} +[testenv:{{ doc.type }}-pdf-docs] +whitelist_externals = + rm + mkdir + make + bash + cp +commands = + rm -rf {{ loc }}/build/pdf + sphinx-build -W --keep-going -b latex {{ loc }}/source {{ loc }}/build/pdf/ + bash -c "for f in {{ loc }}/build/pdf/*.gif; do convert $f $\{f/%gif/png\}; done || true" + bash -c "for f in {{ loc }}/build/pdf/*.tex; do sed -iorig 's/\.gif//g' $f; done" + make -C {{ loc }}/build/pdf + mkdir -p doc/build/pdf + cp {{ loc }}/build/pdf/{{ doc.pdf_name }}.pdf doc/build/pdf/ +{% endif %} +{% endfor %} diff --git a/tools/generate_doc_confpy.py b/tools/generate_doc_confpy.py index b6eb979..a86f36c 100644 --- a/tools/generate_doc_confpy.py +++ b/tools/generate_doc_confpy.py @@ -15,11 +15,9 @@ import argparse import logging -import os import pathlib import requests import subprocess -import warnings from git import exc from git import Repo @@ -54,6 +52,8 @@ def process_repositories(args, service): loader=PackageLoader("otc_metadata"), autoescape=select_autoescape()) conf_py_template = env.get_template("conf.py.j2") + tox_ini_template = env.get_template("tox.ini.j2") + doc_requirements_template = env.get_template("doc_requirements.txt.j2") for repo in service["repositories"]: logging.debug(f"Processing repository {repo}") @@ -116,17 +116,21 @@ def process_repositories(args, service): if args.branch_force: logging.debug("Dropping current branch") try: - repo_to.delete_head(branch_name) - except exc.GitCommandError: + repo_to.delete_head(branch_name, force=True) + repo_to.delete_head(f"refs/heads/{args.branch_name}", force=True) + except exc.GitCommandError as e: + print(e) pass try: new_branch = repo_to.create_head(branch_name, 'main') - except Exception: - logging.warning(f"Skipping service {service}") + except Exception as e: + logging.warning(f"Skipping service {service} due to {e}") return new_branch.checkout() - for doc in data.docs_by_service_type(service["service_type"]): + service_docs = list(data.docs_by_service_type(service["service_type"])) + + for doc in service_docs: logging.debug(f"Analyzing document {doc}") conf_py_path = pathlib.Path(copy_to, doc['rst_location'], 'conf.py') @@ -136,26 +140,65 @@ def process_repositories(args, service): context = dict( repo_name=target_repo["repo"], project=service["service_title"], - pdf_name=doc["pdf_name"], + # pdf_name=doc["pdf_name"], title=f"{service['service_title']} - {doc['title']}" ) + if "pdf_name" in doc: + context["pdf_name"] = doc["pdf_name"] if git_fqdn: context["git_fqdn"] = git_fqdn if target_repo.get("type") != "github": context["git_type"] = target_repo["type"] + if args.target_environment == "internal": + context["html_options"] = dict( + disable_search=True, + site_name="Internal Documentation Portal", + logo_url="https://docs-int.otc-service.com", + ) conf_py_content = conf_py_template.render(**context) with open(conf_py_path, 'w', encoding='utf-8', newline='') as out: logging.debug(f"Generating {conf_py_path} from template...") out.write(conf_py_content) repo_to.index.add([doc["rst_location"]]) + if args.update_tox: + context = dict( + docs=[] + ) + for doc in service_docs: + if doc["type"] == "dev": + doc["type"] = "dev-guide" + context["docs"].append(doc) + + tox_ini_content = tox_ini_template.render(**context) + tox_ini_path = pathlib.Path(copy_to, 'tox.ini') + doc_requirements_content = doc_requirements_template.render(**context) + doc_requirements_path = pathlib.Path(copy_to, "doc", 'requirements.txt') + doc_requirements_path.parent.mkdir(parents=True, exist_ok=True) + with open(tox_ini_path, 'w', encoding='utf-8', newline='') as out: + logging.debug(f"Generating {tox_ini_path} from template...") + out.write(tox_ini_content) + repo_to.index.add(["tox.ini"]) + with open(doc_requirements_path, 'w', encoding='utf-8', newline='') as out: + logging.debug(f"Generating {doc_requirements_path} from template...") + out.write(doc_requirements_content) + repo_to.index.add(["doc/requirements.txt"]) + + + if len(repo_to.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug( + "No changes for service %s required" % service["service_type"]) + return repo_to.index.commit( ( - f"Update conf.py file\n\n" + f"Update tox.ini && conf.py file\n\n" "Performed-by: gitea/infra/otc-metadata/tools/generate_doc_confpy.py" ) ) - push_args = ("--set-upstream", "origin", branch_name) + push_args = ["--set-upstream", "origin", branch_name] + if args.branch_force: + push_args.append("--force") repo_to.git.push(*push_args) if 'github' in url_to: subprocess.run( @@ -168,6 +211,7 @@ def process_repositories(args, service): args, repo["repo"], dict( + title="Update Docs configuration", head=branch_name, ), ) @@ -191,7 +235,7 @@ def open_pr(args, repository, pr_data): ) if rsp.status_code != 201: print(rsp.text) - print(f"Going to open PR with title {pr_data['title']} in {repository}") + #print(f"Going to open PR with title {pr_data['title']} in {repository}") def main(): @@ -206,6 +250,11 @@ def main(): '--service-type', help='Service to update' ) + parser.add_argument( + '--update-tox', + action='store_true', + help='Whether to update tox.ini.' + ) parser.add_argument( '--work-dir', required=True, @@ -232,6 +281,9 @@ def main(): else: services = data.all_services + if args.token: + api_session.headers.update({"Authorization": f"token {args.token}"}) + for service in services: process_repositories(args, service) diff --git a/tools/update_zuul_project_configs.py b/tools/update_zuul_project_configs.py index 25e936c..1ed8fa6 100644 --- a/tools/update_zuul_project_configs.py +++ b/tools/update_zuul_project_configs.py @@ -25,6 +25,7 @@ from git import exc from git import Repo from git import SymbolicReference +from ruamel.yaml import CommentedMap from ruamel.yaml import YAML import otc_metadata.services @@ -115,7 +116,7 @@ def process_repositories(args, service): if args.branch_force: logging.debug("Dropping current branch") try: - git_repo.delete_head(branch_name) + git_repo.delete_head(branch_name, force=True) except exc.GitCommandError: pass try: @@ -127,6 +128,8 @@ def process_repositories(args, service): (zuul_config, zuul_file_name) = load_zuul_config(repo_dir) zuul_templates = None + zuul_jobs = dict() + zuul_new_jobs = list() zuul_config_updated = False for item in zuul_config: if "project" in item.keys(): @@ -134,6 +137,10 @@ def process_repositories(args, service): zuul_templates = project.setdefault("templates", []) if not zuul_templates: zuul_templates = [] + elif "job" in item.keys(): + job = item["job"] + zuul_jobs[job["name"]] = job + print(f"Existing jobs {zuul_jobs}") if "helpcenter-base-jobs" not in zuul_templates: zuul_templates.append("helpcenter-base-jobs") zuul_config_updated = True @@ -145,30 +152,84 @@ def process_repositories(args, service): logging.debug(f"Analyzing document {doc}") if not doc.get("type"): continue - template_name = f"{doc['type']}{job_suffix}" - if template_name not in zuul_templates: - zuul_templates.append(template_name) + if doc["type"] == "dev": + doc_type = "dev-guide" + else: + doc_type = doc["type"] + template_name = f"{doc_type}{job_suffix}" + if doc_type in ["api-ref", "umn", "dev-guide"]: + if template_name not in zuul_templates: + zuul_templates.append(template_name) + else: + job_name = f"build-otc-{doc['service_type']}-{doc_type}" + if job_name not in zuul_jobs: + zuul_config_updated = True + zuul_new_jobs.append( + dict( + job=dict( + name=job_name, + parent="otc-tox-docs", + description=( + f"Build {doc_type} document using tox" + ), + files=[ + f"^{doc['rst_location']}/.*" + ], + vars=dict( + tox_envlist=doc_type + ) + ) + ) + ) if zuul_config_updated: - print("updating") + for new_job in zuul_new_jobs: + zuul_config.insert(0, new_job) + for item in zuul_config: if "project" in item.keys(): project = item["project"] project["templates"] = zuul_templates + # Ensure new jobs are in check + if len(zuul_new_jobs) > 0: + project.setdefault( + "check", + CommentedMap(jobs=[]) + ) + project["check"].yaml_set_comment_before_after_key( + key="jobs", + indent=6, + before=( + "Separate documents are rendered in check, " + "while published through regular " + "otc-tox-docs job part of the basic template" + ) + ) + project["check"]["jobs"].extend( + [x["job"]["name"] for x in zuul_new_jobs]) - # yaml.indent(offset=2, sequence=2) + + #yaml.indent(offset=2, sequence=2) with open(zuul_file_name, "w") as f: yaml.dump(zuul_config, f) git_repo.index.add([zuul_file_name.name]) + if len(git_repo.index.diff("HEAD")) == 0: + # Nothing to commit + logging.debug( + "No changes for service %s required" % + service["service_type"]) + return + git_repo.index.commit( ( "Update zuul.yaml file\n\n" "Performed-by: gitea/infra/otc-metadata/tools/update_zuul_project_config.py" ) ) - push_args = ("--set-upstream", "origin", branch_name) - #if args.branch_force: - # push_args.add("--force") + push_args = ["--set-upstream", "origin", branch_name] + if args.branch_force: + push_args.append("--force") + git_repo.git.push(*push_args) if repo["type"] == "github": subprocess.run(