Rename tox target for pdf build

zuul jobs require target to be called pdf-docs instead of docs-pdf

.stestr.conf

@@ -1,3 +1,3 @@
 [DEFAULT]
-test_path=./otc-metadata/tests
+test_path=./otc_metadata/tests/
 top_dir=./

otc_metadata/data/__init__.py

@@ -28,6 +28,7 @@ def read_data(filename):
     with open(filepath, 'r') as fd:
         return yaml.safe_load(fd)
 
+
 def rewrite_data(filename, data):
     """Rewrites data formatting it
 

otc_metadata/docs.py

@@ -15,7 +15,7 @@ import copy
 
 import otc_metadata.data
 
-__all__ = ['Docs']
+__all__ = ['Service']
 
 BUILTIN_DATA = otc_metadata.data.read_data('docs.yaml')
 

otc_metadata/services.py

@@ -40,7 +40,8 @@ class Services(object):
 
         # sort docs list by <service_type>_<title>
         self._service_data["documents"] = sorted(
-            sorted_docs, key=lambda x: f"{x.get('service_type')}{x.get('title')}"
+            sorted_docs,
+            key=lambda x: f"{x.get('service_type')}{x.get('title')}"
         )
         # sort services by <service_type>_<service_title>
         self._service_data["services"] = sorted(
@@ -183,7 +184,10 @@ class Services(object):
                 ]
             if "repositories" in srv and environment:
                 for repo in srv["repositories"]:
-                    if "environment" in repo and repo["environment"] == environment:
+                    if (
+                        "environment" in repo
+                        and repo["environment"] == environment
+                    ):
                         srv_res["repository"] = repo["repo"]
             for doc in self.all_docs:
                 if (
@@ -205,3 +209,22 @@ class Services(object):
                 doc_struct[srv["service_category"]].append(srv_res)
 
         return dict(categories=doc_struct)
+
+    def get_service_with_docs_by_service_type(self, service_type):
+        """Retrieve service and service docs by service_type
+
+        :param str service_type: Filter by service_type
+        """
+        res = dict()
+        res['service'] = {}
+        docs = []
+        services = self._service_data
+        for doc in services['documents']:
+            if doc['service_type'] == service_type:
+                docs.append(doc)
+        res['documents'] = docs
+        for service in services['services']:
+            if service['service_type'] == service_type:
+                res['service'] = service
+                break
+        return res

otc_metadata/templates/conf.py.j2

@@ -104,6 +104,9 @@ html_title = "{{ title }}"
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+# Do not include sources into the rendered results
+html_copy_source = False
+
 # -- Options for PDF output --------------------------------------------------
 latex_documents = [
 {%- if pdf_name is defined %}

otc_metadata/templates/tox.ini.j2

@@ -21,18 +21,28 @@ deps =
   -r{toxinidir}/requirements.txt
 commands = {posargs}
 
+# This env is invoked in the periodic pipeline and is therefore responsible to
+# build all relevant docs at once.
 [testenv:docs]
 deps = -r{toxinidir}/doc/requirements.txt
+allowlist_externals =
+  mkdir
+  cp
+  sh
 commands =
 {%- for doc in docs %}
   {[testenv:{{ doc.type }}]commands}
   {[testenv:json-{{ doc.type }}]commands}
 {%- endfor %}
 
-[testenv:docs-pdf]
+[testenv:pdf-docs]
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals =
+allowlist_externals =
+  rm
   mkdir
+  make
+  bash
+  cp
 commands =
   mkdir -p doc/build/pdf
 {%- for doc in docs %}
@@ -46,7 +56,7 @@ commands =
 # HTML version
 [testenv:{{ doc.type }}]
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals =
+allowlist_externals =
   cp
   mkdir
 commands =
@@ -65,14 +75,15 @@ commands =
 # Json version (for search)
 [testenv:json-{{ doc.type }}]
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals =
+allowlist_externals =
   cp
   mkdir
-  find
+  sh
 commands =
   sphinx-build -W --keep-going -b json {{ loc }}/source doc/build/json/{{ doc.type }}
-  # Drop data useless for the search
+  # Drop data useless for the search - wrap it also with sh/xargs due to bugs
-  find doc/build/json -type d -and ( -name '_images' -or -name '_static' -or -name '_sources' ) -exec rm {:} ;
+  # in tox
+  sh -c "find doc/build/json -type d -and '(' -name '_images' -or -name '_static' -or -name '_sources' ')' -print0 | xargs -0 rm -rf"
 {%- if doc.type == 'api-ref' %}
   mkdir -p api-ref/build/json
   cp -av doc/build/json/api-ref api-ref/build/json
@@ -88,7 +99,7 @@ commands =
 # PDF version
 [testenv:{{ doc.type }}-pdf-docs]
 deps = -r{toxinidir}/doc/requirements.txt
-whitelist_externals =
+allowlist_externals =
   rm
   mkdir
   make

otc_metadata/tests/base.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2010-2011 OpenStack Foundation
-# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from unittest import TestCase
-
-
-class TestCase(TestCase):
-
-    """Test case base class for all unit tests."""

otc_metadata/tests/test_otc_metadata.py

@@ -19,10 +19,10 @@ test_otc-metadata
 Tests for `otc-metadata` module.
 """
 
-from otc-metadata.tests import base
+from unittest import TestCase
 
 
-class TestOtc-metadata(base.TestCase):
+class TestOtcMetadata(TestCase):
 
     def test_something(self):
         pass

setup.cfg

@@ -1,11 +1,11 @@
 [metadata]
 name = otc-metadata
 summary = Metadata about OTC for Ecosystem
-description-file =
+description_file =
     README.rst
 author = Open Telekom Cloud
-home-page = https://open.telekom.cloud/
+home_page = https://open.telekom.cloud/
-python-requires = >=3.6
+python_requires = >=3.6
 classifier =
     Environment :: OpenStack
     Intended Audience :: Information Technology

setup.py

@@ -11,7 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
 import setuptools
 
 setuptools.setup(

test-requirements.txt

@@ -2,9 +2,6 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-hacking>=3.0,<3.1 # Apache-2.0
+stestr>=2.0.0 # Apache-2.0
-
+testtools>=2.2.0 # MIT
-coverage>=4.0,!=4.4 # Apache-2.0
+flake8
-python-subunit>=0.0.18 # Apache-2.0/BSD
-stestr>=1.0.0 # Apache-2.0
-testtools>=1.4.0 # MIT

tools-requirements.txt

@@ -1,2 +1,6 @@
 GitPython
-ruamel
+ruamel.yaml
+requests
+jinja2
+dirsync
+cookiecutter

tools/convert_data.py

@@ -1,4 +1,3 @@
-import copy
 import re
 
 import otc_metadata.services
@@ -7,21 +6,17 @@ from ruamel.yaml import YAML
 data = otc_metadata.services.Services()
 new_data = data._service_data
 
-#services = data.service_dict
+# services = data.service_dict
 
 for doc in new_data["documents"]:
     hc_location = None
     link = doc.get("link")
     if link:
         print(f"Parsing {link}")
-        #(p1, p2) = link.split("/")
+        # (p1, p2) = link.split("/")
-        doc["link"] = re.sub(
+        doc["link"] = re.sub(r"/(.*)/(.*)/", r"/\2/\1/", link)
-            r"/(.*)/(.*)/",
-            r"/\2/\1/",
-            link
-        )
 
 _yaml = YAML()
 _yaml.indent(mapping=2, sequence=4, offset=2)
-with open('new.yaml', 'w') as fd:
+with open("new.yaml", "w") as fd:
     _yaml.dump(new_data, fd)

tools/generate_docexports_data.py

@@ -13,7 +13,9 @@ def main():
 
     _yaml = YAML()
     _yaml.indent(mapping=2, sequence=4, offset=2)
-    sys.stdout.write("# Auto-generated by otc_metadata.generate_docexports.data\n")
+    sys.stdout.write(
+        "# Auto-generated by otc_metadata.generate_docexports.data\n"
+    )
     _yaml.dump(data.docs_html_by_category("internal"), sys.stdout)
 
 

tools/sort_metadata.py

@@ -1,7 +1,7 @@
-import copy
+# import copy
 
 import otc_metadata.services
-from ruamel.yaml import YAML
+# from ruamel.yaml import YAML
 
 data = otc_metadata.services.Services()
 

tools/update_zuul_project_configs.py

@@ -15,15 +15,15 @@
 
 import argparse
 import logging
-import os
+# import os
 import pathlib
 import requests
 import subprocess
-import warnings
+# import warnings
 
 from git import exc
 from git import Repo
-from git import SymbolicReference
+# from git import SymbolicReference
 
 from ruamel.yaml import CommentedMap
 from ruamel.yaml import YAML
@@ -72,7 +72,7 @@ def process_repositories(args, service):
     workdir.mkdir(exist_ok=True)
 
     copy_to = None
-    repo_to = None
+    # repo_to = None
 
     for repo in service["repositories"]:
         logging.debug(f"Processing repository {repo}")
@@ -130,6 +130,7 @@ def process_repositories(args, service):
         zuul_templates = None
         zuul_jobs = dict()
         zuul_new_jobs = list()
+        zuul_vars = dict()
         zuul_config_updated = False
         for item in zuul_config:
             if "project" in item.keys():
@@ -137,10 +138,11 @@ def process_repositories(args, service):
                 zuul_templates = project.setdefault("templates", [])
                 if not zuul_templates:
                     zuul_templates = []
+                zuul_vars = project.setdefault("vars", {})
             elif "job" in item.keys():
                 job = item["job"]
                 zuul_jobs[job["name"]] = job
-        print(f"Existing jobs {zuul_jobs}")
+        logging.debug(f"Existing jobs {zuul_jobs}")
         if "helpcenter-base-jobs" not in zuul_templates:
             zuul_templates.append("helpcenter-base-jobs")
             zuul_config_updated = True
@@ -148,6 +150,7 @@ def process_repositories(args, service):
         job_suffix = (
             "-hc-int-jobs" if args.environment == "internal" else "-hc-jobs"
         )
+        sphinx_pdf_files = zuul_vars.setdefault('sphinx_pdf_files', [])
         for doc in data.docs_by_service_type(service["service_type"]):
             logging.debug(f"Analyzing document {doc}")
             if not doc.get("type"):
@@ -156,6 +159,12 @@ def process_repositories(args, service):
                 doc_type = "dev-guide"
             else:
                 doc_type = doc["type"]
+            # Collect all PDF files into sphinx_pdf_files var
+            pdf_name = doc.get('pdf_name')
+            if pdf_name and f"{pdf_name}.pdf" not in sphinx_pdf_files:
+                sphinx_pdf_files.append(f"{pdf_name}.pdf")
+                zuul_config_updated = True
+
             template_name = f"{doc_type}{job_suffix}"
             if doc_type in ["api-ref", "umn", "dev-guide"]:
                 if template_name not in zuul_templates:
@@ -190,6 +199,7 @@ def process_repositories(args, service):
                 if "project" in item.keys():
                     project = item["project"]
                     project["templates"] = zuul_templates
+                    project["vars"] = zuul_vars
                     # Ensure new jobs are in check
                     if len(zuul_new_jobs) > 0:
                         project.setdefault(
@@ -208,8 +218,7 @@ def process_repositories(args, service):
                         project["check"]["jobs"].extend(
                             [x["job"]["name"] for x in zuul_new_jobs])
 
-
+            # yaml.indent(offset=2, sequence=2)
-            #yaml.indent(offset=2, sequence=2)
             with open(zuul_file_name, "w") as f:
                 yaml.dump(zuul_config, f)
             git_repo.index.add([zuul_file_name.name])
@@ -223,7 +232,8 @@ def process_repositories(args, service):
             git_repo.index.commit(
                 (
                     "Update zuul.yaml file\n\n"
-                    "Performed-by: gitea/infra/otc-metadata/tools/update_zuul_project_config.py"
+                    "Performed-by: gitea/infra/otc-metadata"
+                    "/tools/update_zuul_project_config.py"
                 )
             )
             push_args = ["--set-upstream", "origin", branch_name]

tox.ini

@@ -1,56 +1,26 @@
 [tox]
 minversion = 3.2.0
-envlist = py39,pep8
+envlist = py3,pep8
-skipsdist = True
 ignore_basepython_conflict = true
 
-
 [testenv]
 basepython = python3
 usedevelop = True
 setenv =
    PYTHONWARNINGS=default::DeprecationWarning
-   OS_STDOUT_CAPTURE=1
-   OS_STDERR_CAPTURE=1
-   OS_TEST_TIMEOUT=60
 deps = -r{toxinidir}/test-requirements.txt
 commands = stestr run {posargs}
 
-[testenv:lower-constraints]
-deps = -r{toxinidir}/test-requirements.txt
-
 [testenv:pep8]
 commands = flake8 {posargs}
 
 [testenv:venv]
 commands = {posargs}
 
-[testenv:cover]
-setenv =
-    VIRTUAL_ENV={envdir}
-    PYTHON=coverage run --source otc_metadata --parallel-mode
-commands =
-    stestr run {posargs}
-    coverage combine
-    coverage html -d cover
-    coverage xml -o cover/coverage.xml
-
-[testenv:docs]
-deps = -r{toxinidir}/doc/requirements.txt
-commands = sphinx-build -W -b html doc/source doc/build/html
-
-[testenv:releasenotes]
-deps = {[testenv:docs]deps}
-commands =
-  sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
-
-[testenv:debug]
-commands = oslo_debug_helper {posargs}
-
 [flake8]
 # E123, E125 skipped as they are invalid PEP-8.
 
 show-source = True
-ignore = E123,E125
+ignore = E123,E125,W503
 builtins = _
 exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build

zuul.yaml

@@ -1,19 +1,11 @@
 - project:
     merge-mode: squash-merge
-    default-branch: master
+    default-branch: main
-    templates:
-      - publish-to-pypi
-      - publish-otc-docs-hc-pti
-      - release-notes-jobs
     check:
       jobs:
         - otc-tox-pep8
-        - otc-tox-py38
         - otc-tox-py39
-        - tox-functional
     gate:
       jobs:
         - otc-tox-pep8
-        - otc-tox-py38
         - otc-tox-py39
-        - tox-functional