re-introducing changes before wafd version 20230423

metadata.yaml

@@ -488,6 +488,11 @@ categories:
           type: dev
           pdf_name: ims-dev-guide
           hc_location: devg/ims
+        - html_location: docs/ims/public-images
+          rst_location: doc/public-images/source
+          title: Public Image Introduction
+          type: public-images
+          pdf_name: ims-public-images
         - html_location: docs/ims/umn
           rst_location: umn/source
           title: User Guide
@@ -590,7 +595,7 @@ categories:
           pdf_name: obs-perms-cfg
         - html_location: docs/obs/api-swift
           rst_location: doc/swiftapi/source
-          title: API Reference (Swift) - Deprecated
+          title: API Reference (Swift)
           type: swiftapi
           pdf_name: obs-swiftapi
           hc_location: api_swift/obs
@@ -1122,11 +1127,6 @@ categories:
     - service_title: GaussDB NoSQL
       service_type: gaussdb_nosql
       docs:
-        - html_location: docs/gaussdb_nosql/api-ref
-          rst_location: api-ref/source
-          title: API Reference
-          type: api-ref
-          pdf_name: gaussdb_nosql-api-ref
         - html_location: docs/gaussdb_nosql/umn
           rst_location: umn/source
           title: User Guide

otc_doc_convertor/comparator.py

@@ -0,0 +1,95 @@
+import argparse
+import logging
+import requests
+import pathlib
+from bs4 import BeautifulSoup
+
+
+def body_filter(tag):
+    return (
+        tag.name == "div"
+        and tag.has_attr("id")
+        and tag["id"].startswith("body")
+    )
+
+
+def simplify_body(data):
+    return data.get_text().replace(" ", "")
+
+
+class OTCComparator:
+
+    def compare(self, url_prefix, file_path, file_name):
+        try:
+            data = requests.get(
+                f"https://docs.otc.t-systems.com/{url_prefix}/"
+                f"{file_name}.json")
+            page_data = None
+            for item in data.json():
+                if (
+                    item.get("url").endswith(f"{file_name}.html")
+                    and item['content']
+                ):
+                    page_data = item["content"]
+                    break
+            original = BeautifulSoup(page_data, 'html.parser')
+            with open(f"{file_path}/{file_name}.html", "r") as f:
+                new_content = f.read()
+            new = BeautifulSoup(new_content, 'html.parser')
+            t1 = original.find(body_filter)
+            t2 = new.find(body_filter)
+            if t1 != t2:
+                if simplify_body(t1) == simplify_body(t2):
+                    logging.error(
+                        "File %s is not matching, but "
+                        "plain text matches" % file_name)
+                    return True
+                else:
+                    logging.error("File %s mismatches" % file_name)
+                    logging.debug(
+                        "Proposed content: %s" %
+                        t2.get_text().encode("unicode_escape").decode("utf-8"))
+                    logging.debug(
+                        "Current content: %s" %
+                        t1.get_text().encode("unicode_escape").decode("utf-8"))
+                    return False
+            else:
+                logging.info("Content matches")
+            return True
+        except Exception as ex:
+            logging.error("Content comparison error %s" % ex)
+            return False
+
+    def main(self):
+        logging.basicConfig(level=logging.DEBUG)
+        parser = argparse.ArgumentParser(description="Compare document data.")
+        parser.add_argument(
+            "path",
+            type=str,
+            help="Path to the document content (i.e. docs/ecs/api-ref")
+        parser.add_argument(
+            "url",
+            type=str,
+            help="url prefix in the helpcenter (i.e. api/ecs)")
+        args = parser.parse_args()
+        match = True
+
+        for f in pathlib.Path(args.path).glob("*.html"):
+            logging.info(f"Comparing {f.name}")
+            if not self.compare(
+                    args.url, args.path, f.name.replace(".html", "")):
+                match = False
+
+        if not match:
+            logging.error("Comparison showed deviations")
+            exit(1)
+        else:
+            logging.info("No deviations found")
+
+
+def main():
+    OTCComparator().main()
+
+
+if __name__ == '__main__':
+    main()

playbooks/pre.yaml

@@ -5,10 +5,12 @@
     - ensure-pip
     - ensure-virtualenv
     - role: "ensure-pandoc"
+      vars:
+        ensure_pandoc_version: "2.19.2"
   tasks:
     - name: Install convertor
       pip:
         chdir: "{{ zuul.project.src_dir }}"
         virtualenv: "{{ ansible_user_dir }}/.venv"
-        name: "{{ ansible_user_dir }}/{{ zuul.projects['gitea.eco.tsi-dev.otc-service.com/docs/doc-convertor'].src_dir }}"
+        name: .
         editable: "yes"

setup.cfg

@@ -1,11 +1,11 @@
 [metadata]
-name = otc-doc-exports
+name = otc-doc-convertor
 author = Open Telekom Cloud - Ecosystem Squad
 author_email = dl-pbcotcdeleco@t-systems.com
-description = Doc sources (HTML) to track changes in the vendors documentation system
+description = Python program to convert docs exported in HTML into RST
 description_file =
     README.md
-home_page = https://gitea.eco.tsi-dev.otc-service.com/docs/doc-exports
+home_page = https://github.com/opentelekomcloud-docs/doc-exports
 classifier =
     License :: OSI Approved :: Apache Software License
     Operating System :: POSIX :: Linux
@@ -17,3 +17,12 @@ classifier =
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
+keywords = Sphinx, search, python
+
+[options]
+packages = otc_doc_convertor
+
+[options.entry_points]
+console_scripts =
+  otc-convert-doc = otc_doc_convertor.convertor:main
+  otc-convert-compare = otc_doc_convertor.comparator:main

templates/conf.py

@@ -18,7 +18,7 @@ import os
 import sys
 
 extensions = [
-    'otcdocstheme',
+    'otcdocstheme'
 ]
 
 otcdocs_auto_name = False