From 166e05613fb6ad588917d7faf4d71bea85d86a18 Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Fri, 6 May 2022 17:57:43 +0200 Subject: [PATCH] Copy referred images into the destination and add git patch (#7) Copy referred images into the destination and add git patch Include referred images in the result Reviewed-by: OpenTelekomCloud Bot --- otc_doc_convertor/convertor.py | 16 +++- playbooks/post.yaml | 54 +++--------- playbooks/propose_update.yaml | 1 - playbooks/run.yaml | 28 +++--- roles/fetch_doc_artifacts/tasks/main.yaml | 31 +++++-- roles/fetch_doc_artifacts/tasks/patch.yaml | 15 ++-- roles/generate_doc_patch/defaults/main.yaml | 2 + roles/generate_doc_patch/tasks/main.yaml | 98 ++++++++++++++++++++- zuul.yaml | 7 +- 9 files changed, 177 insertions(+), 75 deletions(-) create mode 100644 roles/generate_doc_patch/defaults/main.yaml diff --git a/otc_doc_convertor/convertor.py b/otc_doc_convertor/convertor.py index daed81a46..062368bf4 100644 --- a/otc_doc_convertor/convertor.py +++ b/otc_doc_convertor/convertor.py @@ -7,6 +7,7 @@ import logging import os import pathlib import re +import shutil class OTCDocConvertor: @@ -124,6 +125,8 @@ class OTCDocConvertor: cap.name = 'figcaption' figure.append(cap) if img: + # Store all referred images for copying + self.doc_images.add(img['src']) img['src'] = '/_static/images/' + img['src'] figure.append(img) i.replace_with(figure) @@ -233,12 +236,12 @@ class OTCDocConvertor: '--dest', help='Directory to write resulting files') self.args = parser.parse_args() - retval = os.getcwd() meta_data = json.loads(open( pathlib.Path(self.args.path, "CLASS.TXT.json") ).read()) metadata_by_uri = dict() metadata_by_code = dict() + self.doc_images = set() if self.args.dest: dest = pathlib.Path(self.args.dest) else: @@ -415,7 +418,16 @@ class OTCDocConvertor: f" of result/{path}/index.rst") p.unlink() - os.chdir(retval) + # Copy used images + if len(self.doc_images) > 0: + logging.debug("Processing images") + img_dest = pathlib.Path(dest, '_static', 'images') + img_dest.mkdir(parents=True, exist_ok=True) + for img in self.doc_images: + shutil.copyfile( + pathlib.Path(self.args.path, img).resolve(strict=False), + pathlib.Path(img_dest, img).resolve(strict=False) + ) def main(): diff --git a/playbooks/post.yaml b/playbooks/post.yaml index 0feb96fc3..9f51b8230 100644 --- a/playbooks/post.yaml +++ b/playbooks/post.yaml @@ -1,44 +1,18 @@ --- -- hosts: localhost - vars: - vault_addr: "{{ zuul_vault_addr }}" - vault_secret_dest: "{{ zuul.executor.work_root }}/.approle-secret" - vault_token_dest: "{{ zuul.executor.work_root }}/.approle-token" - - roles: - # Get the Vault token from prepared secret-id - - role: create-vault-approle-token - vault_role_id: "{{ zuul_vault.vault_role_id }}" - vault_wrapping_token_id: "{{ lookup('file', vault_secret_dest) }}" - - hosts: all - vars: - vault_token_dest: "{{ zuul.executor.work_root }}/.approle-token" - vault_addr: "{{ zuul_vault_addr }}" tasks: + - name: Read project docs configuration + include_vars: "{{ docs_update_data_file }}" - - name: Fetch organization tokens - no_log: true - check_mode: false - ansible.builtin.uri: - url: "{{ vault.vault_addr }}/v1/{{ vault.vault_token_path }}" - headers: - "X-Vault-Token": "{{ lookup('file', vault_token_dest) }}" - method: "POST" - body: - org_name: "opentelekomcloud-docs" - body_format: "json" - register: "org_token" - - - name: Revoke GitHub token lease - check_mode: false - no_log: true - uri: - url: "{{ vault.vault_addr }}/v1/sys/leases/revoke" - headers: - "X-Vault-Token": "{{ vault.vault_token }}" - method: "PUT" - body: - lease_id: "{{ org_token.json.lease_id }}" - body_format: "json" - status_code: 204 + - name: Fetch generated artifacts + include_role: + name: "fetch_doc_artifacts" + vars: + fetch_doc_artifacts_name: "{{ doc.label }}" + fetch_doc_artifacts_rst_source: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}/{{ docs_new_location }}" + fetch_doc_artifacts_diff_patch: "{{ ansible_user_dir }}/{{ doc.label }}.patch" + fetch_doc_artifacts_git_patch: "{{ ansible_user_dir }}/{{ doc.label }}.git.patch" + loop: "{{ docs }}" + loop_control: + loop_var: "doc" + label: "{{ doc.label }}" diff --git a/playbooks/propose_update.yaml b/playbooks/propose_update.yaml index 97a9893f6..c7df943b7 100644 --- a/playbooks/propose_update.yaml +++ b/playbooks/propose_update.yaml @@ -52,7 +52,6 @@ - "org_token is defined" - "org_token.json is defined" - - hosts: all vars: github_token_dest: "{{ zuul.executor.work_root }}/.github" diff --git a/playbooks/run.yaml b/playbooks/run.yaml index 9752ddf8c..21d5021fb 100644 --- a/playbooks/run.yaml +++ b/playbooks/run.yaml @@ -1,10 +1,5 @@ --- - hosts: all - vars: - docs_rst_location: "docs" - docs_base_location: "base" - docs_new_location: "new" - tasks: - name: Read project docs configuration include_vars: "{{ docs_update_data_file }}" @@ -16,6 +11,12 @@ register: git_log ignore_errors: true + - name: Configure git user name + command: "git config --global user.name 'OpenTelekomCloud Proposal Bot'" + + - name: Configure git user email + command: "git config --global user.email 52695153+otcbot@users.noreply.github.com" + - name: Generate new RSTs include_role: name: "convert_doc" @@ -54,22 +55,13 @@ doc_label: "{{ doc.label }}" generate_doc_patch_base_location: "{{ docs_base_location }}" generate_doc_patch_new_location: "{{ docs_new_location }}" + generate_doc_patch_repository: "{{ doc.repository }}" + generate_doc_patch_project_location: "{{ doc.project_location }}" generate_doc_patch_base: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}" - generate_doc_patch_patch_location: "{{ ansible_user_dir }}/{{ doc.label }}.patch" + generate_doc_patch_diff_patch_location: "{{ ansible_user_dir }}/{{ doc.label }}.patch" + generate_doc_patch_git_patch_location: "{{ ansible_user_dir }}/{{ doc.label }}.git.patch" loop: "{{ docs }}" loop_control: loop_var: "doc" label: "{{ doc.label }}" when: "git_log.stdout is search(doc.html_location) or git_log.stdout is search('otc_doc_convertor')" - - - name: Fetch generated artifacts - include_role: - name: "fetch_doc_artifacts" - vars: - fetch_doc_artifacts_name: "{{ doc.label }}" - fetch_doc_artifacts_rst_source: "{{ ansible_user_dir }}/{{ docs_rst_location }}/{{ doc.label }}/{{ docs_new_location }}" - fetch_doc_artifacts_patch: "{{ ansible_user_dir }}/{{ doc.label }}.patch" - loop: "{{ docs }}" - loop_control: - loop_var: "doc" - label: "{{ doc.label }}" diff --git a/roles/fetch_doc_artifacts/tasks/main.yaml b/roles/fetch_doc_artifacts/tasks/main.yaml index 6b7377fdc..7424d8a55 100644 --- a/roles/fetch_doc_artifacts/tasks/main.yaml +++ b/roles/fetch_doc_artifacts/tasks/main.yaml @@ -9,11 +9,32 @@ when: "doc_dir.matched > 0" include_tasks: doc.yaml -- name: Find {{ fetch_doc_artifacts_name }} Patch +- name: Find {{ fetch_doc_artifacts_name }} diff patch stat: - path: "{{ fetch_doc_artifacts_patch }}" - register: doc_patch + path: "{{ fetch_doc_artifacts_diff_patch }}" + register: doc_diff_patch + when: "fetch_doc_artifacts_diff_patch is defined" -- name: Process {{ fetch_doc_artifacts_name }} Patch file - when: "doc_patch.stat.exists" +- name: Process {{ fetch_doc_artifacts_name }} diff patch file + when: + - "fetch_doc_artifacts_diff_patch is defined" + - "doc_diff_patch.stat.exists" include_tasks: patch.yaml + vars: + patch_file: "{{ fetch_doc_artifacts_diff_patch }}" + patch_type: "diff" + +- name: Find {{ fetch_doc_artifacts_name }} git patch + stat: + path: "{{ fetch_doc_artifacts_git_patch }}" + register: doc_git_patch + when: "fetch_doc_artifacts_git_patch is defined" + +- name: Process {{ fetch_doc_artifacts_name }} git patch file + when: + - "fetch_doc_artifacts_git_patch is defined" + - "doc_git_patch.stat.exists" + include_tasks: patch.yaml + vars: + patch_file: "{{ fetch_doc_artifacts_git_patch }}" + patch_type: "git" diff --git a/roles/fetch_doc_artifacts/tasks/patch.yaml b/roles/fetch_doc_artifacts/tasks/patch.yaml index c9bd52081..cdada8fc1 100644 --- a/roles/fetch_doc_artifacts/tasks/patch.yaml +++ b/roles/fetch_doc_artifacts/tasks/patch.yaml @@ -1,9 +1,9 @@ - block: - name: Fetch archive synchronize: - dest: "{{ zuul.executor.log_root }}/{{ fetch_doc_artifacts_name }}.patch" + dest: "{{ zuul.executor.log_root }}/{{ fetch_doc_artifacts_name }}.{{ patch_type }}.patch" mode: pull - src: "{{ fetch_doc_artifacts_patch }}" + src: "{{ patch_file }}" verify_host: true owner: no group: no @@ -12,8 +12,8 @@ - block: - name: Copy archive copy: - dest: "{{ zuul_output_dir }}/logs/{{ fetch_doc_artifacts_name }}.patch" - src: "{{ fetch_doc_artifacts_patch }}" + dest: "{{ zuul_output_dir }}/logs/{{ fetch_doc_artifacts_name }}.{{ patch_type }}.patch" + src: "{{ patch_file }}" mode: 0644 remote_src: true when: zuul_use_fetch_output @@ -23,9 +23,10 @@ data: zuul: artifacts: - - name: "{{ fetch_doc_artifacts_name }} patch" - url: "patches/{{ fetch_doc_artifacts_name }}.patch" + - name: "{{ fetch_doc_artifacts_name }} {{ patch_type }} patch" + url: "{{ fetch_doc_artifacts_name }}.{{ patch_type }}.patch" metadata: - type: docs_patch + type: "patch" doc_service: "{{ doc.service }}" doc_type: "{{ doc.type }}" + patch_type: "{{ patch_type }}" diff --git a/roles/generate_doc_patch/defaults/main.yaml b/roles/generate_doc_patch/defaults/main.yaml new file mode 100644 index 000000000..2af204c0c --- /dev/null +++ b/roles/generate_doc_patch/defaults/main.yaml @@ -0,0 +1,2 @@ +generate_doc_patch_repo_location: "{{ ansible_user_dir }}/{{ generate_doc_patch_repository }}/{{ generate_doc_patch_project_location }}" +generate_doc_patch_doc_location: "{{ generate_doc_patch_repo_location }}/{{ generate_doc_patch_project_location }}" diff --git a/roles/generate_doc_patch/tasks/main.yaml b/roles/generate_doc_patch/tasks/main.yaml index 87497bdfa..4ee7c1bb7 100644 --- a/roles/generate_doc_patch/tasks/main.yaml +++ b/roles/generate_doc_patch/tasks/main.yaml @@ -9,8 +9,103 @@ path: "{{ generate_doc_patch_base }}/{{ generate_doc_patch_new_location }}" register: new_stat +- name: Create reference {{ generate_doc_patch_repository }} repo + file: + path: "{{ generate_doc_patch_doc_location }}" + state: "directory" + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "generate_doc_patch_git_patch_location is defined" + +- name: Initialize fake git repository + command: "git init" + args: + chdir: "{{ generate_doc_patch_repo_location }}" + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "generate_doc_patch_git_patch_location is defined" + +- name: Copy {{ doc_label }} base version + copy: + src: "{{ generate_doc_patch_base }}/{{ generate_doc_patch_base_location }}/" + dest: "{{ generate_doc_patch_doc_location }}" + remote_src: true + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "generate_doc_patch_git_patch_location is defined" + +- name: Stage base version + command: "git add ." + args: + chdir: "{{ generate_doc_patch_repo_location }}" + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "generate_doc_patch_git_patch_location is defined" + +- name: Commit base version + command: "git commit -m 'base version'" + args: + chdir: "{{ generate_doc_patch_repo_location }}" + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "generate_doc_patch_git_patch_location is defined" + +- name: Copy {{ doc_label }} new version + copy: + src: "{{ generate_doc_patch_base }}/{{ generate_doc_patch_new_location }}/" + dest: "{{ generate_doc_patch_doc_location }}" + remote_src: true + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "generate_doc_patch_git_patch_location is defined" + +- name: Stage updated version + command: "git add ." + args: + chdir: "{{ generate_doc_patch_repo_location }}" + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "generate_doc_patch_git_patch_location is defined" + +- name: Perform git diff to see if there are changes + command: "git diff --staged" + register: diff_output + args: + chdir: "{{ generate_doc_patch_repo_location }}" + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "generate_doc_patch_git_patch_location is defined" + +- name: Commit updated version + command: "git commit -m 'Update {{ doc_label }}'" + args: + chdir: "{{ generate_doc_patch_repo_location }}" + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "diff_output.stdout != ''" + - "generate_doc_patch_git_patch_location is defined" + +- name: Generate {{ doc_label }} patch + shell: "git format-patch --binary -1 --stdout > {{ generate_doc_patch_git_patch_location }}" + args: + chdir: "{{ generate_doc_patch_repo_location }}" + when: + - "base_stat.stat.exists" + - "new_stat.stat.exists" + - "diff_output.stdout != ''" + - "generate_doc_patch_git_patch_location is defined" + - name: Generate {{ doc_label }} patch file - shell: "diff -ruN {{ generate_doc_patch_base_location }} {{ generate_doc_patch_new_location }} > {{ generate_doc_patch_patch_location }}" + shell: "diff -ruN {{ generate_doc_patch_base_location }} {{ generate_doc_patch_new_location }} > {{ generate_doc_patch_diff_patch_location }}" args: chdir: "{{ generate_doc_patch_base }}" register: diff_cmd @@ -19,3 +114,4 @@ when: - "base_stat.stat.exists" - "new_stat.stat.exists" + - "generate_doc_patch_diff_patch_location is defined" diff --git a/zuul.yaml b/zuul.yaml index e12683b27..96c902b27 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -28,12 +28,13 @@ name: otc-doc-exports-convert-base parent: unittests abstract: true - nodeset: ubuntu-focal + nodeset: ubuntu-jammy description: | Convert doc exports from html to rst and generate corresponding rst diff files. pre-run: playbooks/pre.yaml run: playbooks/run.yaml + post-run: playbooks/post.yaml - job: name: otc-doc-exports-convert-compute @@ -92,6 +93,10 @@ - project: merge-mode: squash-merge default-branch: main + vars: + docs_rst_location: "docs" + docs_base_location: "base" + docs_new_location: "new" check: jobs: - otc-tox-pep8: