diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000..ea68e86 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,4 @@ +sphinx>=2.0.0,!=2.1.0 # BSD +otcdocstheme # Apache-2.0 +# releasenotes +reno>=3.1.0 # Apache-2.0 \ No newline at end of file diff --git a/tox.ini b/tox.ini index 3207bd4..88c9dbb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 3.1 -envlist = py36,py37,pep8 +envlist = py39,pep8 skipsdist = True ignore_basepython_conflict = True @@ -8,67 +8,81 @@ ignore_basepython_conflict = True usedevelop = True install_command = pip install {opts} {packages} deps = - -r{toxinidir}/requirements.txt + -r{toxinidir}/requirements.txt commands = stestr run {posargs} - stestr slowest + stestr slowest [testenv:pep8] commands = - doc8 doc/source README.rst + doc8 doc/source README.rst [testenv:venv] deps = - -r{toxinidir}/requirements.txt + -r{toxinidir}/requirements.txt commands = {posargs} [testenv:docs] -deps = - -c{env:UPPER_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} - -r{toxinidir}/doc/requirements.txt commands = - sphinx-build -W --keep-going -b html doc/source/ doc/build/html + {[testenv:api-ref]commands} + {[testenv:umn]commands} + +[testenv:docs-pdf] +whitelist_externals = + mkdir +commands = + mkdir -p doc/build/pdf + {[testenv:api-ref-pdf-docs]commands} + {[testenv:umn-pdf-docs]commands} + [testenv:api-ref] -# This environment is called from CI scripts to test and publish -# the API Ref to docs.otc-service.com -deps = - -r{toxinidir}/requirements.txt -whitelist_externals = rm +whitelist_externals = + cp + mkdir commands = - rm -rf api-ref/build - sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html + sphinx-build -W --keep-going -b html api-ref/source doc/build/html/api-ref + mkdir -p api-ref/build/html + cp -av doc/build/html/api-ref api-ref/build/html [testenv:api-ref-pdf-docs] -deps = {[testenv:api-ref]deps} -envdir = {toxworkdir}/api-ref whitelist_externals = rm + mkdir make + bash + cp commands = rm -rf api-ref/build/pdf - sphinx-build -a -E -W -b latex api-ref/source api-ref/build/pdf + sphinx-build -W --keep-going -b latex api-ref/source api-ref/build/pdf/ + bash -c "for f in api-ref/build/pdf/*.gif; do convert $f $\{f/%gif/png\}; done || true" + bash -c "for f in api-ref/build/pdf/*.tex; do sed -iorig 's/\.gif//g' $f; done" make -C api-ref/build/pdf + mkdir -p doc/build/pdf + cp api-ref/build/pdf/dis-api-ref.pdf doc/build/pdf/ + [testenv:umn] -# This environment is called from CI scripts to test and publish -# the UMN to docs.otc-service.com -deps = - -r{toxinidir}/requirements.txt -whitelist_externals = rm +whitelist_externals = + cp + mkdir commands = - rm -rf umn/build - sphinx-build -W -b html -d umn/build/doctrees umn/source umn/build/html + sphinx-build -W --keep-going -b html umn/source doc/build/html/umn + mkdir -p umn/build/html + cp -av doc/build/html/umn umn/build/html [testenv:umn-pdf-docs] -deps = {[testenv:umn]deps} -envdir = {toxworkdir}/umn whitelist_externals = rm + mkdir make - sh + bash + cp commands = rm -rf umn/build/pdf - sphinx-build -a -E -W -b latex umn/source umn/build/pdf - sh -c "for f in umn/build/pdf/*.gif; do gif2png -dprsO $f; done || true" - sh -c "for f in umn/build/pdf/*.tex; do sed -iorig 's/\.gif//g' $f; done" + sphinx-build -W --keep-going -b latex umn/source umn/build/pdf/ + bash -c "for f in umn/build/pdf/*.gif; do convert $f $\{f/%gif/png\}; done || true" + bash -c "for f in umn/build/pdf/*.tex; do sed -iorig 's/\.gif//g' $f; done" make -C umn/build/pdf + mkdir -p doc/build/pdf + cp umn/build/pdf/dis-umn.pdf doc/build/pdf/ + diff --git a/umn/source/_static/images/en-us_image_0000001222097282.png b/umn/source/_static/images/en-us_image_0000001222097282.png new file mode 100644 index 0000000..8e492bf Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222097282.png differ diff --git a/umn/source/_static/images/en-us_image_0000001222097286.jpg b/umn/source/_static/images/en-us_image_0000001222097286.jpg new file mode 100644 index 0000000..0379d50 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222097286.jpg differ diff --git a/umn/source/_static/images/en-us_image_0000001222097302.png b/umn/source/_static/images/en-us_image_0000001222097302.png new file mode 100644 index 0000000..6c8c79c Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222097302.png differ diff --git a/umn/source/_static/images/en-us_image_0000001222097314.png b/umn/source/_static/images/en-us_image_0000001222097314.png new file mode 100644 index 0000000..fcb4648 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222097314.png differ diff --git a/umn/source/_static/images/en-us_image_0000001222097374.png b/umn/source/_static/images/en-us_image_0000001222097374.png new file mode 100644 index 0000000..d58f250 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222097374.png differ diff --git a/umn/source/_static/images/en-us_image_0000001222257278.jpg b/umn/source/_static/images/en-us_image_0000001222257278.jpg new file mode 100644 index 0000000..c328146 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222257278.jpg differ diff --git a/umn/source/_static/images/en-us_image_0000001222257286.png b/umn/source/_static/images/en-us_image_0000001222257286.png new file mode 100644 index 0000000..ec90ea2 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222257286.png differ diff --git a/umn/source/_static/images/en-us_image_0000001222257350.png b/umn/source/_static/images/en-us_image_0000001222257350.png new file mode 100644 index 0000000..cd0b130 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222257350.png differ diff --git a/umn/source/_static/images/en-us_image_0000001222417238.jpg b/umn/source/_static/images/en-us_image_0000001222417238.jpg new file mode 100644 index 0000000..0379d50 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222417238.jpg differ diff --git a/umn/source/_static/images/en-us_image_0000001222417242.png b/umn/source/_static/images/en-us_image_0000001222417242.png new file mode 100644 index 0000000..6c8c79c Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222417242.png differ diff --git a/umn/source/_static/images/en-us_image_0000001222417262.png b/umn/source/_static/images/en-us_image_0000001222417262.png new file mode 100644 index 0000000..dfac6bc Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222417262.png differ diff --git a/umn/source/_static/images/en-us_image_0000001222417338.png b/umn/source/_static/images/en-us_image_0000001222417338.png new file mode 100644 index 0000000..fa8c278 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222417338.png differ diff --git a/umn/source/_static/images/en-us_image_0000001222577290.png b/umn/source/_static/images/en-us_image_0000001222577290.png new file mode 100644 index 0000000..cd0b130 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001222577290.png differ diff --git a/umn/source/_static/images/en-us_image_0000001266697405.png b/umn/source/_static/images/en-us_image_0000001266697405.png new file mode 100644 index 0000000..83f331f Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266697405.png differ diff --git a/umn/source/_static/images/en-us_image_0000001266777397.png b/umn/source/_static/images/en-us_image_0000001266777397.png new file mode 100644 index 0000000..cd0b130 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266777397.png differ diff --git a/umn/source/_static/images/en-us_image_0000001266777405.png b/umn/source/_static/images/en-us_image_0000001266777405.png new file mode 100644 index 0000000..7d91796 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266777405.png differ diff --git a/umn/source/_static/images/en-us_image_0000001266777501.jpg b/umn/source/_static/images/en-us_image_0000001266777501.jpg new file mode 100644 index 0000000..a485658 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266777501.jpg differ diff --git a/umn/source/_static/images/en-us_image_0000001266897385.png b/umn/source/_static/images/en-us_image_0000001266897385.png new file mode 100644 index 0000000..4aef785 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266897385.png differ diff --git a/umn/source/_static/images/en-us_image_0000001266897429.jpg b/umn/source/_static/images/en-us_image_0000001266897429.jpg new file mode 100644 index 0000000..33e12e6 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266897429.jpg differ diff --git a/umn/source/_static/images/en-us_image_0000001266897449.jpg b/umn/source/_static/images/en-us_image_0000001266897449.jpg new file mode 100644 index 0000000..0379d50 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266897449.jpg differ diff --git a/umn/source/_static/images/en-us_image_0000001266897473.jpg b/umn/source/_static/images/en-us_image_0000001266897473.jpg new file mode 100644 index 0000000..cb31e21 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266897473.jpg differ diff --git a/umn/source/_static/images/en-us_image_0000001266977421.png b/umn/source/_static/images/en-us_image_0000001266977421.png new file mode 100644 index 0000000..5948d33 Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266977421.png differ diff --git a/umn/source/_static/images/en-us_image_0000001266977433.png b/umn/source/_static/images/en-us_image_0000001266977433.png new file mode 100644 index 0000000..6c8c79c Binary files /dev/null and b/umn/source/_static/images/en-us_image_0000001266977433.png differ diff --git a/umn/source/cdr_specification/billing_model.rst b/umn/source/cdr_specification/billing_model.rst new file mode 100644 index 0000000..e2d1348 --- /dev/null +++ b/umn/source/cdr_specification/billing_model.rst @@ -0,0 +1,52 @@ +:original_name: dis_01_0223.html + +.. _dis_01_0223: + +Billing Model +============= + +Data Ingestion Service (DIS) is charged for resource usage, there are three main billing factors: the duration of Partition and PUT Payload Unit number. + +- General partition contains duration, PUT Payload Unit number, Data storage size billing factor. Advanced partition contains duration billing factor. +- The duration of Partition: Partition is the base capacity unit of DIS. There are two types of partition: General partition and advanced partition. The general partition provides a capacity of 1MB/sec data input , 2MB/sec data outputand support up to 1000 records put per second. The advanced partition provides a capacity of 5MB/sec data input ,10MB/sec data output and support up to 2000 records put per second .You specify the number of partitions when create the stream based on the throughput requirements. You are charged for each partition at an hourly rate. +- PUT Payload Unit number: A record is the data that your data producer adds to DIS stream. A PUT Payload Unit is counted in 25KB payload "chunks" that comprise a record. For example, a 5KB record contains one PUT Payload Unit, a 45KB record contains two PUT Payload Units, and a 1000KB record contains 40 PUT Payload Units. PUT Payload Unit is charged with a per million PUT Payload Units rate. +- Data storage size: the data storage size in DIS input from the partition. The Customer can specify stream data storage time: from One day to Seven days when create the stream. If the data storage time exceeds the specified time, the data will be automatically aged. In general, the data storage one day is free, it means that 84GB of storage space for each partition is free to use. Of course, the final free space size can be specified according to the actual situation. + +DIS is charged for resource usage as follow: + +- One Partition Costs = partition time \* partition rate + Payload Unit Num \* Payload Unit Rate + Data storage size \* storage size rate +- Total Costs per stream = Partition 1 Cost + Partition 2 Cost + … + Partition n Cost + +Both resources will be charged at the same time. Each charged will be listed separately. There will be 3 CDR records for each stream in the billing interval. + +The data will be cached for some time in the service, and default cache time is 24 hours. If the user want dump the data to OBS, the operation in OBS will be charged separately. + +Example: + +Let us assume that our data producers put 50 records per second in aggregate, and each record is 35KB. In this case, the total data input rate is 1.7MB/sec (50 records/sec*35KB/record). For simplicity, we assume that the throughput and data size of each record are stable and constant throughout the day. + +We first calculate the number of partitions needed for our stream to achieve the required throughput. As one partition provides a capacity of 1MB/sec data input and supports 1000 records/sec, two partitions provide a capacity of 2MB/sec data input and support 2000 records/sec. So a stream with two partitions satisfies our required throughput of 1.7MB/sec at 50 records/sec. + +We then calculate our costs using DIS pricing: + +- Partition Hour: One Partition costs $X per hour. Our stream has two partitions so that it costs $2X per hour. +- PUT Payload Unit (25KB): The 1 million Payload Unit costs $Y. As our record is 35KB, each record contains two PUT Payload Units. Our data producers put 50 records or 100 PUT Payload Units per second in aggregate. That is 180000 records or 360000 PUT Payload Units per hour.so it costs 0.36*Y per hour. + +If the data in the DIS stored 1 day, the Data storage size is always less than 84GB, so the total cost per hour is :$(2X + Y) + +Optionally, we can choose to increase the data retention period of our stream from 24 hours to up to 7 days. In this example, if the throughput is stable, the data storage size is 514.08GB per partition(0.85MB/sec \* 3600sec \* 24hour \* 7day).We assume that the data storage rate is $Z per GB*Hour, the cost is $860.16*Z( the detail is (514.08-84)GB \* 2 partitions \* Z). The total cost should $(2*X + Y + 860.16*Z). + +**Shard Hour** + +Shard is the base throughput unit of an Amazon Kinesis stream. One shard provides a capacity of 1MB/sec data input and 2MB/sec data output. One shard can support up to 1000 records per second. You specify the number of shards needed within your stream based on your throughput requirements. You are charged for each shard at an hourly rate. + +**PUT Payload Unit (25KB)** + +A record is the data that your data producer adds to your Amazon Kinesis stream. A PUT Payload Unit is counted in 25KB payload "chunks" that comprise a record. For example, a 5KB record contains one PUT Payload Unit, a 45KB record contains two PUT Payload Units, and a 1MB record contains 40 PUT Payload Units. PUT Payload Unit is charged with a per million PUT Payload Units rate. + +**Extended Data Retention (Up to 7 days)** + +Amazon Kinesis stores your data for 24 hours by default. You can choose to increase the data retention period of your stream to up to 7 days. You are charged for an additional rate on each shard hour incurred by your stream once you enable extended data retention. + +- Getting records from Amazon Kinesis stream is free. +- Data transfer is free. AWS does not charge for data transfer from your data producers to Amazon Kinesis Streams, or from Amazon Kinesis Streams to your Amazon Kinesis Applications. diff --git a/umn/source/cdr_specification/cdr_folder_and_file_name/cdr_folder.rst b/umn/source/cdr_specification/cdr_folder_and_file_name/cdr_folder.rst new file mode 100644 index 0000000..55aeaa0 --- /dev/null +++ b/umn/source/cdr_specification/cdr_folder_and_file_name/cdr_folder.rst @@ -0,0 +1,16 @@ +:original_name: dis_01_0226.html + +.. _dis_01_0226: + +CDR Folder +========== + ++-------------------------------------------+------------------------------------------+ +| Export Directory | Backup Directory | ++===========================================+==========================================+ +| - /opt/meterfiles/uploads/dispartition | - /opt/meterfiles/backup/dispartition | ++-------------------------------------------+------------------------------------------+ +| - /opt/meterfiles/uploads/dispayloadunit | - /opt/meterfiles/backup/dispayloadunit | ++-------------------------------------------+------------------------------------------+ +| - /opt/meterfiles/uploads/disdatasize | - /opt/meterfiles/backup/disdatasize | ++-------------------------------------------+------------------------------------------+ diff --git a/umn/source/cdr_specification/cdr_folder_and_file_name/index.rst b/umn/source/cdr_specification/cdr_folder_and_file_name/index.rst new file mode 100644 index 0000000..d38dbc8 --- /dev/null +++ b/umn/source/cdr_specification/cdr_folder_and_file_name/index.rst @@ -0,0 +1,18 @@ +:original_name: dis_01_0224.html + +.. _dis_01_0224: + +CDR Folder and File Name +======================== + +- :ref:`Overview ` +- :ref:`CDR Folder ` +- :ref:`Service CDR File Name ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + overview + cdr_folder + service_cdr_file_name diff --git a/umn/source/cdr_specification/cdr_folder_and_file_name/overview.rst b/umn/source/cdr_specification/cdr_folder_and_file_name/overview.rst new file mode 100644 index 0000000..32611c4 --- /dev/null +++ b/umn/source/cdr_specification/cdr_folder_and_file_name/overview.rst @@ -0,0 +1,10 @@ +:original_name: dis_01_0225.html + +.. _dis_01_0225: + +Overview +======== + +Similar to other cloud services , CDR files for DIS will be generated, containing DIS usage information, OTC billing system will fetch these files from certain directory on Common CDR SFTP via public key to complete the charging of DIS. + +The file compression agonization, MD5 verification, error file handling are totally same as the CDR files of other cloud services.for details, please refer to document of"OTC CDR File Specifications". diff --git a/umn/source/cdr_specification/cdr_folder_and_file_name/service_cdr_file_name.rst b/umn/source/cdr_specification/cdr_folder_and_file_name/service_cdr_file_name.rst new file mode 100644 index 0000000..9f23ee8 --- /dev/null +++ b/umn/source/cdr_specification/cdr_folder_and_file_name/service_cdr_file_name.rst @@ -0,0 +1,22 @@ +:original_name: dis_01_0227.html + +.. _dis_01_0227: + +Service CDR File Name +===================== + +OTC_{region_code}_dispart_YYYYMMDDHHMMSS_YYYYMMDDHHMMSS_{az_code}_{serial_number}.csv/zip + +OTC_{region_code}_dispayunit_YYYYMMDDHHMMSS_YYYYMMDDHHMMSS_{az_code}_{serial_number}.csv/zip + +OTC_{region_code}_dissize_YYYYMMDDHHMMSS_YYYYMMDDHHMMSS_{az_code}_{serial_number}.csv/zip + +The {az_code} is not used by dis, the value is set the AZ01 number by default. + +**For example:** + +OTC_eu-de_dispart_20170427100000_20170427105959_eu-de-01_0001.csv/zip + +OTC_eu-de_dispayunit_20170427100000_20170427105959_eu-de-01_0001.csv/zip + +OTC_eu-de_dissize\ **\_20170427100000_20170427105959\_ eu-de-01 \_0001.csv/zip** diff --git a/umn/source/cdr_specification/cdr_specification/index.rst b/umn/source/cdr_specification/cdr_specification/index.rst new file mode 100644 index 0000000..9a36288 --- /dev/null +++ b/umn/source/cdr_specification/cdr_specification/index.rst @@ -0,0 +1,20 @@ +:original_name: dis_01_0228.html + +.. _dis_01_0228: + +CDR Specification +================= + +- :ref:`The CDR Specification ` +- :ref:`The Partition CDR Content ` +- :ref:`The PUT Payload Unit CDR Content ` +- :ref:`The Data Store Size CDR Content ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + the_cdr_specification + the_partition_cdr_content + the_put_payload_unit_cdr_content + the_data_store_size_cdr_content diff --git a/umn/source/cdr_specification/cdr_specification/the_cdr_specification.rst b/umn/source/cdr_specification/cdr_specification/the_cdr_specification.rst new file mode 100644 index 0000000..3fcd767 --- /dev/null +++ b/umn/source/cdr_specification/cdr_specification/the_cdr_specification.rst @@ -0,0 +1,30 @@ +:original_name: dis_01_0229.html + +.. _dis_01_0229: + +The CDR Specification +===================== + +Charging requirement: Charging starts from the time when the DIS instance is created and stops when the DIS instance is deleted. + +CDR record cycle: The minimum granularity is minute, and the current cycle is 60 minutes. The interval for uploading offline CDR files is one hour. + +There are three types of Product ID corresponding to three billing factors in DIS. The detail information of the product ID looks like below: + +.. table:: **Table 1** DIS Product ID + + +--------------------------------+-----------------------+-------------------------------------------------------------+-----------------+ + | ResourceTypeCode | ID Commercial Product | Describe of Commercial Product EN | Unit of Measure | + +================================+=======================+=============================================================+=================+ + | *otc*.resource.type.dispart | OTC_DIS_GEN_TIME | The length of time the user is using the general partition | second | + +--------------------------------+-----------------------+-------------------------------------------------------------+-----------------+ + | | OTC_DIS_ADV_TIME | The length of time the user is using the advanced partition | second | + +--------------------------------+-----------------------+-------------------------------------------------------------+-----------------+ + | *otc*.resource.type.dispayunit | OTC_DIS_GEN_UNIT | The number of the PUT Payload Unit for general partition | number | + +--------------------------------+-----------------------+-------------------------------------------------------------+-----------------+ + | | OTC_DIS_ADV_UNIT | The number of the PUT Payload Unit for advanced partition | number | + +--------------------------------+-----------------------+-------------------------------------------------------------+-----------------+ + | *otc*.resource.type.dissize | OTC_DIS_GEN_STORE | The data storage size in general partition. | byte | + +--------------------------------+-----------------------+-------------------------------------------------------------+-----------------+ + | | OTC_DIS_ADV_STORE | The data storage size in advanced partition. | byte | + +--------------------------------+-----------------------+-------------------------------------------------------------+-----------------+ diff --git a/umn/source/cdr_specification/cdr_specification/the_data_store_size_cdr_content.rst b/umn/source/cdr_specification/cdr_specification/the_data_store_size_cdr_content.rst new file mode 100644 index 0000000..3a23304 --- /dev/null +++ b/umn/source/cdr_specification/cdr_specification/the_data_store_size_cdr_content.rst @@ -0,0 +1,109 @@ +:original_name: dis_01_0232.html + +.. _dis_01_0232: + +The Data Store Size CDR Content +=============================== + +.. table:: **Table 1** DIS Data Store Size CDR Record Content + + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Field | Location | Type | Description | Remarks | + +======================+=============+=============+==================================================+===============================================================================================================================================================================================================================================================================================+ + | RecordType | 1 | String | Type of a record in a file. | Number of normal offline CDR records: 20 | + | | | | | | + | | | | | Number of corrected error offline CDR records: 30 | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | TimeStamp | 2 | String | Time when an offline CDR record is generated. | UTC Format: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | UserID | 3 | String | Unique ID of a subscriber. | The value is the same as the value of **ProjectID**. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | RegionCode | 4 | String | ID of a region. | ID of a region. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | AZCode | 5 | String | Code of an available zone. | Code of an available zone. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | CloudServiceTypeCode | 6 | String | Type of a cloud service. | Cloud service type corresponding to **ResourceID**. For example, cloudservicetype for VM in **EC2** is recorded as **otc.service.type.ec2**, and cloudservicetype for VM in **VDC** is recorded as **otc.service.type.vdc**. | + | | | | | | + | | | | | For DIS: the string is **otc.service.type.dis** | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ResourceTypeCode | 7 | String | Type of a cloud resource. | Cloud resource type corresponding to **ResourceID**, such as VM. | + | | | | | | + | | | | | **ResourceType** is a resource type specified by **CloudServiceType**, and **CloudServiceType** contains multiple sets of **ResourceType**. For example, EC2 is provided by VM, Volume, IP, and Image, and EC2 is a **CloudServiceType**, and VM, Volume, IP, and Image are **ResourceType**. | + | | | | | | + | | | | | For DIS: the string is **otc.resource.type.dis** | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ResourceSpecCode | 8 | String | Code of specified resource specifications. | There are two type resourcespeccode for DIS: | + | | | | | | + | | | | | dis.general.partition | + | | | | | | + | | | | | dis.advanced.partition | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ResourceID | 9 | String | ID of a resource instance. | ID of the partition in the DIS stream instance. | + | | | | | | + | | | | | The ID is different from partitions. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | BSSParams | 10 | String | Extended operation parameter. | The contract ID of one tenant. The contract ID is the xdomain_id of user token. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | BeginTime | 11 | String | Start time of the statistics period of a record. | UTC Format: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | EndTime | 12 | String | End time of the statistics period of a record. | UTC Format: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | DataStoreSize | 13 | String | Name of a specific accumulative factor. | Accumulative factor: Data Store Size | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | DataStoreSizeValue | 14 | Double | Accumulative value of **AccumulateFactorName**. | Accumulative value of DataStore. | + | | | | | | + | | | | | This value is the size of the data store in DIS input by the partition in the period ranging from **BeginTime** to **EndTime.**. | + | | | | | | + | | | | | The value is 0 or a positive number, the value unit is byte. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ExtendParams | 15 | String | Extended field. | This field is not used in this CDR record, the value is empty. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ProductID | 16 | String | Product ID of this service | Product ID: | + | | | | | | + | | | | | OTC_DIS_GEN_STORE | + | | | | | | + | | | | | OTC_DIS_ADV_STORE | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | CET/CEST BeginTime | 17 | String | Start time of the statistics period of a record. | Server Local Time: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | CET/CEST EndTime | 18 | String | End time of the statistics period of a record. | Server Local Time: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Tag | 19 | String | Extended field. | This field is not used in DIS service, the value is empty. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Sample scenario for the user: + +Starting at 11:15(UTC time), a user created a stream containing 2 partitions, the data store time is 1 day. The user put 50 records per second, and each record is 35KB. + +At 12:00 (UTC time) , The partition 1 receives 70,000 records and Partition2 receives 65,000 records.when billing system generates CDR file, CDR records of the stream looks like below: + +The Partition Duration CDR content: +----------------------------------- + +20|20161013150423 \| a21be352dfa7682dfa768dfa7682dfdf \| eu-west-0 *\|* \| *otc.service.type.dis* \| *otc.resource.type. dispartition* \| dis.general.partition \| 1dbcee33-7837-4316-852d-5d9f27f11e28 \|\ *00000000001000003344* \| 20161013111500 \| 20161013115959 \| Duration \| 2700 \| \| OTC_DIS_GEN_TIME \| 20161013120000 \| 20161013125959\| + +20|20161013150423 \| a21be352dfa7682dfa768dfa7682dfdf \| eu-west-0 *\|* \| *otc.service.type.dis* \| *otc.resource.type. dispartition* \| dis.general.partition \| 2cegee33-2173-4982-881c-7w8f27d23h54 \|\ *00000000001000003344* \| 20161013111500 \| 20161013115959 \| Duration \| 2700 \| \| OTC_DIS_GEN_TIME \| 20161013120000 \| 20161013125959\| + +The PUT payload unit CDR content: +--------------------------------- + +20|20161013150423 \| a21be352dfa7682dfa768dfa7682dfdf \| eu-west-0 \| \| otc.service.type.dis \| otc.resource.type. *dispayloadunit* \| dis.general.partition \| 1dbcee33-7837-4316-852d-5d9f27f11e28 \|00000000001000003344 \| 20161013111500 \| 20161013115959 \| InputUnitNum \| 140000 \| 2508800000 \| OTC_DIS_GEN_UNIT \|20161013120000 \| 20161013125959\| + +20|20161013150423 \| a21be352dfa7682dfa768dfa7682dfdf \| eu-west-0 *\|* \| *otc.service.type.dis* \| *otc.resource.type. dispayloadunit* \| dis.general.partition \| 2cegee33-2173-4982-881c-7w8f27d23h54 \|\ *00000000001000003344* \| 20161013111500 \| 20161013115959 \| InputUnitNum \| 130000 \| 2329600000 \| OTC_DIS_GEN_UNIT \|20161013120000 \| 20161013125959\| + +.. note:: + + As our record is 35KB, each record contains two PUT Payload Units. The user put 50 records or 100 PUT Payload Units per second. At 12:00, there are 70000 records or 14000 PUT Payload Unit in partition 1, and 65000 records or 130000 PUT Payload Unit in partition 2. + + The ExtendParams in PUT Payload Unit CDR describe the amount of data the user is actually uploading in the billing period. + +The Date Store Size CDR content: +-------------------------------- + +20|20161013150423 \| a21be352dfa7682dfa768dfa7682dfdf \| eu-west-0 \| \| otc.service.type.dis \| otc.resource.type.disdatasize \| dis.general.partition \| 1dbcee33-7837-4316-852d-5d9f27f11e28 \|00000000001000003344 \| 20161013111500 \| 20161013115959 \| DataStoreSize \| 2508800000\| \| OTC_DIS_GEN_STORE \|20161013120000 \| 20161013125959 \| + +20|20161013150423 \| a21be352dfa7682dfa768dfa7682dfdf \| eu-west-0 \| \| otc.service.type.dis \| otc.resource.type. disdatasize \| dis.general.partition \| 2cegee33-2173-4982-881c-7w8f27d23h54 \|00000000001000003344 \| 20161013111500 \| 20161013115959 \| DataStoreSize \| 2329600000\| \| OTC_DIS_GEN_STORE \|20161013120000 \| 20161013125959 \| + +.. note:: + + Each record is 35KB. The Partition 1 have 70000 records and The partition 2 have 65000 records. The data store time is 1 day so all the data is store in the DIS, so the data store size is 2508800000(70000*35*1024)byte in partition 1, the data store size is 2329600000(65000*35*1024) byte in partition 2. diff --git a/umn/source/cdr_specification/cdr_specification/the_partition_cdr_content.rst b/umn/source/cdr_specification/cdr_specification/the_partition_cdr_content.rst new file mode 100644 index 0000000..7d55ad3 --- /dev/null +++ b/umn/source/cdr_specification/cdr_specification/the_partition_cdr_content.rst @@ -0,0 +1,74 @@ +:original_name: dis_01_0230.html + +.. _dis_01_0230: + +The Partition CDR Content +========================= + +.. table:: **Table 1** DIS Partition CDR Record Content + + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Field | Location | Type | Description | Remarks | + +======================+=============+=============+==================================================+===============================================================================================================================================================================================================================================================================================+ + | RecordType | 1 | String | Type of a record in a file. | Number of normal offline CDR records: 20 | + | | | | | | + | | | | | Number of corrected error offline CDR records: 30 | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | TimeStamp | 2 | String | Time when an offline CDR record is generated. | UTC Format: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | UserID | 3 | String | Unique ID of a subscriber. | The value is the same as the value of **ProjectID**. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | RegionCode | 4 | String | ID of a region. | ID of a region. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | AZCode | 5 | String | Code of an available zone. | The DIS is Region Service, The AZCode is set AZ01 by default. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | CloudServiceTypeCode | 6 | String | Type of a cloud service. | Cloud service type corresponding to **ResourceID**. For example, cloudservicetype for VM in **EC2** is recorded as **otc.service.type.ec2**, and cloudservicetype for VM in **VDC** is recorded as **otc\ .\ service.type.vdc**. | + | | | | | | + | | | | | For DIS: the string is **otc.service.type.dis** | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ResourceTypeCode | 7 | String | Type of a cloud resource. | Cloud resource type corresponding to **ResourceID**, such as VM. | + | | | | | | + | | | | | **ResourceType** is a resource type specified by **CloudServiceType**, and **CloudServiceType** contains multiple sets of **ResourceType**. For example, EC2 is provided by VM, Volume, IP, and Image, and EC2 is a **CloudServiceType**, and VM, Volume, IP, and Image are **ResourceType**. | + | | | | | | + | | | | | For DIS: the string is **otc.resource.type.dis** | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ResourceSpecCode | 8 | String | Code of specified resource specifications. | There are two type resourcespeccode for DIS: | + | | | | | | + | | | | | dis.general.partition | + | | | | | | + | | | | | dis.advanced.partition | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ResourceID | 9 | String | ID of a resource instance. | ID of the partition in the DIS stream instance. | + | | | | | | + | | | | | The ID is different from partitions. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | BSSParams | 10 | String | Extended operation parameter. | The contract ID of one tenant. The contract ID is the xdomain_id of user token. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | BeginTime | 11 | String | Start time of the statistics period of a record. | UTC Format: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | EndTime | 12 | String | End time of the statistics period of a record. | UTC Format: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Duration | 13 | String | Name of a specific accumulative factor. | Accumulative factor: Duration | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Time | 14 | Double | Accumulative value of **AccumulateFactorName**. | Accumulative value of duration in the period ranging from BeginTime to EndTime multiply the number of partitions. The value is 0 or a positive number. | + | | | | | | + | | | | | The time unit is hour. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ExtendParams | 15 | String | Extended field. | This field is not used in this CDR record, the value is empty. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ProductID | 16 | String | Product ID of this service | Product ID : | + | | | | | | + | | | | | OTC_DIS_GEN_TIME | + | | | | | | + | | | | | OTC_DIS_ADV_TIME | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | CET/CEST BeginTime | 17 | String | Start time of the statistics period of a record. | Server Local Time: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | CET/CEST EndTime | 18 | String | End time of the statistics period of a record. | Server Local Time: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Tag | 19 | String | Extended field. | This field is not used in DIS service, the value is empty. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Example of Partition Duration CDR content: + +20|20161013150423 \| a21be352dfa7682dfa768dfa7682dfdf \| eu-west-0 *\|* \| *otc.service.type.dis* \| *otc.resource.type.dispartition* \| dis.general.partition \| 2cegee33-2173-4982-881c-7w8f27d23h54 \|\ *00000000001000003344* \| 20161013130000 \| 20161013135959 \| Duration \| 3600 \| \| OTC_DIS_GEN_TIME \| 20161013140000 \| 20161013145959\| diff --git a/umn/source/cdr_specification/cdr_specification/the_put_payload_unit_cdr_content.rst b/umn/source/cdr_specification/cdr_specification/the_put_payload_unit_cdr_content.rst new file mode 100644 index 0000000..713a09b --- /dev/null +++ b/umn/source/cdr_specification/cdr_specification/the_put_payload_unit_cdr_content.rst @@ -0,0 +1,78 @@ +:original_name: dis_01_0231.html + +.. _dis_01_0231: + +The PUT Payload Unit CDR Content +================================ + +.. table:: **Table 1** DIS PUT Payload Unit CDR Record Content + + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Field | Location | Type | Description | Remarks | + +======================+=============+=============+==================================================+===============================================================================================================================================================================================================================================================================================+ + | RecordType | 1 | String | Type of a record in a file. | Number of normal offline CDR records: 20 | + | | | | | | + | | | | | Number of corrected error offline CDR records: 30 | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | TimeStamp | 2 | String | Time when an offline CDR record is generated. | UTC Format: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | UserID | 3 | String | Unique ID of a subscriber. | The value is the same as the value of **ProjectID**. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | RegionCode | 4 | String | ID of a region. | ID of a region. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | AZCode | 5 | String | Code of an available zone. | Code of an available zone. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | CloudServiceTypeCode | 6 | String | Type of a cloud service. | Cloud service type corresponding to **ResourceID**. For example, cloudservicetype for VM in **EC2** is recorded as **otc.service.type.ec2**, and cloudservicetype for VM in **VDC** is recorded as **otc.service.type.vdc**. | + | | | | | | + | | | | | For DIS: the string is **otc.service.type.dis** | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ResourceTypeCode | 7 | String | Type of a cloud resource. | Cloud resource type corresponding to **ResourceID**, such as VM. | + | | | | | | + | | | | | **ResourceType** is a resource type specified by **CloudServiceType**, and **CloudServiceType** contains multiple sets of **ResourceType**. For example, EC2 is provided by VM, Volume, IP, and Image, and EC2 is a **CloudServiceType**, and VM, Volume, IP, and Image are **ResourceType**. | + | | | | | | + | | | | | For DIS: the string is **otc.resource.type.dis** | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ResourceSpecCode | 8 | String | Code of specified resource specifications. | There are two type resourcespeccode for DIS: | + | | | | | | + | | | | | dis.general.partition | + | | | | | | + | | | | | dis.advanced.partition | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ResourceID | 9 | String | ID of a resource instance. | ID of the partition in the DIS stream instance. | + | | | | | | + | | | | | The ID is different from partitions. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | BSSParams | 10 | String | Extended operation parameter. | The contract ID of one tenant. The contract ID is the xdomain_id of user token. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | BeginTime | 11 | String | Start time of the statistics period of a record. | UTC Format: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | EndTime | 12 | String | End time of the statistics period of a record. | UTC Format: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | InputTraffic | 13 | String | Name of a specific accumulative factor. | Accumulative factor: InputUnitNum | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | InputTrafficValue | 14 | Double | Accumulative value of **AccumulateFactorName**. | Accumulative value of InputUnitNum. | + | | | | | | + | | | | | This is the total Num of input record(sum/million) in the partition during this time period. | + | | | | | | + | | | | | The value unit is sum/million | + | | | | | | + | | | | | . | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ExtendParams | 15 | String | Extended field. | This field indicates the amount of data the user is actually putting. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | ProductID | 16 | String | Product ID of this service | Product ID: | + | | | | | | + | | | | | OTC_DIS_GEN_UNIT | + | | | | | | + | | | | | OTC_DIS_ADV_UNIT | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | CET/CEST BeginTime | 17 | String | Start time of the statistics period of a record. | Server Local Time: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | CET/CEST EndTime | 18 | String | End time of the statistics period of a record. | Server Local Time: YYYYMMDDHH24MISS | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Tag | 19 | String | Extended field. | This field is not used in DIS service, the value is empty. | + +----------------------+-------------+-------------+--------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Example of PUT Payload Unit CDR content: + +20|20161013150423 \| a21be352dfa7682dfa768dfa7682dfdf \| eu-west-0 *\|* \| *otc.service.type.dis* \| *otc.resource.type.dispayloadunit* \| dis.general.partition \| 1dbcee33-7837-4316-852d-5d9f27f11e28 \|\ *00000000001000003344* \| 20161013130000 \| 20161013135959 \| InputUnitNum \| 12000 \| 3201284 \| OTC_DIS_GEN_UNIT \|20161013140000 \| 20161013145959\| diff --git a/umn/source/cdr_specification/index.rst b/umn/source/cdr_specification/index.rst new file mode 100644 index 0000000..fd81e8c --- /dev/null +++ b/umn/source/cdr_specification/index.rst @@ -0,0 +1,18 @@ +:original_name: dis_01_0222.html + +.. _dis_01_0222: + +CDR Specification +================= + +- :ref:`Billing Model ` +- :ref:`CDR Folder and File Name ` +- :ref:`CDR Specification ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + billing_model + cdr_folder_and_file_name/index + cdr_specification/index diff --git a/umn/source/change_history.rst b/umn/source/change_history.rst new file mode 100644 index 0000000..c823d56 --- /dev/null +++ b/umn/source/change_history.rst @@ -0,0 +1,38 @@ +:original_name: dis_01_0014.html + +.. _dis_01_0014: + +Change History +============== + ++-----------------------------------+----------------------------------------------------------------------------------+ +| Release Date | What's New | ++===================================+==================================================================================+ +| 2022-06-15 | Modified the following section: | +| | | +| | :ref:`What Is DIS? ` | +| | | +| | Added the following sections: | +| | | +| | :ref:`CDR Specification ` | ++-----------------------------------+----------------------------------------------------------------------------------+ +| 2020-01-03 | Modified the following section: | +| | | +| | :ref:`Viewing Stream Monitoring Metrics ` | ++-----------------------------------+----------------------------------------------------------------------------------+ +| 2019-06-12 | Modified the following section: | +| | | +| | :ref:`Related Services ` | ++-----------------------------------+----------------------------------------------------------------------------------+ +| 2019-05-15 | Modified the following section: | +| | | +| | :ref:`What Is DIS? ` | ++-----------------------------------+----------------------------------------------------------------------------------+ +| 2019-01-29 | Modified the following section: | +| | | +| | :ref:`Step 2: Preparing a DIS Application Development Environment ` | ++-----------------------------------+----------------------------------------------------------------------------------+ +| 2018-11-09 | Accepted in OTC 3.2. | ++-----------------------------------+----------------------------------------------------------------------------------+ +| 2018-03-16 | This is the first official release. | ++-----------------------------------+----------------------------------------------------------------------------------+ diff --git a/umn/source/conf.py b/umn/source/conf.py index 2b714de..0083128 100644 --- a/umn/source/conf.py +++ b/umn/source/conf.py @@ -78,7 +78,7 @@ html_theme = 'otcdocs' # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - "disable_search": "True", + "disable_search": True, "site_name": "Internal Documentation Portal", "logo_url": "https://docs-int.otc-service.com", } diff --git a/umn/source/dis_sdk_usage_guide/configuring_sample_projects.rst b/umn/source/dis_sdk_usage_guide/configuring_sample_projects.rst new file mode 100644 index 0000000..460175e --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/configuring_sample_projects.rst @@ -0,0 +1,74 @@ +:original_name: dis_06_0013.html + +.. _dis_06_0013: + +Configuring Sample Projects +=========================== + +The **dis-sdk-**\ *1.2.3*\ **.zip** package of the latest version in :ref:`Downloading SDKs ` provides a sample project. You can use a development tool (such as Eclipse) on the local server to compile and run the sample project. You can also develop your applications based on the sample project. The sample project code is available in the **\\dis-sdk-demo\\src\\com\\bigdata\\dis\\sdk\\demo** directory. + +================= ==================== +Sample Code Description +================= ==================== +ConsumerDemo.java How to download data +ProducerDemo.java How to upload data +================= ==================== + +Procedure +--------- + +#. Decompress the **dis-sdk-**\ *1.2.3*\ **.zip** package obtained in :ref:`Downloading SDKs ` to obtain the **dis-sdk-demo** package and sample project. +#. Import the Eclipse project. + + a. Start Eclipse and choose **File** > **Import**. The **Import** dialog box is displayed. + + b. Choose **General** > **Existing Projects into Workspace** and click **Next**. The **Import** dialog box is displayed. + + c. Click **Browse** and select a save location for the **dis-sdk-demo** sample project. In the **Projects** area, select a sample project, as shown in :ref:`Figure 1 `. + + .. _dis_06_0013__f23cd224614d04888bfa7c759107662f3: + + .. figure:: /_static/images/en-us_image_0000001222097302.png + :alt: **Figure 1** Import Projects + + **Figure 1** Import Projects + + d. Click **Finish** to import the project. + +#. Configure the demo project. + + a. Set the project code to **UTF-8**. + + #. In the navigation tree, right-click the required project under **Project Explorer** and choose **Properties** from the shortcut menu. The **Properties for dis-sdk-demo** dialog box is displayed. + #. In the navigation tree, choose **Resource**. The **Resource** page is displayed in the right pane. + #. In the **Other** drop-down list, select **UTF-8**. + #. Click **Apply and Close**. + + + .. figure:: /_static/images/en-us_image_0000001222417262.png + :alt: **Figure 2** Resource + + **Figure 2** Resource + + b. Import a JAR dependency package. + + #. In the navigation tree, right-click the required project under **Project Explorer** and choose **Properties** from the shortcut menu. The **Properties for dis-sdk-demo** dialog box is displayed. + #. In the navigation tree, choose **Java Build Path**. The **Java Build Path** page is displayed in the right pane. + #. Click the **Libraries** tab, and then click **Add External JARs**. The **JAR Selection** dialog box is displayed. + #. Select the location of the **dis-sdk** folder in the decompressed **dis-sdk-**\ 1.2.3 file, and then click **Open**. + #. On the **Properties for dis-sdk-demo** page, click **Apply and Close** to import all the **.jar** files in the current path and the **third_lib** folder to the project. + + + .. figure:: /_static/images/en-us_image_0000001266977433.png + :alt: **Figure 3** Java Build Path + + **Figure 3** Java Build Path + + c. Add the JDK. + + #. In the navigation tree, right-click the required project under **Project Explorer** and choose **Properties** from the shortcut menu. The **Properties for dis-sdk-demo** dialog box is displayed. + #. In the navigation tree, choose **Java Build Path**. The **Java Build Path** page is displayed in the right pane. + #. Click the **Libraries** tab, and then click **Add Library**. The **Add Library** dialog box is displayed. + #. Select **JRE System Library** and click **Next**. Verify that the version of **Workspace default JRE** is **jdk1.8** or later. + #. Click **Finish** to exit the **Add Library** dialog box. + #. Click **Apply and Close**. diff --git a/umn/source/dis_sdk_usage_guide/connecting_to_kafka_consumer.rst b/umn/source/dis_sdk_usage_guide/connecting_to_kafka_consumer.rst new file mode 100644 index 0000000..9f516a8 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/connecting_to_kafka_consumer.rst @@ -0,0 +1,8 @@ +:original_name: dis_06_0020.html + +.. _dis_06_0020: + +Connecting to kafka consumer +============================ + +The DIS SDK provides compatibility with native kafka client APIs. diff --git a/umn/source/dis_sdk_usage_guide/creating_a_stream.rst b/umn/source/dis_sdk_usage_guide/creating_a_stream.rst new file mode 100644 index 0000000..9e9e2d9 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/creating_a_stream.rst @@ -0,0 +1,30 @@ +:original_name: dis_06_0501.html + +.. _dis_06_0501: + +Creating a Stream +================= + +Initialize a DIS client instance named **dic**. For details, see :ref:`Initializing a DIS Client `. + +When you use the DIS SDK to create a DIS stream, specify the stream name, number of partitions in the stream, and stream type. + +**STREAM_TYPE_COMMON** indicates a common stream, and **STREAM_TYPE_ADVANCED** indicates an advanced stream. + +.. code-block:: + + //Configure the channel name. + String streamName = "myStream"; + //Configure the number of partitions in the stream. + Integer partitionCount = new Integer(3); + CreateStreamRequest createStreamRequest = new CreateStreamRequest(); + createStreamRequest.setStreamName(streamName); + createStreamRequest.setPartitionCount(partitionCount); + //Configure the stream type. + createStreamRequest.setStreamType(CreateStreamRequest.STREAM_TYPE_COMMON); + +After configuring **CreateStreamRequest**, you can create a stream by calling **createStream**. + +.. code-block:: + + dic.createStream(createStreamRequest); diff --git a/umn/source/dis_sdk_usage_guide/deleting_a_stream.rst b/umn/source/dis_sdk_usage_guide/deleting_a_stream.rst new file mode 100644 index 0000000..65629bf --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/deleting_a_stream.rst @@ -0,0 +1,23 @@ +:original_name: dis_06_0502.html + +.. _dis_06_0502: + +Deleting a Stream +================= + +Initialize a DIS client instance named **dic**. For details, see :ref:`Initializing a DIS Client `. + +Use the DIS SDK to delete a specified DIS stream. + +.. code-block:: + + //Specify the name of the stream to be deleted. + String streamName = "myStream"; + DeleteStreamRequest deleteStreamRequest = new DeleteStreamRequest(); + deleteStreamRequest.setStreamName(streamName); + +After configuring **DeleteStreamRequest**, you can delete a stream by calling deleteStream. + +.. code-block:: + + dic.deleteStream(deleteStreamRequest); diff --git a/umn/source/dis_sdk_usage_guide/downloading_sdks.rst b/umn/source/dis_sdk_usage_guide/downloading_sdks.rst new file mode 100644 index 0000000..224fc00 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/downloading_sdks.rst @@ -0,0 +1,10 @@ +:original_name: dis_06_0005.html + +.. _dis_06_0005: + +Downloading SDKs +================ + +Data Ingestion Service Software Development Kit (DIS SDK) is the encapsulation of RESTful APIs provided by DIS to simplify development. You can use DIS by calling API functions provided by DIS SDK. + +Download the package from https://dis.obs.eu-de.otc.t-systems.com/dis/download/dis-sdk-1.2.3.zip. diff --git a/umn/source/dis_sdk_usage_guide/downloading_streaming_data.rst b/umn/source/dis_sdk_usage_guide/downloading_streaming_data.rst new file mode 100644 index 0000000..4a709bf --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/downloading_streaming_data.rst @@ -0,0 +1,81 @@ +:original_name: dis_06_0017.html + +.. _dis_06_0017: + +Downloading Streaming Data +========================== + +Sample Code +----------- + +Use the initialized client instance to obtain data through the DIS stream. The sample code is available in the **ConsumerDemo.java** file under the **dis-sdk-demo\\src\\main\\java\\com\\bigdata\\dis\\sdk\\demo** directory. + +The value of **streamName** must be the same as that of **Stream Name** configured in :ref:`Step 1: Creating a DIS Stream `. + +.. code-block:: + + //Configure the stream name. + String streamName = "streamName"; + // Configure the ID of the partition for data download. + String partitionId = "0"; + //Configure the sequence number for data download. + String startingSequenceNumber = "0"; + //Configure the data download mode. + String cursorType = PartitionCursorTypeEnum.AT_SEQUENCE_NUMBER.name(); + try + { + //Obtain data cursors. + GetPartitionCursorRequest request = new GetPartitionCursorRequest(); + request.setStreamName(streamName); + request.setPartitionId(partitionId); + request.setStartingSequenceNumber(startingSequenceNumber); + request.setCursorType(cursorType); + GetPartitionCursorResult response = dic.getPartitionCursor(request); + String cursor = response.getPartitionCursor(); + log.info("Get stream {}[partitionId={}] cursor success : {}", streamName, partitionId, cursor); + GetRecordsRequest recordsRequest = newGetRecordsRequest(); + GetRecordsResult recordResponse = null; + while (true) + { + recordsRequest.setPartitionCursor(cursor); + recordsRequest.setLimit(2); + recordResponse = dic.getRecords(recordsRequest); + //Obtain the next-batch data cursors. + cursor = recordResponse.getNextPartitionCursor(); + for (Record record : recordResponse.getRecords()) + { + log.info("Get Record [{}], partitionKey [{}], sequenceNumber [{}].", + new String(record.getData().array()), + record.getPartitionKey(), + record.getSequenceNumber()); + } + Thread.sleep(1000); + } + } + catch (DISClientException e) + { + log.error("Failed to get a normal response, please check params and retry. Error message [{}]", e.getMessage(), e); + } + catch (ResourceAccessException e) + { + log.error("Failed to access endpoint. Error message [{}]", e.getMessage(), e); + } + catch (Exception e) + { + log.error(e.getMessage(), e); + } + +Running the Program +------------------- + +Right-click the program and choose **Run As > 1 Java Application** from the shortcut menu. If the program runs successfully, the information similar to the following is displayed on the console: + +.. code-block:: + + 14:55:42.954 [main] INFOcom.bigdata.dis.sdk.DISConfig - get from classLoader + 14:55:44.103 [main] INFOcom.bigdata.dis.sdk.util.config.ConfigurationUtils - get from classLoader + 14:55:44.105 [main] INFOcom.bigdata.dis.sdk.util.config.ConfigurationUtils - propertyMapFromFile size : 2 + 14:55:45.235 [main] INFOcom.bigdata.dis.sdk.demo.ConsumerDemo - Get stream streamName[partitionId=0] cursor success : eyJnZXRJdGVyYXRvclBhcmFtIjp7InN0cmVhbS1uYW1lIjoiZGlzLTEzbW9uZXkiLCJwYXJ0aXRpb24taWQiOiIwIiwiY3Vyc29yLXR5cGUiOiJBVF9TRVFVRU5DRV9OVU1CRVIiLCJzdGFydGluZy1zZXF1ZW5jZS1udW1iZXIiOiIxMDY4OTcyIn0sImdlbmVyYXRlVGltZXN0YW1wIjoxNTEzNjY2NjMxMTYxfQ + 14:55:45.305 [main] INFOcom.bigdata.dis.sdk.demo.ConsumerDemo - Get Record [hello world.], partitionKey [964885], sequenceNumber [0]. + 14:55:45.305 [main] INFOcom.bigdata.dis.sdk.demo.ConsumerDemo - Get Record [hello world.], partitionKey [910960], sequenceNumber [1]. + 14:55:46.359 [main] INFOcom.bigdata.dis.sdk.demo.ConsumerDemo - Get Record [hello world.], partitionKey [528377], sequenceNumber [2]. diff --git a/umn/source/dis_sdk_usage_guide/importing_third-party_dependency_packages.rst b/umn/source/dis_sdk_usage_guide/importing_third-party_dependency_packages.rst new file mode 100644 index 0000000..1a559a0 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/importing_third-party_dependency_packages.rst @@ -0,0 +1,9 @@ +:original_name: dis_06_0014.html + +.. _dis_06_0014: + +Importing Third-party Dependency Packages +========================================= + +#. In the navigation tree, click **Project Explorer**, right-click a project and choose **Properties > Java Build Path > Add External JARs**. +#. Locate the directory containing the decompressed **dis-sdk** folder, and import all the files with the extension **JAR** in the current directory and the **third_party** folder to the project. diff --git a/umn/source/dis_sdk_usage_guide/index.rst b/umn/source/dis_sdk_usage_guide/index.rst new file mode 100644 index 0000000..69c2a80 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/index.rst @@ -0,0 +1,40 @@ +:original_name: dis_06_0011.html + +.. _dis_06_0011: + +DIS SDK Usage Guide +=================== + +- :ref:`Downloading SDKs ` +- :ref:`Preparing the Environment ` +- :ref:`Connecting to kafka consumer ` +- :ref:`Configuring Sample Projects ` +- :ref:`Importing Third-party Dependency Packages ` +- :ref:`Initializing a DIS Client ` +- :ref:`Creating a Stream ` +- :ref:`Deleting a Stream ` +- :ref:`Querying a Stream List ` +- :ref:`Querying Stream Details ` +- :ref:`Uploading Streaming Data ` +- :ref:`Downloading Streaming Data ` +- :ref:`Uploading Real-Time Files ` +- :ref:`Obtaining the Data Cursor ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + downloading_sdks + preparing_the_environment + connecting_to_kafka_consumer + configuring_sample_projects + importing_third-party_dependency_packages + initializing_a_dis_client + creating_a_stream + deleting_a_stream + querying_a_stream_list + querying_stream_details + uploading_streaming_data + downloading_streaming_data + uploading_real-time_files/index + obtaining_the_data_cursor diff --git a/umn/source/dis_sdk_usage_guide/initializing_a_dis_client.rst b/umn/source/dis_sdk_usage_guide/initializing_a_dis_client.rst new file mode 100644 index 0000000..7bf512e --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/initializing_a_dis_client.rst @@ -0,0 +1,35 @@ +:original_name: dis_06_0015.html + +.. _dis_06_0015: + +Initializing a DIS Client +========================= + +You can use either of the following methods to initialize a DIS SDK client instance: For details about **endpoint**, **ak**, **sk**, **region**, and **projectId**, see :ref:`Obtaining Authentication Information `. + +- Use the code to initialize the DIS SDK client instance. For details about the code example, see the **ProducerDemo.java** file. + + .. code-block:: + + //Create a DIS client instance. + DIS dic = DISClientBuilder.standard() + .withEndpoint("https://ip:port") + .withAk("xxxx") + .withSk("xxxx") + .withProjectId("xxxxxxx") + .withRegion("xxxx") + .build(); + +- Use the configuration file to initialize a DIS SDK client instance. + + Add the following configuration items to the **dis.propertites** file in the **dis-sdk-demo\\resources** directory: + + - ak/sk: AK/SK created on the IAM + - region: region of the stream + - endpoint: access address of the DIS + - projectId: project ID of the stream + + .. code-block:: + + //Create a DIS client instance. + DIS dic = DISClientBuilder.standard().build(); diff --git a/umn/source/dis_sdk_usage_guide/obtaining_the_data_cursor.rst b/umn/source/dis_sdk_usage_guide/obtaining_the_data_cursor.rst new file mode 100644 index 0000000..8cc0ce0 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/obtaining_the_data_cursor.rst @@ -0,0 +1,29 @@ +:original_name: dis_06_0505.html + +.. _dis_06_0505: + +Obtaining the Data Cursor +========================= + +Initialize a DIS client instance named **dic**. For details, see :ref:`Initializing a DIS Client `. + +Use the DIS SDK to obtain the information about the data cursor. + +.. code-block:: + + //Configure the stream name. + String streamName = "myStream"; + // Configure the ID of the partition for data download. + String partitionId = "0"; + //Configure the sequence number for data download. + String startingSequenceNumber = "0"; + //Configure the data download mode. + String cursorType = PartitionCursorTypeEnum.AT_SEQUENCE_NUMBER.name(); + + GetPartitionCursorRequest request = new GetPartitionCursorRequest(); + request.setStreamName(streamName); + request.setPartitionId(partitionId); + request.setStartingSequenceNumber(startingSequenceNumber); + request.setCursorType(cursorType); + GetPartitionCursorResult response = dic.getPartitionCursor(request); + String cursor = response.getPartitionCursor(); diff --git a/umn/source/dis_sdk_usage_guide/preparing_the_environment.rst b/umn/source/dis_sdk_usage_guide/preparing_the_environment.rst new file mode 100644 index 0000000..3422463 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/preparing_the_environment.rst @@ -0,0 +1,10 @@ +:original_name: dis_06_0012.html + +.. _dis_06_0012: + +Preparing the Environment +========================= + +- Download JDK1.8 or a later version from the `Oracle official website `__ and install it, and configure Java environment variables. +- Download Eclipse IDE for Java Developers of the latest version from the `Eclipse's official website `__, and install it. +- Configure the JDK in Eclipse. diff --git a/umn/source/dis_sdk_usage_guide/querying_a_stream_list.rst b/umn/source/dis_sdk_usage_guide/querying_a_stream_list.rst new file mode 100644 index 0000000..2ac580f --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/querying_a_stream_list.rst @@ -0,0 +1,24 @@ +:original_name: dis_06_0503.html + +.. _dis_06_0503: + +Querying a Stream List +====================== + +Initialize a DIS client instance named **dic**. For details, see :ref:`Initializing a DIS Client `. + +Use the DIS SDK to list active streams. + +Use the **setLimit** method to set the number of streams returned each time. If **setLimit** is not specified, a maximum of 10 streams are returned by default. + +.. code-block:: + + ListStreamsRequest listStreamsRequest = new ListStreamsRequest(); + listStreamsRequest.setLimit(5); + System.out.println("listStream: " + JsonUtils.objToJson(dic.listStreams(listStreamsRequest))); + +The returned stream list is as follows: + +.. code-block:: + + listStream: {"total_number":20,"stream_names":["Stream0","Stream1","Stream2","Stream3","Stream4"],"has_more_streams":true} diff --git a/umn/source/dis_sdk_usage_guide/querying_stream_details.rst b/umn/source/dis_sdk_usage_guide/querying_stream_details.rst new file mode 100644 index 0000000..d90e4b8 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/querying_stream_details.rst @@ -0,0 +1,23 @@ +:original_name: dis_06_0504.html + +.. _dis_06_0504: + +Querying Stream Details +======================= + +Initialize a DIS client instance named **dic**. For details, see :ref:`Initializing a DIS Client `. + +Use the DIS SDK to query the details about a specified stream. + +.. code-block:: + + String streamName = "myStream"; + DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); + describeStreamRequest.setStreamName(streamName); + System.out.println("descStream: " + JsonUtils.objToJson(dic.describeStream(describeStreamRequest))); + +The returned stream details are as follows: + +.. code-block:: + + descStream: {"stream_name":"myStream","create_time":1515140868451,"last_modified_time":1515140868451,"retention_period":24,"status":"RUNNING","stream_type":"ADVANCED","partitions":[{"status":"ACTIVE","partition_id":"shardId-0000000000","hash_range":"[0 : 4611686018427387902]","sequence_number_range":"[0 : 0]"},{"status":"ACTIVE","partition_id":"shardId-0000000001","hash_range":"[4611686018427387903 : 9223372036854775807]","sequence_number_range":"[0 : 0]"}],"has_more_partitions":false} diff --git a/umn/source/dis_sdk_usage_guide/uploading_real-time_files/creating_a_demo_class.rst b/umn/source/dis_sdk_usage_guide/uploading_real-time_files/creating_a_demo_class.rst new file mode 100644 index 0000000..292ffc4 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/uploading_real-time_files/creating_a_demo_class.rst @@ -0,0 +1,73 @@ +:original_name: dis_06_0506.html + +.. _dis_06_0506: + +Creating a Demo Class +===================== + +Creating a FileProducerAsyncDemo Class +-------------------------------------- + +In the Project Explorer pane, right-click **com.bigdata.dis.sdk.demo**, and choose **New > Class** from the shortcut menu. The **New Java Class** dialog box is displayed. Set the category name to **FileProducerAsyncDemo**. Add the following codes to the new category: + +.. code-block:: + + package com.bigdata.dis.sdk.demo; + + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + + import com.bigdata.dis.data.iface.request.PutFilesRequest; + import com.bigdata.dis.data.iface.response.PutFilesResult; + import com.bigdata.dis.sdk.DISAsync; + import com.bigdata.dis.sdk.DISClientAsyncBuilder; + import com.bigdata.dis.sdk.core.builder.DefaultExecutorFactory; + import com.bigdata.dis.sdk.core.handler.AsyncHandler; + + public class FileProducerAsyncDemo + { + private static final Logger log = LoggerFactory.getLogger(FileProducerAsyncDemo.class); + + public static void main(String args[]) + throws Exception + { + DISAsync dicAsync = DISClientAsyncBuilder.standard() + .withEndpoint("https://ip:port") + .withAk("xxxx") + .withSk("xxxx") + .withProjectId("xxxx") + .withRegion("xxxx") + .withExecutorFactory(new DefaultExecutorFactory()) + .build(); + + // Configure the stream name. + String streamName = "fileUploadStream"; + + PutFilesRequest putFilesRequest = new PutFilesRequest(); + putFilesRequest.setStreamName(streamName); + + //Configure the path of the file to be uploaded. + putFilesRequest.setFilePath("xxxx"); + + //Configure the file name for storing the uploaded real-time data file. + putFilesRequest.setFileName("xxxx"); + + dicAsync.putFilesAsync(putFilesRequest, new AsyncHandler() + { + + @Override + public void onSuccess(PutFilesResult result) + { + log.info("Upload file to DIS successful!"); + } + + @Override + public void onError(Exception exception) + { + log.error("Fail to upload file to DIS.", exception); + } + }); + + + } + } diff --git a/umn/source/dis_sdk_usage_guide/uploading_real-time_files/index.rst b/umn/source/dis_sdk_usage_guide/uploading_real-time_files/index.rst new file mode 100644 index 0000000..2ac6a21 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/uploading_real-time_files/index.rst @@ -0,0 +1,18 @@ +:original_name: dis_06_0022.html + +.. _dis_06_0022: + +Uploading Real-Time Files +========================= + +- :ref:`Creating a Demo Class ` +- :ref:`Initializing a DIS SDK Asynchronous Client Instance ` +- :ref:`Uploading Files ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + creating_a_demo_class + initializing_a_dis_sdk_asynchronous_client_instance + uploading_files diff --git a/umn/source/dis_sdk_usage_guide/uploading_real-time_files/initializing_a_dis_sdk_asynchronous_client_instance.rst b/umn/source/dis_sdk_usage_guide/uploading_real-time_files/initializing_a_dis_sdk_asynchronous_client_instance.rst new file mode 100644 index 0000000..97c1352 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/uploading_real-time_files/initializing_a_dis_sdk_asynchronous_client_instance.rst @@ -0,0 +1,20 @@ +:original_name: dis_06_0023.html + +.. _dis_06_0023: + +Initializing a DIS SDK Asynchronous Client Instance +=================================================== + +Initialize a DIS SDK asynchronous client instance. For details about **endpoint**, **ak**, **sk**, **region**, and **projectId**, see :ref:`Obtaining Authentication Information `. + +.. code-block:: + + //Create a DIS client instance. + DISAsync dicAsync = DISClientAsyncBuilder.standard() + .withEndpoint("https://ip:port") + .withAk("xxxx") + .withSk("xxxx") + .withProjectId("xxxx") + .withRegion("xxxx") + .withExecutorFactory(new DefaultExecutorFactory()) + .build(); diff --git a/umn/source/dis_sdk_usage_guide/uploading_real-time_files/uploading_files.rst b/umn/source/dis_sdk_usage_guide/uploading_real-time_files/uploading_files.rst new file mode 100644 index 0000000..7dd1a06 --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/uploading_real-time_files/uploading_files.rst @@ -0,0 +1,50 @@ +:original_name: dis_06_0507.html + +.. _dis_06_0507: + +Uploading Files +=============== + +Configuring a Stream for Uploading Real-Time Files +-------------------------------------------------- + +You must use the stream enabled to transmit real-time files. Here, **fileUploadStream** is used as an example. + +The value of **streamName** must be the same as that of **Stream Name** configured in :ref:`Step 1: Creating a DIS Stream `. + +.. code-block:: + + //Configure the stream name. + String streamName = "fileUploadStream"; + +Specifying the Path of the Real-Time File to Be Uploaded +-------------------------------------------------------- + +Enter the absolute path of the file to be uploaded. + +.. code-block:: + + //Specify the path of the file to be uploaded. + putFilesRequest.setFilePath("xxxx"); + +Specifying the Name of the File for Storing the Uploaded Real-Time Data +----------------------------------------------------------------------- + +Set the file name to be uploaded. If the file name contains the backslash (\\), use the escape character to escape the backslash. + +.. code-block:: + + //Specify the file name for storing the uploaded file. + putFilesRequest.setFileName("xxxx"); + +Running the Program +------------------- + +Right-click the program and choose **Run As > 1 Java Application** from the shortcut menu. If the program is run successfully, you can view the information similar to the following on the console: + +.. code-block:: + + 17:47:09.103 [main] INFO com.bigdata.dis.sdk.DISConfig - get from classLoader + 17:47:10.275 [pool-2-thread-1] INFO com.bigdata.dis.sdk.util.config.ConfigurationUtils - get from classLoader + 17:47:10.291 [pool-2-thread-1] INFO com.bigdata.dis.sdk.util.config.ConfigurationUtils - propertyMapFromFile size : 2 + 17:47:20.012 [pool-2-thread-1] INFO com.bigdata.dis.sdk.demo.FileProducerAsyncDemo - Upload file to DIS successful! diff --git a/umn/source/dis_sdk_usage_guide/uploading_streaming_data.rst b/umn/source/dis_sdk_usage_guide/uploading_streaming_data.rst new file mode 100644 index 0000000..55e1bbf --- /dev/null +++ b/umn/source/dis_sdk_usage_guide/uploading_streaming_data.rst @@ -0,0 +1,94 @@ +:original_name: dis_06_0016.html + +.. _dis_06_0016: + +Uploading Streaming Data +======================== + +Sample Code +----------- + +Use the initialized client instance to upload your streaming data to DIS. The sample code is available in the **ProducerDemo.java** file in the **dis-sdk-demo\\src\\com\\bigdata\\dis\\sdk\\demo** directory. + +The value of **streamName** must be the same as that of **Stream Name** configured in :ref:`Step 1: Creating a DIS Stream `. + +The code for uploading streaming data is as follows: + +.. code-block:: + + //Initialize an asynchronous sending client. + DISConfig disConfig = new DISConfig().setAK("xxxx").setSK("xxxx").setProjectId("xxxx").setRegion("xxxx").setEndpoint("xxxx"); + DISProducer producer = new DISProducer(disConfig); + + //Configure the stream name. + String streamName = "streamName"; + //Configure the data to be uploaded. + String message = "hello world."; + ByteBuffer buffer = ByteBuffer.wrap(message.getBytes()); + PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry(); + putRecordsRequestEntry.setData(buffer); + putRecordsRequestEntry.setPartitionKey(String.valueOf(ThreadLocalRandom.current().nextInt(1000000))); + log.info("========== BEGIN PUT ============"); + + int count = 10; + CountDownLatch cd = new CountDownLatch(count); + for (int i = 0; i < count; i++) + { + putRecordsRequestEntry.setPartitionKey(String.valueOf(ThreadLocalRandom.current().nextInt(1000000))); + try + { + producer.putRecordAsync(streamName, putRecordsRequestEntry, new AsyncHandler() + { + @Override + public void onSuccess(PutRecordsResultEntry result) + { + log.info(result.toString()); + cd.countDown(); + } + + @Override + public void onError(Exception exception) + { + log.error(exception.getMessage(), exception); + cd.countDown(); + } + }); + } + catch (Exception e) + { + log.error(e.getMessage(), e); + cd.countDown(); + } + } + + cd.await(); + log.info("========== PUT OVER ============"); + producer.close(); + +Running the Program +------------------- + +Right-click the program and choose **Run As > 1 Java Application** from the shortcut menu. If the program runs successfully, the information similar to the following is displayed on the console: + +.. code-block:: + + 17:27:49.130 [main] INFO com.bigdata.dis.sdk.DISConfig - get from classLoader + 17:27:49.142 [Sender Thread] DEBUG com.bigdata.dis.sdk.producer.internals.Sender - Starting Kafka producer I/O thread. + 17:27:49.145 [main] INFO DISProducerDemo - ========== BEGIN PUT ============ + 17:27:49.202 [Sender Thread] DEBUG com.bigdata.dis.sdk.producer.internals.Sender - begin to send : 1 + 17:27:49.203 [Sender Thread] DEBUG com.bigdata.dis.sdk.producer.internals.Sender - batch size: 10, 120 + 17:27:50.197 [pool-2-thread-1] INFO com.bigdata.dis.sdk.util.config.ConfigurationUtils - get from classLoader + 17:27:50.197 [pool-2-thread-1] INFO com.bigdata.dis.sdk.util.config.ConfigurationUtils - propertyMapFromFile size : 2 + 17:27:51.531 [pool-2-thread-1] DEBUG com.bigdata.dis.sdk.producer.internals.Sender - batches success. dis-alAR-nb, 10 + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.532 [pool-2-thread-1] INFO DISProducerDemo - PutRecordsResultEntry [partitionId=shardId-0000000000, sequenceNumber=76, errorCode=null, errorMessage=null] + 17:27:51.533 [main] INFO DISProducerDemo - ========== PUT OVER ============ + 17:27:51.571 [Sender Thread] DEBUG com.bigdata.dis.sdk.producer.internals.Sender - Beginning shutdown of Kafka producer I/O thread, sending remaining records. diff --git a/umn/source/faqs/how_do_i_check_software_package_integrity.rst b/umn/source/faqs/how_do_i_check_software_package_integrity.rst new file mode 100644 index 0000000..7aa0f56 --- /dev/null +++ b/umn/source/faqs/how_do_i_check_software_package_integrity.rst @@ -0,0 +1,37 @@ +:original_name: dis_faq_0009.html + +.. _dis_faq_0009: + +How Do I Check Software Package Integrity? +========================================== + +This section describes how to verify integrity of the DIS SDK software package on a Linux system by using a verification file. + +Prerequisites +------------- + +- The PuTTY tool is available. +- The WinSCP tool is available. + +Procedure +--------- + +#. Upload the DIS SDK software package **dis-sdk-1.2.3.zip** to any directory on the Linux system by using WinSCP. + +#. .. _dis_faq_0009__l8c35f745b7e945219234467020564558: + + Log in to the Linux system by using PuTTY. In the directory in which **dis-sdk-1.2.3.zip** is stored, run the following command to obtain the verification code of the DIS SDK software package: + + **sha256sum dis-sdk-1.2.3.zip** + + Example verification code: + + .. code-block:: + + # sha256sum dis-sdk-1.2.3.zip + 8be2c937e8d78b1a9b99777cee4e7131f8bf231de3f839cf214e7c5b5ba3c088 dis-sdk-1.2.3.zip + +#. Open the DIS SDK verification file **dis-sdk-1.2.3.zip.sha256sum** and compare it with the verification code obtained in :ref:`2 `. + + - If they are consistent, it indicates that the DIS SDK software package is not tampered with. + - If they are inconsistent, the DIS SDK software package is tampered with and you need to obtain an untampered package. diff --git a/umn/source/faqs/how_does_dis_send_and_receive_data.rst b/umn/source/faqs/how_does_dis_send_and_receive_data.rst new file mode 100644 index 0000000..9a7fc37 --- /dev/null +++ b/umn/source/faqs/how_does_dis_send_and_receive_data.rst @@ -0,0 +1,13 @@ +:original_name: dis_faq_0003.html + +.. _dis_faq_0003: + +How Does DIS Send and Receive Data? +=================================== + +#. Create a DIS stream, and obtain your Access Key ID/Secret Access Key (AK/SK) from the Identity and Access Management (IAM) service. +#. Download the package from https://dis.obs.eu-de.otc.t-systems.com/dis/download/dis-sdk-1.2.3.zip. +#. Create a project and configure the user AK/SK, endpoint, project ID, region, stream name, and partition quantity. +#. After the configuration is completed, run the application to send data. +#. Create a project and configure the user AK/SK, endpoint, project, region, stream name, partition ID, and startingSequenceNumber. +#. After the configuration is completed, run the application to receive data. diff --git a/umn/source/faqs/index.rst b/umn/source/faqs/index.rst new file mode 100644 index 0000000..8560f46 --- /dev/null +++ b/umn/source/faqs/index.rst @@ -0,0 +1,18 @@ +:original_name: dis_faq_0000.html + +.. _dis_faq_0000: + +FAQs +==== + +- :ref:`How Does DIS Send and Receive Data? ` +- :ref:`What Is the Difference Between Storing Data into DIS and Dumping Data Elsewhere? ` +- :ref:`How Do I Check Software Package Integrity? ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + how_does_dis_send_and_receive_data + what_is_the_difference_between_storing_data_into_dis_and_dumping_data_elsewhere + how_do_i_check_software_package_integrity diff --git a/umn/source/faqs/what_is_the_difference_between_storing_data_into_dis_and_dumping_data_elsewhere.rst b/umn/source/faqs/what_is_the_difference_between_storing_data_into_dis_and_dumping_data_elsewhere.rst new file mode 100644 index 0000000..f1add31 --- /dev/null +++ b/umn/source/faqs/what_is_the_difference_between_storing_data_into_dis_and_dumping_data_elsewhere.rst @@ -0,0 +1,27 @@ +:original_name: dis_faq_0007.html + +.. _dis_faq_0007: + +What Is the Difference Between Storing Data into DIS and Dumping Data Elsewhere? +================================================================================ + +After DIS is enabled, data is stored to DIS by default. After a dump task is added, the data can be dumped to other resources. :ref:`Table 1 ` describes the specific differences. + +- Data is stored to DIS by default. +- If **Dump Destination** is set to **OBS**, data is stored in DIS and periodically imported to Object Storage Service (OBS). + +.. _dis_faq_0007__tdc4c4ec0b2454cfeb8421d92412355e6: + +.. table:: **Table 1** Difference between storing data into DIS and dumping data elsewhere + + +---------------------------------------------------------------------+-----------------------------------------------------------------------------+ + | DIS Storage | OBS Storage | + +=====================================================================+=============================================================================+ + | You can store data into DIS without applying for storage resources. | You must apply for OBS resources before dumping data to OBS. | + +---------------------------------------------------------------------+-----------------------------------------------------------------------------+ + | No additional payment is required. | Additional cost for the use of OBS. For details, see the OBS price details. | + +---------------------------------------------------------------------+-----------------------------------------------------------------------------+ + | Data is temporarily stored in DIS for up to 168 hours. | Data is stored in OBS until your OBS bucket expires. | + +---------------------------------------------------------------------+-----------------------------------------------------------------------------+ + | Data is stored only in DIS. | Data is stored in DIS and periodically dumped to OBS. | + +---------------------------------------------------------------------+-----------------------------------------------------------------------------+ diff --git a/umn/source/getting_started/creating_an_iam_agency.rst b/umn/source/getting_started/creating_an_iam_agency.rst new file mode 100644 index 0000000..5c7f279 --- /dev/null +++ b/umn/source/getting_started/creating_an_iam_agency.rst @@ -0,0 +1,63 @@ +:original_name: dis_01_0605.html + +.. _dis_01_0605: + +Creating an IAM Agency +====================== + +Introduction +------------ + +If you choose to dump data from DIS to OBS, create an IAM agency that grants DIS permissions to access OBS. + + +Creating an IAM Agency +---------------------- + +#. Log in to the management console. + +#. Click **Service List**. Under **Management & Deployment**, select **Identify and Access Management**. + +#. Select **Agencies** in the navigation tree pane, and click **Create Agency**. + + + .. figure:: /_static/images/en-us_image_0000001266897473.jpg + :alt: **Figure 1** Creating an IAM agency + + **Figure 1** Creating an IAM agency + +#. Configure agency parameters and click **OK**. + + + .. figure:: /_static/images/en-us_image_0000001266777501.jpg + :alt: **Figure 2** Configuring agency parameters + + **Figure 2** Configuring agency parameters + + .. table:: **Table 1** Agency parameters + + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Parameter | Description | + +===================================+============================================================================================================================================================+ + | Agency Name | Name of the agency to be created. The value of this parameter is 1 to 64 characters long and cannot be left unspecified. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Agency Type | Type of the agency to be created. This parameter must be set to **Cloud service**. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Cloud Service | Click **Select** next to **Cloud Service**. In the **Select Cloud Service** dialog box, select **DIS** and click **OK**. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Validity Period | Select **Unlimited**. | + | | | + | | .. note:: | + | | | + | | Currently, this parameter must be set to **Unlimited**. Using another value may result in authorization failures. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Description | Agency description. The entered description cannot exceed 255 characters. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Permissions | If **Dump Destination** is **OBS**, policy settings are as follows: Region: Global service Project: OBS Policy: Tenant Administrator | + | | | + | | To modify agency policies, click **Modify** in the **Operation** column. In the **Available Policies** area, select your required policy and click **OK**. | + | | | + | | .. note:: | + | | | + | | After an agency is created, its policies cannot be modified. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/umn/source/getting_started/general_procedure.rst b/umn/source/getting_started/general_procedure.rst new file mode 100644 index 0000000..e1286d9 --- /dev/null +++ b/umn/source/getting_started/general_procedure.rst @@ -0,0 +1,24 @@ +:original_name: dis_01_0009.html + +.. _dis_01_0009: + +General Procedure +================= + +The following is the general procedure for using DIS: + +:ref:`Step 1: Creating a DIS Stream ` + +You need to create a stream before using DIS. + +:ref:`Step 2: Preparing a DIS Application Development Environment ` + +Before developing a DIS application, install an application development tool, and import your SDK package and sample project into the development environment. + +:ref:`Step 3: Sending Data to DIS ` + +Write a producer application and run it to send data to the cloud. The DIS stream information can be viewed on the DIS console. + +:ref:`Step 4: Obtaining Data from DIS ` + +Write a consumer application and run it to retrieve data from the cloud. diff --git a/umn/source/getting_started/index.rst b/umn/source/getting_started/index.rst new file mode 100644 index 0000000..22bfa1c --- /dev/null +++ b/umn/source/getting_started/index.rst @@ -0,0 +1,28 @@ +:original_name: dis_01_0008.html + +.. _dis_01_0008: + +Getting Started +=============== + +- :ref:`General Procedure ` +- :ref:`Step 1: Creating a DIS Stream ` +- :ref:`Step 2: Preparing a DIS Application Development Environment ` +- :ref:`Step 3: Sending Data to DIS ` +- :ref:`Step 4: Obtaining Data from DIS ` +- :ref:`Obtaining Authentication Information ` +- :ref:`Interconnecting with OBS ` +- :ref:`Creating an IAM Agency ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + general_procedure + step_1_creating_a_dis_stream + step_2_preparing_a_dis_application_development_environment + step_3_sending_data_to_dis + step_4_obtaining_data_from_dis + obtaining_authentication_information + interconnecting_with_obs + creating_an_iam_agency diff --git a/umn/source/getting_started/interconnecting_with_obs.rst b/umn/source/getting_started/interconnecting_with_obs.rst new file mode 100644 index 0000000..3d4a620 --- /dev/null +++ b/umn/source/getting_started/interconnecting_with_obs.rst @@ -0,0 +1,21 @@ +:original_name: dis_01_0045.html + +.. _dis_01_0045: + +Interconnecting with OBS +======================== + +Introduction +------------ + +DIS can upload data to Object Storage Service (OBS). + +Prerequisites +------------- + +An IAM agency has been created by following the procedure in :ref:`Creating an IAM Agency `. This IAM agency entrusts DIS to access your OBS resources. + +Data Dumping +------------ + +In section :ref:`Creating a Dump Task `, you can select a location to which data from DIS streams will be dumped. If Dump Destination is set to **OBS**, DIS periodically imports data from DIS streams to OBS. diff --git a/umn/source/getting_started/obtaining_authentication_information.rst b/umn/source/getting_started/obtaining_authentication_information.rst new file mode 100644 index 0000000..c24441f --- /dev/null +++ b/umn/source/getting_started/obtaining_authentication_information.rst @@ -0,0 +1,21 @@ +:original_name: dis_01_0043.html + +.. _dis_01_0043: + +Obtaining Authentication Information +==================================== + +Obtaining AK/SK +--------------- + +Access Key ID/Secret Access Key (AK/SK) is created on Identity and Access Management (IAM) to authenticate calls to application programming interfaces (APIs) on the public cloud. To obtain AK/SK, choose **My Credential** > **Access Keys**. + +Obtaining Project ID +-------------------- + +A project is a group of tenant resources. To view the project IDs of different regions, choose **My Credential** > **Projects**. + +Obtaining Region Information and Endpoint Information +----------------------------------------------------- + +For details about regions and endpoints, see `Regions and Endpoints `__. diff --git a/umn/source/getting_started/step_1_creating_a_dis_stream.rst b/umn/source/getting_started/step_1_creating_a_dis_stream.rst new file mode 100644 index 0000000..23761c9 --- /dev/null +++ b/umn/source/getting_started/step_1_creating_a_dis_stream.rst @@ -0,0 +1,86 @@ +:original_name: dis_01_0601.html + +.. _dis_01_0601: + +Step 1: Creating a DIS Stream +============================= + +You can create a DIS stream on the DIS management console. + +Procedure +--------- + +#. Use the account to log in to the DIS console. + + + .. figure:: /_static/images/en-us_image_0000001266897449.jpg + :alt: **Figure 1** DIS management console + + **Figure 1** DIS management console + +#. Click |image1| in the upper left corner of the page and select a region and project. + +#. .. _dis_01_0601__li23032735111458: + + Click Create Stream and set related parameters. + + .. table:: **Table 1** Stream parameters + + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Parameter | Description | Example | + +========================+===================================================================================================================================================================================================================================================================================================+=======================+ + | Region | Physical location of the cloud service. You can select a different region from the drop-down list. | ``-`` | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | **Basic Information** | | | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Stream Name | Name of the DIS stream to be created. A stream name is 1 to 64 characters long. Only letters, digits, hyphens (-), and underscores (_) are allowed. | dis-Tido | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Stream Type | - **Common**: Each partition supports a maximum read speed of 2 MB/s and a maximum write speed of 1 MB/s. | ``-`` | + | | - **Advanced**: Each partition supports a maximum read speed of 10 MB/s and a maximum write speed of 5 MB/s. | | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Partitions | Partitions are the base throughput unit of a DIS stream. | 5 | + | | | | + | | - For a common DIS stream, the value of **Partitions** is an integer from 1 to 50. A tenant can create a maximum of 50 partitions. | | + | | - For an advanced DIS stream, the value of **Partitions** is an integer from 1 to 10. A tenant can create a maximum of 10 partitions. | | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Partition Calculator | Calculator used to calculate the estimated number of partitions based on the information you entered. | ``-`` | + | | | | + | | a. Click **Partition Calculator**. | | + | | b. In the **Partition Calculator** dialog box, configure the **Average Record Size (KB)**, **Max. Records Written**, and **Consumer Applications** parameters. The **Estimated Partitions** field then displays the recommended number of partitions. The value of this field cannot be modified. | | + | | | | + | | .. note:: | | + | | | | + | | Partition calculation formulas: | | + | | | | + | | - Based on the traffic (the final value must be rounded up): | | + | | | | + | | Common stream: Average record size x (1 + 20%) x Maximum records written/ (1 x 1024 KB) (20% is the reserved partition percentage.) | | + | | | | + | | Advanced stream: Average record size x (1 + 20%) x Maximum records written/ (5 x 1024 KB) (20% is the reserved partition percentage.) | | + | | | | + | | - Based on the consumer program quantity (the final value must be rounded up): | | + | | | | + | | (Number of consumer programs/2) x Number of partitions calculated based on the traffic (The result of the number of consumer programs/2 must reserve two decimals.) | | + | | | | + | | The largest value among the values calculated based on the previous three formulas is considered as the estimated partition value. | | + | | | | + | | c. Click **Use Estimated Value**. The estimated value is automatically used as the value of **Partitions**. | | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Data Retention (hours) | The maximum number of hours for which data can be preserved in DIS. Data will be deleted when the retention period expires. | 24 | + | | | | + | | Value range: an integer ranging from 24 to 72. | | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Configure | Click **Configure now**. The **Tag** parameter is displayed. | ``-`` | + | | | | + | | For details about how to add a tag, see :ref:`Managing Stream Tags `. | | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Skip | No advanced settings need to be configured. | ``-`` | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Tag | Identifier of the stream. Adding tags to streams can help you identify and manage your stream resources. | ``-`` | + +------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+ + +#. Click . The **Details** page is displayed. + +#. Click **Submit**. + +.. |image1| image:: /_static/images/en-us_image_0000001222257350.png diff --git a/umn/source/getting_started/step_2_preparing_a_dis_application_development_environment.rst b/umn/source/getting_started/step_2_preparing_a_dis_application_development_environment.rst new file mode 100644 index 0000000..064a963 --- /dev/null +++ b/umn/source/getting_started/step_2_preparing_a_dis_application_development_environment.rst @@ -0,0 +1,123 @@ +:original_name: dis_01_0602.html + +.. _dis_01_0602: + +Step 2: Preparing a DIS Application Development Environment +=========================================================== + +Before developing DIS applications, prepare an application development environment, and then obtain a software development kit (SDK) and sample project and import them to the development environment. + +Prerequisites +------------- + +- JDK 1.8 or later has been installed. +- Eclipse has been installed. + +Procedure +--------- + +#. Configure a JDK using Eclipse. + + a. Start Eclipse and choose **Window** > **Preferences**. The **Preferences** dialog box is displayed. + + b. In the navigation tree, choose **Java**. On the **Java** page, configure general settings for Java development and then click **OK**. + + + .. figure:: /_static/images/en-us_image_0000001266697405.png + :alt: **Figure 1** Preferences + + **Figure 1** Preferences + + c. In the navigation tree, choose **Java** > **Installed JREs**. + + - Ensure that configured JDK environmental variables are displayed on the **Installed JREs** page. Then go to :ref:`1.c.i `. + - To configure different variables for different versions of JDK, perform :ref:`1.c.ii ` to :ref:`1.c.iv `. + + + .. figure:: /_static/images/en-us_image_0000001266777405.png + :alt: **Figure 2** Installed JREs + + **Figure 2** Installed JREs + + #. .. _dis_01_0602__li12377149194529: + + Select the installed JDK and click **OK**. + + #. .. _dis_01_0602__li40268681153416: + + Click **Add**. The **Add JRE** dialog box is displayed. + + + .. figure:: /_static/images/en-us_image_0000001222257286.png + :alt: **Figure 3** JRE Type + + **Figure 3** JRE Type + + #. Select a JRE type and click **Next**. + + + .. figure:: /_static/images/en-us_image_0000001266897385.png + :alt: **Figure 4** JRE Definition + + **Figure 4** JRE Definition + + #. .. _dis_01_0602__li6851708153416: + + Configure the basic information about JDK and click **Finish**. + + - JRE home: JDK installation path. + - Default VM arguments: JDK running parameters. + +#. Download resource packages. + + Download the DIS SDK from https://dis.obs.eu-de.otc.t-systems.com/dis/download/dis-sdk-1.2.3.zip and its integrity check file from https://dis.obs.eu-de.otctest.t-systems.com/dis/download/dis-sdk-1.2.3.zip.sha256sum. + +#. Import the Eclipse project. + + a. Start Eclipse. Choose **File** > **Import**. The **Import** dialog box is displayed. + + b. Choose **General** > **Existing Projects into Workspace** and click **Next**. The **Import** dialog box is displayed. + + c. Click **Browse** and select a save location for the **dis-sdk-demo** sample project. In the **Projects** area, select a sample project. + + + .. figure:: /_static/images/en-us_image_0000001222417242.png + :alt: **Figure 5** Importing a project + + **Figure 5** Importing a project + + d. Click **Finish**. + +#. Configure the demo project. + + a. Set the project code to **UTF-8**. + + #. In the navigation tree, right-click the required project under **Project Explorer** and choose **Properties** from the shortcut menu. The **Properties for dis-sdk-demo** dialog box is displayed. + #. In the navigation tree, choose **Resource**. The **Resource** page is displayed in the right pane. + #. In the **Other** drop-down list, select **UTF-8**. + #. Click **Apply and Close**. + + b. Import a dependency package. + + #. In the navigation pane, choose **Project Explorer**. Right-click the chosen project and choose **Properties** from the shortcut menu. + #. In the navigation tree, choose **Java Build Path**. The **Java Build Path** page is displayed in the right pane. + #. Click the **Libraries** tab, and then click **Add External JARs**. The **JAR Selection** dialog box is displayed. + #. Select the directory where the **dis-sdk** folder is located and click **Open**. + #. On the **Properties for dis-sdk-demo** page, click **Apply and Close** to import all the **.jar** files in the current path and the **third_lib** folder to the project. + + + .. figure:: /_static/images/en-us_image_0000001222097282.png + :alt: **Figure 6** Java Build Path + + **Figure 6** Java Build Path + + c. Add the JDK. + + #. In the navigation pane, choose **Project Explorer**. Right-click the chosen project and choose **Properties** from the shortcut menu. + #. In the navigation tree, choose **Java Build Path**. The **Java Build Path** page is displayed in the right pane. + #. Click the **Libraries** tab, and then click **Add Library**. The **Add Library** dialog box is displayed. + #. Select **JRE System Library** and click **Next**. Verify that the version of **Workspace default JRE** is **jdk1.8** or later. + #. Click **Finish** to exit the **Add Library** dialog box. + #. Click **Apply and Close**. + +#. Initialize a DIS client sample. For details about **endpoint**, **ak**, **sk**, **region**, and **projectId**, see :ref:`Obtaining Authentication Information `. diff --git a/umn/source/getting_started/step_3_sending_data_to_dis.rst b/umn/source/getting_started/step_3_sending_data_to_dis.rst new file mode 100644 index 0000000..ab410a5 --- /dev/null +++ b/umn/source/getting_started/step_3_sending_data_to_dis.rst @@ -0,0 +1,47 @@ +:original_name: dis_01_0603.html + +.. _dis_01_0603: + +Step 3: Sending Data to DIS +=========================== + +Function +-------- + +Local data is continuously uploaded to DIS. + +.. note:: + + Data can be stored in DIS, OBS, and DLI. For details about how to configure a storage location, see :ref:`Creating a Dump Task `. + + The maximum number of days for DIS to preserve data cannot exceed **Data Retention (days)**. + +Sample Code +----------- + +The example code file is the **ProducerDemo.java** file in the **\\dis-sdk-demo\\src\\main\\java\\com\\bigdata\\dis\\sdk\\demo** directory decompressed from the **dis-sdk-1.2.3.zip** package. The compression package is downloaded in :ref:`Step 2: Preparing a DIS Application Development Environment `. + +Running the Producer Program +---------------------------- + +Right-click the producer application and choose **Run As** > **1 Java Application** from the shortcut menu. + + +.. figure:: /_static/images/en-us_image_0000001266977421.png + :alt: **Figure 1** Running a producer application + + **Figure 1** Running a producer application + +While data is being sent to DIS, the DIS console displays DIS stream information. If information similar to the following is displayed, the data has been successfully sent to DIS: + +.. code-block:: + + 14:40:20.090 [main] INFOcom.bigdata.dis.sdk.DISConfig - get from classLoader + 14:40:20.093 [main] INFODEMOT - ========== BEGIN PUT ============ + 14:40:21.186 [main] INFOcom.bigdata.dis.sdk.util.config.ConfigurationUtils - get from classLoader + 14:40:21.187 [main] INFOcom.bigdata.dis.sdk.util.config.ConfigurationUtils - propertyMapFromFile size : 2 + 14:40:22.092 [main] INFOcom.bigdata.dis.sdk.demo.ProducerDemo - Put 3 records[3 successful / 0 failed]. + 14:40:22.092 [main] INFOcom.bigdata.dis.sdk.demo.ProducerDemo - [hello world.] put success, partitionId [shardId-0000000000], partitionKey [964885], sequenceNumber [0] + 14:40:22.092 [main] INFOcom.bigdata.dis.sdk.demo.ProducerDemo - [hello world.] put success, partitionId [shardId-0000000000], partitionKey [910960], sequenceNumber [1] + 14:40:22.092 [main] INFOcom.bigdata.dis.sdk.demo.ProducerDemo - [hello world.] put success, partitionId [shardId-0000000000], partitionKey [528377], sequenceNumber [2] + 14:40:22.092 [main] INFOcom.bigdata.dis.sdk.demo.ProducerDemo - ========== PUT OVER ============ diff --git a/umn/source/getting_started/step_4_obtaining_data_from_dis.rst b/umn/source/getting_started/step_4_obtaining_data_from_dis.rst new file mode 100644 index 0000000..8c5dbdc --- /dev/null +++ b/umn/source/getting_started/step_4_obtaining_data_from_dis.rst @@ -0,0 +1,31 @@ +:original_name: dis_01_0604.html + +.. _dis_01_0604: + +Step 4: Obtaining Data from DIS +=============================== + +Function +-------- + +You can retrieve data from DIS when needed. + +Sample Code +----------- + +The example code file is the **ConsumerDemo.java** file in the **\\dis-sdk-demo\\src\\main\\java\\com\\bigdata\\dis\\sdk\\demo** directory decompressed from the **dis-sdk-XXX.zip** package. The compression package is downloaded in :ref:`Step 2: Preparing a DIS Application Development Environment `. + +Running the Consumer Application +-------------------------------- + +If information similar to the following appears, data has been successfully retrieved from DIS: + +.. code-block:: + + 14:55:42.954 [main] INFOcom.bigdata.dis.sdk.DISConfig - get from classLoader + 14:55:44.103 [main] INFOcom.bigdata.dis.sdk.util.config.ConfigurationUtils - get from classLoader + 14:55:44.105 [main] INFOcom.bigdata.dis.sdk.util.config.ConfigurationUtils - propertyMapFromFile size : 2 + 14:55:45.235 [main] INFOcom.bigdata.dis.sdk.demo.ConsumerDemo - Get stream streamName[partitionId=0] cursor success : eyJnZXRJdGVyYXRvclBhcmFtIjp7InN0cmVhbS1uYW1lIjoiZGlzLTEzbW9uZXkiLCJwYXJ0aXRpb24taWQiOiIwIiwiY3Vyc29yLXR5cGUiOiJBVF9TRVFVRU5DRV9OVU1CRVIiLCJzdGFydGluZy1zZXF1ZW5jZS1udW1iZXIiOiIxMDY4OTcyIn0sImdlbmVyYXRlVGltZXN0YW1wIjoxNTEzNjY2NjMxMTYxfQ + 14:55:45.305 [main] INFOcom.bigdata.dis.sdk.demo.ConsumerDemo - Get Record [hello world.], partitionKey [964885], sequenceNumber [0]. + 14:55:45.305 [main] INFOcom.bigdata.dis.sdk.demo.ConsumerDemo - Get Record [hello world.], partitionKey [910960], sequenceNumber [1]. + 14:55:46.359 [main] INFOcom.bigdata.dis.sdk.demo.ConsumerDemo - Get Record [hello world.], partitionKey [528377], sequenceNumber [2]. diff --git a/umn/source/glossary.rst b/umn/source/glossary.rst new file mode 100644 index 0000000..7e3485d --- /dev/null +++ b/umn/source/glossary.rst @@ -0,0 +1,8 @@ +:original_name: dis_01_0000.html + +.. _dis_01_0000: + +Glossary +======== + +For details, see `Glossary `__. diff --git a/umn/source/index.rst b/umn/source/index.rst index beec702..589da33 100644 --- a/umn/source/index.rst +++ b/umn/source/index.rst @@ -2,3 +2,16 @@ Data Ingestion Service - User Guide =================================== +.. toctree:: + :maxdepth: 1 + + overview/index + getting_started/index + managing_dis_streams/index + managing_a_dump_task/index + monitoring/index + cdr_specification/index + dis_sdk_usage_guide/index + faqs/index + change_history + glossary diff --git a/umn/source/managing_a_dump_task/creating_a_dump_task.rst b/umn/source/managing_a_dump_task/creating_a_dump_task.rst new file mode 100644 index 0000000..2b0cb5a --- /dev/null +++ b/umn/source/managing_a_dump_task/creating_a_dump_task.rst @@ -0,0 +1,157 @@ +:original_name: dis_01_0047.html + +.. _dis_01_0047: + +Creating a Dump Task +==================== + +If a dump task is created for a DIS stream, data sent to the DIS stream can be automatically dumped to the selected target specified in the dump task. + +#. Use the account to log in to the DIS console. + +#. Click |image1| in the upper left corner of the page and select a region and project. + +#. In the navigation tree on the left, choose Stream Management. + +#. Click the name of a stream that you want to view. On the displayed page, click the **Dump Management** tab. + +#. Click **Add Dump Task**. On the **Add Dump Task** page, configure dump parameters. Dump task parameters are described in :ref:`Table 1 `. + + .. note:: + + A maximum of five dump tasks can be created for each stream. + +#. Click **Create Now**. + +#. In the **Operation** column of the corresponding **Task Name**, click **More** > **View Dump Log** to view the dump task details of the stream. :ref:`Table 2 ` describes the dump log parameters. + + .. _dis_01_0047__table1544912219359: + + .. table:: **Table 1** Dump task parameters + + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + | Parameter | Description | Remarks | + +======================================+=========================================================================================================================================================================================================================================+================================================================================+ + | **Data Dumping** | | | + | | | | + | Location to save data in the stream. | | | + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + | Dump Destination | **OBS**: After the streaming data is stored to DIS, it is then periodically imported to OBS. | ``-`` | + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + | Task Name | Name of the dump task. The names of dump tasks created for the same stream must be unique. A task name must be unique in the stream and is 1 to 64 characters long. Only letters, digits, hyphens (-), and underscores (_) are allowed. | ``-`` | + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + | Dump File Format | Text | The **Text** format can be selected only when **Source Data Type** is **OBS**. | + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + | Dump Bucket | Name of the OBS bucket used to store data from the DIS stream. The bucket name is created when you create a bucket in OBS. | ``-`` | + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + | File Directory | Directory created in OBS to store files from the DIS stream. Different directory levels are separated by a forward slash (/). The value cannot start with a forward slash. | ``-`` | + | | | | + | | This directory name is 0 to 50 characters long. | | + | | | | + | | By default, this parameter is left unspecified. | | + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + | Time Directory Format | Directory format based on the time. Data will be saved to the directory in the format of time layer under the dump file directory in the OBS bucket. | ``-`` | + | | | | + | | For example, if the time directory is accurate to day, the data save path is "bucket name/dump file directory/year/month/day". | | + | | | | + | | Possible values are as follows: | | + | | | | + | | - N/A: If this field is left blank, the time directory format will not be used. | | + | | - yyyy: year. | | + | | - yyyy/MM: year and month. | | + | | - yyyy/MM/dd: year, month, and day. | | + | | - yyyy/MM/dd/HH: year, month, day, and hour. | | + | | - yyyy/MM/dd/HH/mm: year, month, day, hour, and minute. | | + | | | | + | | You can only select but not enter a value in this field. | | + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + | Record Delimiter | Delimiter used to separate different dump records. | ``-`` | + | | | | + | | Possible values are as follows: | | + | | | | + | | - Comma (,) | | + | | - Semicolon (;) | | + | | - Vertical bar (|) | | + | | - Newline (\\n) | | + | | - NULL | | + | | | | + | | You can only select but not enter a value in this field. | | + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + | Dump Interval (s) | User-defined interval at which data is imported from the current DIS stream into OBS. If no data is pushed to the DIS stream during the current interval, no dump file package will be generated. | ``-`` | + | | | | + | | Value range: 30s to 900s | | + | | | | + | | Unit: second | | + | | | | + | | Default value: 300s | | + +--------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ + + .. _dis_01_0047__table104664221353: + + .. table:: **Table 2** Dump log parameters + + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Parameter | Description | + +===================================+================================================================================================================================================================================================+ + | Start Time | Time when the dump log is created. | + | | | + | | Format: YYYY/MM/dd HH:mm:ss GTM | + | | | + | | - YYYY: year. | + | | - MM: month. | + | | - dd: date. | + | | - HH: hour. | + | | - mm: minute. | + | | - ss: second. | + | | - GMT: time zone. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | End Time | Time when you finish creating the dump log. | + | | | + | | Format: YYYY/MM/dd HH:mm:ss GTM | + | | | + | | - YYYY: year. | + | | - MM: month. | + | | - dd: date. | + | | - HH: hour. | + | | - mm: minute. | + | | - ss: second. | + | | - GMT: time zone. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Status | Dump status. | + | | | + | | - Succeeded | + | | - Failed | + | | - Abnormal | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Dump File Name | Name of the file that is dumped to the target service. The user records read from the stream are written into the file and then dumped to the target service (such as OBS) in the file format. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Records | Number of the records uploaded between the time when you start to create a dump log to the time when you finish creating it. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Data Amount (bytes) | Amount of the data uploaded between the time when you start to create the dump log to the time when you finish creating it. | + | | | + | | Unit: byte | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Operation | Dump failure details. | + | | | + | | - If **Status** is **Succeeded**, the column is not operable. | + | | - If **Status** is **Failed**, click **View Details** to view dump details. | + | | - If **Status** is **Abnormal**, click **View Details** to view dump details. | + +-----------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Modifying and Enabling Dump Tasks +--------------------------------- + +After creating a stream and adding a dump task successfully, you can modify the attributes of the created stream. + +#. Log in to the DIS console. +#. Click |image2| in the upper left corner of the page and select a region and project. +#. In the navigation tree, choose **Stream Management**. +#. Click the name of a stream that you want to view. On the displayed page, click the **Dump Management** tab. Alternatively, in the **Operation** column of a stream that you want to view, click **More** and choose **View Dump Task** from the drop-down list. +#. In the **Operation** column of the stream for which a dump task has been added, perform the following operations: + + a. Choose **More > Modify** to modify the dump task. + b. Choose **More > Start** to start the dump task. + c. Choose **More > Pause** to pause the dump task. + +.. |image1| image:: /_static/images/en-us_image_0000001222577290.png +.. |image2| image:: /_static/images/en-us_image_0000001222577290.png diff --git a/umn/source/managing_a_dump_task/index.rst b/umn/source/managing_a_dump_task/index.rst new file mode 100644 index 0000000..174411b --- /dev/null +++ b/umn/source/managing_a_dump_task/index.rst @@ -0,0 +1,14 @@ +:original_name: dis_01_0115.html + +.. _dis_01_0115: + +Managing a Dump Task +==================== + +- :ref:`Creating a Dump Task ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + creating_a_dump_task diff --git a/umn/source/managing_dis_streams/deleting_a_stream.rst b/umn/source/managing_dis_streams/deleting_a_stream.rst new file mode 100644 index 0000000..1175f37 --- /dev/null +++ b/umn/source/managing_dis_streams/deleting_a_stream.rst @@ -0,0 +1,34 @@ +:original_name: dis_01_0013.html + +.. _dis_01_0013: + +Deleting a Stream +================= + +.. note:: + + A deleted stream will not be charged and cannot be restored. Exercise caution when performing this operation. + +#. Use the account to log in to the DIS console. + +#. Click |image1| in the upper left corner of the page and select a region and project. + +#. In the navigation tree on the left, choose **Stream Management**. + + + .. figure:: /_static/images/en-us_image_0000001222417238.jpg + :alt: **Figure 1** Managing DIS streams + + **Figure 1** Managing DIS streams + +#. Click **Delete** in the **Operation** column of the stream that you want to delete. The **Delete Stream** dialog box is displayed. + + + .. figure:: /_static/images/en-us_image_0000001222257278.jpg + :alt: **Figure 2** Deleting a stream + + **Figure 2** Deleting a stream + +#. Click **OK** to delete the selected stream. + +.. |image1| image:: /_static/images/en-us_image_0000001266777397.png diff --git a/umn/source/managing_dis_streams/index.rst b/umn/source/managing_dis_streams/index.rst new file mode 100644 index 0000000..eb72bc8 --- /dev/null +++ b/umn/source/managing_dis_streams/index.rst @@ -0,0 +1,20 @@ +:original_name: dis_01_0010.html + +.. _dis_01_0010: + +Managing DIS Streams +==================== + +- :ref:`Listing DIS Streams ` +- :ref:`Viewing Stream Monitoring Metrics ` +- :ref:`Managing Stream Tags ` +- :ref:`Deleting a Stream ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + listing_dis_streams + viewing_stream_monitoring_metrics + managing_stream_tags + deleting_a_stream diff --git a/umn/source/managing_dis_streams/listing_dis_streams.rst b/umn/source/managing_dis_streams/listing_dis_streams.rst new file mode 100644 index 0000000..d56c33e --- /dev/null +++ b/umn/source/managing_dis_streams/listing_dis_streams.rst @@ -0,0 +1,26 @@ +:original_name: dis_01_0011.html + +.. _dis_01_0011: + +Listing DIS Streams +=================== + +The **Stream Management** page displays all DIS streams created. After clicking a stream, you can view the following information about this stream: + + +.. figure:: /_static/images/en-us_image_0000001222097286.jpg + :alt: **Figure 1** Stream Management + + **Figure 1** Stream Management + +- **Name/ID**: Unique name of the DIS stream to be created. A stream name is 1 to 64 characters long. Only letters, digits, hyphens (-), and underscores (_) are allowed. +- **Status**: Stream status. +- **Stream Type**: **Common** and **Advanced**. + + - **Common**: Each partition supports a maximum read speed of 2 MB/s and a maximum write speed of 1 MB/s or 1,000 records/s. + - **Advanced**: Each partition supports a maximum read speed of 10 MB/s and a maximum write speed of 5 MB/s or 2,000 records/s. + +- **Partitions**: The number of partitions into which data records in the newly created DIS stream will be distributed. Multiple partitions of a stream can concurrently transmit data to improve efficiency. +- **Data Retention (hours)**: The maximum number of hours for DIS to preserve data. Data will be deleted when the retention period expires. Value range: an integer ranging from 24 to 72. Unit: hour +- **Created**: Time at which the DIS stream is created. The creation time is in the *yyyy/MM/dd HH:mm:ss* **GMT** format. For example, 2017/05/09 08:00:00 GMT+08:00. +- **Operation**: Supported operations include **Delete**, and **View Dump Task**. diff --git a/umn/source/managing_dis_streams/managing_stream_tags.rst b/umn/source/managing_dis_streams/managing_stream_tags.rst new file mode 100644 index 0000000..6721b38 --- /dev/null +++ b/umn/source/managing_dis_streams/managing_stream_tags.rst @@ -0,0 +1,105 @@ +:original_name: dis_01_0050.html + +.. _dis_01_0050: + +Managing Stream Tags +==================== + +A tag is an identifier of a stream. Adding tags to streams can help you identify and manage your stream resources. + +You can add a maximum of 10 tags to a stream when creating the stream or add them on the details page of the created stream. + +A tag consists of a tag key and a tag value. :ref:`Table 1 ` describes the rules for naming the tag key and value. + +.. _dis_01_0050__en-us_topic_0110219762_table16316649132010: + +.. table:: **Table 1** Naming rules for a tag key and value + + +-----------------------+----------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Parameter | Rule | Example | + +=======================+============================================================================================================================+=======================+ + | Key | A tag key cannot be left blank. | Organization | + | | | | + | | A tag key must be unique for a stream. | | + | | | | + | | A tag key contains a maximum of 36 characters. | | + | | | | + | | A tag value cannot contain special characters ``=*<>\,|/`` or start or end with a space. | | + +-----------------------+----------------------------------------------------------------------------------------------------------------------------+-----------------------+ + | Tag value | A tag value contains a maximum of 43 characters. | Apache | + | | | | + | | A tag value cannot contain special characters ``=*<>\,|/`` or start or end with a space. This parameter can be left blank. | | + +-----------------------+----------------------------------------------------------------------------------------------------------------------------+-----------------------+ + +Adding a Tag to a Stream +------------------------ + +You can add a tag to a stream on the **Buy Stream** page. + +#. Log in to the management console. + +#. Choose **Enterprise Intelligence** > **Data Ingestion Service**. + +#. On the DIS management console, click **Buy Stream**. + +#. On the **Advanced Settings** tab page, select **Configure**. + + Enter the key and value of a tag to be added. + + You can add a maximum of 10 tags to the stream and use intersections of tags to search for the target stream. + + .. note:: + + You can also add tags to existing streams. For details, see :ref:`Managing Tags `. + +Searching for a Target Stream +----------------------------- + +You can search for a target stream by tag on the **Stream Management** page. + +#. Log in to the management console. + +#. Choose **EI Enterprise Intelligent** > **Data Ingestion Service**. + +#. In the navigation tree, choose **Ingestion Management** > **Stream Management**. In the upper right corner of the page, click **Search by Tag**. + +#. Enter the tag key and value of the stream you are searching for. + + You can select a tag key or tag value from its drop-down list. When the tag key or tag value is exactly matched, the system can automatically locate the target stream. If you enter multiple tags, their intersections are used to search for the stream. + +#. Click **Search**. + + The system searches for the target stream by tag key or value. + +.. _dis_01_0050__section188067265123: + +Managing Tags +------------- + +You can add, delete, modify, and view tags on the **Tags** tab page of a stream. + +#. Log in to the management console. + +#. Choose **EI Enterprise Intelligent** > **Data Ingestion Service**. + +#. In the navigation tree, choose **Ingestion Management** > **Stream Management**. Click a stream to which the tags to be managed belong to. + + The stream details page is displayed. + +#. Click the **Tags** tab and add, deleted, modify, and view tags. + + - View + + On the **Tags** tab page, you can view details about tags of the stream, including the number of tags and the key and value of each tag. + + - Add + + Click **Add Tag** in the upper left corner. In the displayed **Add Tag** dialog box, enter the key and value of the tag to be added, and click **OK**. + + - Modify + + In the **Operation** column of a tag, click **Edit**. In the displayed **Edit Tag** page, enter a new tag key and value and click **OK**. + + - Delete + + In the **Operation** column of the tag, click **Delete**. After confirmation, click **OK** on the displayed **Delete Tag** page. diff --git a/umn/source/managing_dis_streams/viewing_stream_monitoring_metrics.rst b/umn/source/managing_dis_streams/viewing_stream_monitoring_metrics.rst new file mode 100644 index 0000000..19fb45e --- /dev/null +++ b/umn/source/managing_dis_streams/viewing_stream_monitoring_metrics.rst @@ -0,0 +1,62 @@ +:original_name: dis_01_0012.html + +.. _dis_01_0012: + +Viewing Stream Monitoring Metrics +================================= + +You can view stream monitoring information on the console and monitor the data consumed by applications in the stream. + +#. Use the account to log in to the DIS console. + +#. Click |image1| in the upper left corner and select a region and a project. + +#. In the navigation tree on the left, choose **Stream Management**. + +#. In the stream list, click the name of the DIS stream whose monitoring metrics will be viewed. The monitoring page is displayed. + + + .. figure:: /_static/images/en-us_image_0000001266897429.jpg + :alt: **Figure 1** Monitoring page + + **Figure 1** Monitoring page + +#. On the **Monitoring** page, click the **Streams** or **Partitions** tab to view stream or partition monitoring metrics. :ref:`Table 1 ` describes the monitoring parameters. For details about basic stream information, see :ref:`3 `. + + .. _dis_01_0012__table2942144318834: + + .. table:: **Table 1** DIS monitoring information + + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | Parameter | Description | + +======================================+===========================================================================================================================+ + | Time Range | - Monitoring time range. | + | | | + | | Values: | + | | | + | | - 1h | + | | - 2h | + | | - 3h | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | **Partitions** | | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | Partition ID | ID of the partition. It starts from 0 by default. Select any of the following values from the **Partition ID** drop-down. | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | Data Rate (KB/s) | Rates at which data is sent to and retrieved from the chosen partition within the specified time range. Unit: KB/s | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | Records Per Second | The number of records sent to and retrieved from the chosen partition within the specified time range. | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | **Streams** | | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | Data Rate (KB/s) | Rates at which data is sent to and retrieved from the chosen DIS stream within the specified time range. Unit: KB/s | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | Records Per Second | The number of records sent to and retrieved from the chosen DIS stream within the specified time range. | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | Successful Requests Per Second | The number of PutRecords and GetRecords requests successfully fulfilled within the specified time range. | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | Throttled Requests Per Second | The number of PutRecords and GetRecords requests rejected within the specified time range due to flow control. | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + | Average Request Processing Time (ms) | The average amount of time spent in processing a PutRecords or GetRecords request. | + +--------------------------------------+---------------------------------------------------------------------------------------------------------------------------+ + +.. |image1| image:: /_static/images/en-us_image_0000001266777397.png diff --git a/umn/source/monitoring/index.rst b/umn/source/monitoring/index.rst new file mode 100644 index 0000000..4c8ac80 --- /dev/null +++ b/umn/source/monitoring/index.rst @@ -0,0 +1,18 @@ +:original_name: dis_01_0130.html + +.. _dis_01_0130: + +Monitoring +========== + +- :ref:`Supported Metrics ` +- :ref:`Setting Alarm Rules ` +- :ref:`Querying Metrics ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + supported_metrics + setting_alarm_rules + querying_metrics diff --git a/umn/source/monitoring/querying_metrics.rst b/umn/source/monitoring/querying_metrics.rst new file mode 100644 index 0000000..8ef33fb --- /dev/null +++ b/umn/source/monitoring/querying_metrics.rst @@ -0,0 +1,52 @@ +:original_name: dis_01_0134.html + +.. _dis_01_0134: + +Querying Metrics +================ + +Scenario +-------- + +Cloud Eye monitors the running status of the DIS stream. You can obtain the monitoring metrics of CDM on the Cloud Eye management console. + +Monitored data requires a period of time for transmission and display. The status of CDM displayed on the Cloud Eye page is the status obtained 5 to 10 minutes before. You can view the monitored data of a newly created DIS stream 5 to 10 minutes later. + +Prerequisites +------------- + +- The DIS stream is running properly. + + .. note:: + + Cloud Eye will delete a deleted stream from the monitoring list and will not monitor it any more. However, you need to manually clear its alarm rules. + +- Alarm rules have been configured on the Cloud Eye page. For details, see :ref:`Setting Alarm Rules `. + +Procedure +--------- + +#. Use the account to log in to the DIS console. + +#. Click |image1| in the upper left corner of the page and select a region and project. + +#. In the navigation tree on the left, choose **Stream Management**. + +#. In the stream list, click the name of the DIS stream whose monitoring metrics will be viewed. The monitoring page is displayed. + +#. On the **Streams** tab page, click **View details** to switch to the Cloud Eye console. + +#. On the DIS monitoring page, you can view the graphs of all monitoring metrics. + + + .. figure:: /_static/images/en-us_image_0000001222417338.png + :alt: **Figure 1** Viewing DIS monitoring metrics + + **Figure 1** Viewing DIS monitoring metrics + +#. Click |image2| in the upper right corner of the graphs to zoom out the graphs. + + You can view the raw monitoring data curves of different metrics in the last 1 hour, 3 hours, and 12 hours. You can determine whether to enable **Auto Refresh**. Cloud Eye provides an automatic refresh interval of 60 seconds. + +.. |image1| image:: /_static/images/en-us_image_0000001222577290.png +.. |image2| image:: /_static/images/en-us_image_0000001222097374.png diff --git a/umn/source/monitoring/setting_alarm_rules.rst b/umn/source/monitoring/setting_alarm_rules.rst new file mode 100644 index 0000000..ecb708e --- /dev/null +++ b/umn/source/monitoring/setting_alarm_rules.rst @@ -0,0 +1,26 @@ +:original_name: dis_01_0132.html + +.. _dis_01_0132: + +Setting Alarm Rules +=================== + +Scenario +-------- + +You can customize the monitored objects and notification policies by setting alarm rules to learn DIS running status in a timely manner. + +A DIS alarm rule includes the alarm rule name, monitored object, metric, threshold, monitoring interval, and whether to send a notification. This section describes how to set DIS alarm rules. + +Procedure +--------- + +#. Log in to the management console. +#. Choose **All Services** > **Management & Deployment** > **Cloud Eye**. +#. In the left navigation pane, choose **Alarm Management** > **Alarm Rules** > **Create Alarm Rule**. +#. Set alarm rules for the DIS stream as prompted. Currently, only custom alarm rules are supported. +#. After the setting is complete, click **Create**. When an alarm that meets the rule is generated, the system automatically sends a notification. + + .. note:: + + For more information about DIS alarm rules, see the *Cloud Eye User Guide*. diff --git a/umn/source/monitoring/supported_metrics.rst b/umn/source/monitoring/supported_metrics.rst new file mode 100644 index 0000000..96c1272 --- /dev/null +++ b/umn/source/monitoring/supported_metrics.rst @@ -0,0 +1,78 @@ +:original_name: dis_01_0131.html + +.. _dis_01_0131: + +Supported Metrics +================= + +Description +----------- + +This section describes metrics reported by DIS to Cloud Eye as well as their namespaces and dimensions. You can use Cloud Eye to query metric information generated for DIS. + +Namespace +--------- + +SYS.DAYU + +Metrics +------- + +:ref:`Table 1 ` lists the DIS metrics. + +.. _dis_01_0131__table77137321225: + +.. table:: **Table 1** DIS metrics + + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Metric Name | Meaning | Value Range | Monitored Object | Monitoring Period (Raw Data) | + +==============================================+===================================================================================================+==============+==================+==============================+ + | Total Input Traffic | The amount of data uploaded through a stream during a specific period. | >= 0 bytes/s | Stream | 1 minute | + | | | | | | + | | Unit: byte/s | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Total Output Traffic | The amount of data downloaded through a stream during a specific period. | >= 0 bytes/s | Stream | 1 minute | + | | | | | | + | | Unit: byte/s | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Total Input Records | The number of records uploaded through a stream during a specific period. | >= 0 Count/s | Stream | 1 minute | + | | | | | | + | | Unit: count/s | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Total Output Records | The number of records downloaded through a stream during a specific period. | >= 0 Count/s | Stream | 1 minute | + | | | | | | + | | Unit: count/s | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Successful Upload Requests | The number of successful requests for uploading data through a stream during a specific period. | >= 0 Count/s | Stream | 1 minute | + | | | | | | + | | Unit: count/s | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Successful Download Requests | The number of successful requests for downloading data through a stream during a specific period. | >= 0 Count/s | Stream | 1 minute | + | | | | | | + | | Unit: count/s | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Average Processing Time of Upload Requests | Average upload request delay of a stream during a specific period. | 0 to 50 ms | Stream | 1 minute | + | | | | | | + | | Unit: ms | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Average Processing Time of Download Requests | Average download request delay of a stream during a specific period. | 0 to 50 ms | Stream | 1 minute | + | | | | | | + | | Unit: ms | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Throttled Upload Requests | The number of rejected upload requests due to flow control. | 0 to 1 | Stream | 1 minute | + | | | | | | + | | Unit: count/s | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + | Throttled Download Requests | The number of rejected download requests due to flow control. | 0 to 1 | Stream | 1 minute | + | | | | | | + | | Unit: count/s | | | | + +----------------------------------------------+---------------------------------------------------------------------------------------------------+--------------+------------------+------------------------------+ + +Dimension +--------- + +========= ======================== +Key Value +========= ======================== +stream_id Real-time data ingestion +========= ======================== diff --git a/umn/source/overview/functions.rst b/umn/source/overview/functions.rst new file mode 100644 index 0000000..c89042b --- /dev/null +++ b/umn/source/overview/functions.rst @@ -0,0 +1,45 @@ +:original_name: dis_01_0003.html + +.. _dis_01_0003: + +Functions +========= + +DIS manages the infrastructure, storage, networking, and configuration needed to stream your data. You do not have to worry about provisioning, deployment, and constant maintenance of hardware. In addition, DIS synchronously replicates data across availability zones, providing high availability and data durability. + +Key Modules +----------- + +DIS consists of the following four functional modules: + +- Service control + + - Creates, deletes, and configures DIS streams; synchronizes user information to the data plane. + - Allocates resources and automatically deploys DIS on the data plane. + +- Data processing + + - Receives user requests, and receives and stores authenticated data. + - Receives data read requests and returns the requested data to authorized users. + - Removes old data from DIS streams according to data aging policies. + - Stores user data into OBS. + +- Service maintenance + + - Installs and upgrades DIS. + - Performs configuration, preventive maintenance, monitoring, and log collection and analysis for DIS. + - Processes service orders. + +- User SDK + + - Provides Java APIs for users to push and pull data. + - Encrypts data. + +Key Capabilities +---------------- + +- Scalable: A DIS stream can seamlessly scale its data throughput from megabytes to terabytes per hour, and from thousands to millions of PUT records per second. +- Easy to use: You can create a DIS stream within seconds. It is easy to put data into a stream, and build a data processing application. +- Low cost: DIS has no upfront cost and you only pay for the resources you use. +- Parallel processing: DIS allows you to have multiple DIS applications processing the same stream concurrently. For example, you can have one application running real-time analytics and another sending data to OBS from the same stream. +- Secure and reliable: DIS can retain data for 24 to 72 hours to prevent data loss in case of application faults, individual machine faults, or facility faults. diff --git a/umn/source/overview/index.rst b/umn/source/overview/index.rst new file mode 100644 index 0000000..88a0686 --- /dev/null +++ b/umn/source/overview/index.rst @@ -0,0 +1,24 @@ +:original_name: dis_01_0600.html + +.. _dis_01_0600: + +Overview +======== + +- :ref:`What Is DIS? ` +- :ref:`What Can I Do with DIS? ` +- :ref:`Functions ` +- :ref:`Related Services ` +- :ref:`Permissions ` +- :ref:`Precautions ` + +.. toctree:: + :maxdepth: 1 + :hidden: + + what_is_dis + what_can_i_do_with_dis + functions + related_services + permissions + precautions diff --git a/umn/source/overview/permissions.rst b/umn/source/overview/permissions.rst new file mode 100644 index 0000000..475a39a --- /dev/null +++ b/umn/source/overview/permissions.rst @@ -0,0 +1,42 @@ +:original_name: dis_01_0005.html + +.. _dis_01_0005: + +Permissions +=========== + +You need an account on the management console to create, query, and delete DIS streams, and need an Access Key ID/Secret Access Key (AK/SK) file to push, pull, and dump data. + +For details about how to perform operations on IAM, see *IAM User Guide*. + +For details about DIS permissions, see :ref:`Table 1 ` and `Permissions `__. + +.. _dis_01_0005__table55585464113541: + +.. table:: **Table 1** Permission list + + +-------------+----------------------+------------------------+------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | Node Name | Permission Name | Managed Cloud Resource | Description | How to Assign Permissions | + +=============+======================+========================+==============================================================================+========================================================================================================================================================================================+ + | Base | Tenant Administrator | All services | Users have permission to operate all cloud resources owned by an enterprise. | Permission parameter settings: | + | | | | | | + | | | | | **Region**: set to the region to which your cloud resources belong. **Project**: set to the project to which your cloud resources belong. **Policy**: set to **Tenant Administrator**. | + +-------------+----------------------+------------------------+------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | DIS | DIS Administrator | DIS | Users have permissions to perform the following operations: | Permission parameter settings: | + | | | | | | + | | | | - Create, delete, modify, and query DIS streams | - **Region**: set to the region to which DIS streams belong. | + | | | | - Create, delete, modify, and query dump tasks | - **Project**: set to the project to which DIS streams belong. | + | | | | - Upload and download data through a stream | - **Policy**: set to **DIS Administrator**. | + | | | | - Query stream monitoring metrics | | + | | | | | Assign different permissions based on the destination to which data of DIS streams will be dumped. | + | | | | | | + | | | | | - To create DIS streams capable of dumping data to OBS and ensure that the users have permissions to use both DIS and OBS. | + | | | | | | + | | | | | - Parameter settings required for using OBS: | + | | | | | | + | | | | | **Region**: set to **Global service**. | + | | | | | | + | | | | | **Project**: set to **OBS**. | + | | | | | | + | | | | | **Policy**: set to **Tenant Administrator**. | + +-------------+----------------------+------------------------+------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/umn/source/overview/precautions.rst b/umn/source/overview/precautions.rst new file mode 100644 index 0000000..c02e7ad --- /dev/null +++ b/umn/source/overview/precautions.rst @@ -0,0 +1,21 @@ +:original_name: dis_01_0006.html + +.. _dis_01_0006: + +Precautions +=========== + +Read the following precautions before using DIS: + +- Recommended browser to access the DIS console + + - Google Chrome: 48.0 or later + + - Mozilla FireFox: 43.0 or later + + - Internet Explorer: 9.0 or later + + If you cannot log in to the DIS console using Internet Explorer 9.0, run the browser as the Administrator user or re-install the browser as the Administrator user (by default, the Administrator user is disabled in certain Windows systems, such as Windows 7 Ultimate. That is, users other than the Administrator user are selected by default to install Internet Explorer.) Reinstall Internet Explorer 9.0 or later (recommended) or run Internet Explorer 9.0 as user **Administrator**. + +- Sensitive data must be encrypted using DIS SDK or another encryption tools before being put into DIS streams. +- When a user uploads data using a DIS stream, the total size of records in a single request cannot exceed 5 MB. diff --git a/umn/source/overview/related_services.rst b/umn/source/overview/related_services.rst new file mode 100644 index 0000000..7c3fde1 --- /dev/null +++ b/umn/source/overview/related_services.rst @@ -0,0 +1,20 @@ +:original_name: dis_01_0004.html + +.. _dis_01_0004: + +Related Services +================ + +DIS works with the following services: + +- Object Storage Service (OBS) + + If you set **Dump Destination** to OBS, the uploaded data will be stored to OBS. + +- Identity and Access Management (IAM) + + IAM provides user authentication for DIS. + +- Cloud Trace Service (CTS) + + CTS is used to record the operations you have performed on DIS for later querying, auditing, or backtracking. For details, see the *Cloud Trace Service User Guide*. diff --git a/umn/source/overview/what_can_i_do_with_dis.rst b/umn/source/overview/what_can_i_do_with_dis.rst new file mode 100644 index 0000000..5d8504f --- /dev/null +++ b/umn/source/overview/what_can_i_do_with_dis.rst @@ -0,0 +1,24 @@ +:original_name: dis_01_0002.html + +.. _dis_01_0002: + +What Can I Do with DIS? +======================= + +You can use DIS for rapid data intake from producers and continuous data processing. The following are typical scenarios for using DIS: + +- Accelerated log and data feed intake + + You do not need to batch data on servers before you submit it for intake. Instead, you can have producers push data into streams immediately after data is produced. + + For example, system and application logs are pushed as they are streaming in and they will be available for processing in seconds. This prevents the data from being lost if the data producer fails. + +- Real-time metrics and reporting + + You can retrieve data from DIS streams for simple data analysis and reporting in real time. For example, your DIS applications can work on metrics and reporting for system and application logs as streaming data is being pushed to DIS streams using application programming interfaces (APIs), rather than wait to receive batches of data. + +- Real-time data analysis: DIS can be used to analyze real-time data. For example, you can transform data into valuable information and business intelligence by simply putting data into a DIS stream. This can be done in minutes instead of hours or days. + +- Complex stream processing + + You can create Directed Acyclic Graphs (DAGs) of DIS applications and streams. This typically involves putting data from one or multiple DIS applications into another stream for downstream processing by a different DIS application. diff --git a/umn/source/overview/what_is_dis.rst b/umn/source/overview/what_is_dis.rst new file mode 100644 index 0000000..92d4750 --- /dev/null +++ b/umn/source/overview/what_is_dis.rst @@ -0,0 +1,42 @@ +:original_name: dis_01_0001.html + +.. _dis_01_0001: + +What Is DIS? +============ + +Data Ingestion Service (DIS) addresses the challenge of transmitting data from outside the cloud to inside the cloud. DIS builds data intake streams for custom applications capable of processing or analyzing streaming data. DIS continuously captures, transmits, and stores terabytes of data from hundreds of thousands of sources every hour, such as logs, social media feeds, website clickstreams, and location-tracking events. + +Data Flows +---------- + +If you set the dump destination to Object Storage Service (OBS), the streaming data will be dumped to OBS. + + +.. figure:: /_static/images/en-us_image_0000001222097314.png + :alt: **Figure 1** Data flow-1 + + **Figure 1** Data flow-1 + +Basic Concepts +-------------- + +- Stream: A DIS stream is an ordered sequence of data records. Streams are distinguished from each other by the stream names assigned during DIS stream creation. When reading or writing streaming data, you need to specify the name of DIS stream from or to which data will be read or written. + +- Partition: Data records in DIS streams are distributed into partitions. A partition is a base throughput unit of a DIS stream. When creating a DIS stream, you are expected to specify the number of partitions needed within your stream. + +- Record: A record is a unit of data stored in a DIS stream. A record consists of a serial number, partition key, and data block. Data blocks are important data that your data producer adds to the data stream. The maximum size of a data block (data payload before base64 encoding) is 1 MB. + +- Sequence number: Each data record has a sequence number that is unique within its partition. The sequence number is assigned by DIS when a data producer calls PutRecords to add data to a DIS stream. + + Sequence numbers for the same partition key generally increase over time; the longer the time period between write requests (PutRecords requests), the larger the sequence numbers become. + +- DIS application: DIS applications write, read, and process data in DIS streams. You can develop DIS applications using the client library software development kit (SDK). + +- SDK: SDK is a Java-based client library. With SDK, you can build DIS applications easily to write, read, and process data in DIS streams. + +- Project: Projects are used to group and isolate OpenStack resources (computing resources, storage resources, and network resources). A project can be a department or a project team. Multiple projects can be created for one tenant account. A region has multiple projects, but one project is related to one region. DIS streams in different projects cannot communicate with each other. + +- Checkpoint: When an application consumes data, the latest serial number of the consumed data is recorded as a checkpoint. When the data is reconsumed, the consumption can be continued based on this checkpoint. + +- Application: When multiple applications consume data in the same stream, an APP is used as an identifier to distinguish consumption checkpoints of different applications.