diff --git a/docs/dms/umn/ALL_META.TXT.json b/docs/dms/umn/ALL_META.TXT.json new file mode 100644 index 000000000..a3f5366b2 --- /dev/null +++ b/docs/dms/umn/ALL_META.TXT.json @@ -0,0 +1,1282 @@ +[ + { + "uri":"kafka-ug-0723001.html", + "product_code":"dms", + "code":"1", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Service Overview", + "title":"Service Overview", + "githuburl":"" + }, + { + "uri":"kafka-pd-190605001.html", + "product_code":"dms", + "code":"2", + "des":"Apache Kafka is distributed message middleware that features high throughput, data persistence, horizontal scalability, and stream data processing. It adopts the publish-", + "doc_type":"usermanual", + "kw":"What Is DMS?,Service Overview,User Guide", + "title":"What Is DMS?", + "githuburl":"" + }, + { + "uri":"kafka-advantage.html", + "product_code":"dms", + "code":"3", + "des":"DMS provides easy-to-use message queuing based on Apache Kafka. Services can be quickly migrated to the cloud without any change, reducing maintenance and usage costs.Rap", + "doc_type":"usermanual", + "kw":"Product Advantages,Service Overview,User Guide", + "title":"Product Advantages", + "githuburl":"" + }, + { + "uri":"kafka-scenarios.html", + "product_code":"dms", + "code":"4", + "des":"Kafka is popular message-oriented middleware that features highly reliable, asynchronous message delivery. It is widely used for transmitting data between different syste", + "doc_type":"usermanual", + "kw":"Application Scenarios,Service Overview,User Guide", + "title":"Application Scenarios", + "githuburl":"" + }, + { + "uri":"Kafka-specification.html", + "product_code":"dms", + "code":"5", + "des":"Kafka instances are compatible with open-source Kafka 1.1.0, 2.3.0, and 2.7. The instance specifications are classified based on bandwidth, namely, 100 MB/s, 300 MB/s, 60", + "doc_type":"usermanual", + "kw":"Specifications,Service Overview,User Guide", + "title":"Specifications", + "githuburl":"" + }, + { + "uri":"kafka-pd-200720001.html", + "product_code":"dms", + "code":"6", + "des":"DMS is compatible with open-source Kafka and has customized and enhanced Kafka features. In addition to the advantages of open-source Kafka, DMS for Kafka provides more r", + "doc_type":"usermanual", + "kw":"Comparing DMS for Kafka and Open-Source Kafka,Service Overview,User Guide", + "title":"Comparing DMS for Kafka and Open-Source Kafka", + "githuburl":"" + }, + { + "uri":"kafka-pd-190605003.html", + "product_code":"dms", + "code":"7", + "des":"This section describes the notes and constraints on DMS.", + "doc_type":"usermanual", + "kw":"Notes and Constraints,Service Overview,User Guide", + "title":"Notes and Constraints", + "githuburl":"" + }, + { + "uri":"kafka-pd-190605002.html", + "product_code":"dms", + "code":"8", + "des":"Cloud Trace Service (CTS)CTS generates traces to provide you with a history of operations performed on cloud service resources. The traces include operation requests sent", + "doc_type":"usermanual", + "kw":"Related Services,Service Overview,User Guide", + "title":"Related Services", + "githuburl":"" + }, + { + "uri":"glossary-kafka.html", + "product_code":"dms", + "code":"9", + "des":"DMS for Kafka of the cloud service platform uses Kafka as the message engine. This chapter presents explanations of basic concepts of Kafka.A topic is a category for mess", + "doc_type":"usermanual", + "kw":"Basic Concepts,Service Overview,User Guide", + "title":"Basic Concepts", + "githuburl":"" + }, + { + "uri":"dms-ug-001.html", + "product_code":"dms", + "code":"10", + "des":"This section provides recommendations on configuring common parameters for Kafka producers and consumers.", + "doc_type":"usermanual", + "kw":"Best Practices,Service Overview,User Guide", + "title":"Best Practices", + "githuburl":"" + }, + { + "uri":"dms-ug-190128001.html", + "product_code":"dms", + "code":"11", + "des":"By default, there are two types of user permissions: user management and resource management.User management refers to the management of users, user groups, and user grou", + "doc_type":"usermanual", + "kw":"Permissions,Service Overview,User Guide", + "title":"Permissions", + "githuburl":"" + }, + { + "uri":"kafka-ug-180604012.html", + "product_code":"dms", + "code":"12", + "des":"Before creating a Kafka instance, ensure the availability of resources, including a virtual private cloud (VPC), subnet, security group, and security group rules. Each Ka", + "doc_type":"usermanual", + "kw":"Preparing Required Resources,User Guide", + "title":"Preparing Required Resources", + "githuburl":"" + }, + { + "uri":"dms-ug-180604013.html", + "product_code":"dms", + "code":"13", + "des":"Kafka instances are physically isolated and exclusively occupied by each tenant. You can customize the computing capabilities and storage space of an instance based on se", + "doc_type":"usermanual", + "kw":"Creating an Instance,User Guide", + "title":"Creating an Instance", + "githuburl":"" + }, + { + "uri":"kafka-ug190605003.html", + "product_code":"dms", + "code":"14", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Accessing a Kafka Instance", + "title":"Accessing a Kafka Instance", + "githuburl":"" + }, + { + "uri":"kafka-ug-180604020.html", + "product_code":"dms", + "code":"15", + "des":"This section describes how to use an open-source Kafka client to access a Kafka instance if SASL access is not enabled for the instance. There are two scenarios. For cros", + "doc_type":"usermanual", + "kw":"Accessing a Kafka Instance Without SASL,Accessing a Kafka Instance,User Guide", + "title":"Accessing a Kafka Instance Without SASL", + "githuburl":"" + }, + { + "uri":"kafka-ug-180801001.html", + "product_code":"dms", + "code":"16", + "des":"If you enable SASL_SSL when creating an instance, data will be encrypted before transmission for enhanced security.For security purposes, TLS_ECDHE_RSA_WITH_AES_128_GCM_S", + "doc_type":"usermanual", + "kw":"Accessing a Kafka Instance with SASL,Accessing a Kafka Instance,User Guide", + "title":"Accessing a Kafka Instance with SASL", + "githuburl":"" + }, + { + "uri":"kafka-ug-0001.html", + "product_code":"dms", + "code":"17", + "des":"VPCs are logically isolated from each other. If a Kafka instance and a Kafka client are in different VPCs within a region, they cannot communicate with each other. In thi", + "doc_type":"usermanual", + "kw":"Cross-VPC Access to a Kafka Instance,Accessing a Kafka Instance,User Guide", + "title":"Cross-VPC Access to a Kafka Instance", + "githuburl":"" + }, + { + "uri":"kafka-dnat.html", + "product_code":"dms", + "code":"18", + "des":"You can use destination NAT (DNAT) to access a Kafka instance so that the instance can provide services on the public network through port mapping.You have created EIPs. ", + "doc_type":"usermanual", + "kw":"Using DNAT to Access a Kafka Instance,Accessing a Kafka Instance,User Guide", + "title":"Using DNAT to Access a Kafka Instance", + "githuburl":"" + }, + { + "uri":"kafka-ug-180604011.html", + "product_code":"dms", + "code":"19", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Managing Instances", + "title":"Managing Instances", + "githuburl":"" + }, + { + "uri":"kafka-ug-180604014.html", + "product_code":"dms", + "code":"20", + "des":"View detailed information about a Kafka instance on the DMS console, for example, the IP addresses and port numbers for accessing the instance.Select the region where you", + "doc_type":"usermanual", + "kw":"Viewing an Instance,Managing Instances,User Guide", + "title":"Viewing an Instance", + "githuburl":"" + }, + { + "uri":"kafka-ug-180604015.html", + "product_code":"dms", + "code":"21", + "des":"Restart one or more Kafka instances at a time on the DMS console.When a Kafka instance is being restarted, message retrieval and creation requests of clients will be reje", + "doc_type":"usermanual", + "kw":"Restarting an Instance,Managing Instances,User Guide", + "title":"Restarting an Instance", + "githuburl":"" + }, + { + "uri":"kafka-ug-180604016.html", + "product_code":"dms", + "code":"22", + "des":"On the DMS console, you can delete one or more Kafka instances that have been created or failed to be created.Deleting a Kafka instance will delete the data in the instan", + "doc_type":"usermanual", + "kw":"Deleting an Instance,Managing Instances,User Guide", + "title":"Deleting an Instance", + "githuburl":"" + }, + { + "uri":"kafka-ug-180604017.html", + "product_code":"dms", + "code":"23", + "des":"After creating a Kafka instance, you can modify some parameters of the instance based on service requirements, including the instance name, description, security group, a", + "doc_type":"usermanual", + "kw":"Modifying the Information About an Instance,Managing Instances,User Guide", + "title":"Modifying the Information About an Instance", + "githuburl":"" + }, + { + "uri":"kafka-ug-0319001.html", + "product_code":"dms", + "code":"24", + "des":"To access a Kafka instance over a public network, enable public access and configure EIPs for the instance.If you no longer need public access to the instance, you can di", + "doc_type":"usermanual", + "kw":"Configuring Public Access,Managing Instances,User Guide", + "title":"Configuring Public Access", + "githuburl":"" + }, + { + "uri":"kafka-ug-180718001.html", + "product_code":"dms", + "code":"25", + "des":"You can reset the SASL_SSL password for accessing a Kafka instance by resetting Kafka password if you forget it.You can reset the Kafka password only if Kafka SASL_SSL ha", + "doc_type":"usermanual", + "kw":"Resetting Kafka Password,Managing Instances,User Guide", + "title":"Resetting Kafka Password", + "githuburl":"" + }, + { + "uri":"TagManagement.html", + "product_code":"dms", + "code":"26", + "des":"Tags facilitate Kafka instance identification and management.You can add tags to a Kafka instance when creating the instance or add tags on the Tags tab page of the creat", + "doc_type":"usermanual", + "kw":"Managing Instance Tags,Managing Instances,User Guide", + "title":"Managing Instance Tags", + "githuburl":"" + }, + { + "uri":"kafka-ug-200119002.html", + "product_code":"dms", + "code":"27", + "des":"After you initiate certain instance operations such as configuring public access and modifying the capacity threshold policy, a background task will start for each operat", + "doc_type":"usermanual", + "kw":"Viewing Background Tasks,Managing Instances,User Guide", + "title":"Viewing Background Tasks", + "githuburl":"" + }, + { + "uri":"kafka-ug-0004.html", + "product_code":"dms", + "code":"28", + "des":"On the Kafka console, you can view the disk usage of each broker.Select the region where your Kafka instance is located.You can query topics that use the most disk space ", + "doc_type":"usermanual", + "kw":"Viewing Disk Usage,Managing Instances,User Guide", + "title":"Viewing Disk Usage", + "githuburl":"" + }, + { + "uri":"kafka-ug-0720001.html", + "product_code":"dms", + "code":"29", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Managing Topics", + "title":"Managing Topics", + "githuburl":"" + }, + { + "uri":"dms-ug-180604018.html", + "product_code":"dms", + "code":"30", + "des":"A topic is a stream of messages. If automatic topic creation is not enabled during Kafka instance creation, you need to manually create topics for creating and retrieving", + "doc_type":"usermanual", + "kw":"Creating a Topic,Managing Topics,User Guide", + "title":"Creating a Topic", + "githuburl":"" + }, + { + "uri":"kafka-ug-180604019.html", + "product_code":"dms", + "code":"31", + "des":"Delete a topic using either of the following methods:By using the consoleBy using Kafka CLIA Kafka instance has been created, and a topic has been created in this instanc", + "doc_type":"usermanual", + "kw":"Deleting a Topic,Managing Topics,User Guide", + "title":"Deleting a Topic", + "githuburl":"" + }, + { + "uri":"kafka-ug-200506001.html", + "product_code":"dms", + "code":"32", + "des":"Aging time is a period that messages in the topic are retained for. Consumers must retrieve messages before this period ends. Otherwise, the messages will be deleted and ", + "doc_type":"usermanual", + "kw":"Modifying Topic Aging Time,Managing Topics,User Guide", + "title":"Modifying Topic Aging Time", + "githuburl":"" + }, + { + "uri":"kafka-ug-0006.html", + "product_code":"dms", + "code":"33", + "des":"After creating a topic, you can increase the number of partitions based on service requirements.Changing the number of partitions does not affect services.Methods for cha", + "doc_type":"usermanual", + "kw":"Changing Partition Quantity,Managing Topics,User Guide", + "title":"Changing Partition Quantity", + "githuburl":"" + }, + { + "uri":"kafka_ug_0022.html", + "product_code":"dms", + "code":"34", + "des":"Synchronous replication: A message is returned to the client only after the message creation request has been received and the message has been acknowledged by all replic", + "doc_type":"usermanual", + "kw":"Modifying Synchronous Replication and Flushing Settings,Managing Topics,User Guide", + "title":"Modifying Synchronous Replication and Flushing Settings", + "githuburl":"" + }, + { + "uri":"kafka_ug_0024.html", + "product_code":"dms", + "code":"35", + "des":"On the console, view sample code for creating and retrieving messages in Java, Go, and Python.Select the region where your Kafka instance is located.View sample code for ", + "doc_type":"usermanual", + "kw":"Viewing Sample Code,Managing Topics,User Guide", + "title":"Viewing Sample Code", + "githuburl":"" + }, + { + "uri":"kafka_ug_0027.html", + "product_code":"dms", + "code":"36", + "des":"Export topics on the console. Batch export is supported.A topic has been created.Select the region where your Kafka instance is located.The topic list contains the follow", + "doc_type":"usermanual", + "kw":"Exporting Topics,Managing Topics,User Guide", + "title":"Exporting Topics", + "githuburl":"" + }, + { + "uri":"kafka-ug-0720002.html", + "product_code":"dms", + "code":"37", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Managing Messages", + "title":"Managing Messages", + "githuburl":"" + }, + { + "uri":"kafka-ug-190904001.html", + "product_code":"dms", + "code":"38", + "des":"You can view the offset of different partitions, the message size, creation time, and body of messages in topics.Select the region where your Kafka instance is located.If", + "doc_type":"usermanual", + "kw":"Querying Messages,Managing Messages,User Guide", + "title":"Querying Messages", + "githuburl":"" + }, + { + "uri":"kafka-ug-0009.html", + "product_code":"dms", + "code":"39", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Managing Users", + "title":"Managing Users", + "githuburl":"" + }, + { + "uri":"kafka-ug-0003.html", + "product_code":"dms", + "code":"40", + "des":"DMS supports ACL permission management for topics. You can differentiate the operations that different users are allowed to perform on a topic by granting the users diffe", + "doc_type":"usermanual", + "kw":"Creating a SASL_SSL User,Managing Users,User Guide", + "title":"Creating a SASL_SSL User", + "githuburl":"" + }, + { + "uri":"kafka-ug-0002.html", + "product_code":"dms", + "code":"41", + "des":"DMS supports ACL permission management for topics. You can differentiate the operations that different users are allowed to perform on a topic by granting the users diffe", + "doc_type":"usermanual", + "kw":"Granting Permissions to a SASL_SSL User,Managing Users,User Guide", + "title":"Granting Permissions to a SASL_SSL User", + "githuburl":"" + }, + { + "uri":"kafka_ug_0025.html", + "product_code":"dms", + "code":"42", + "des":"If you forget the password of a SASL_SSL user created on the Users tab page, you can reset the password and use the new password to connect to the Kafka instance.If you f", + "doc_type":"usermanual", + "kw":"Resetting the SASL_SSL Password,Managing Users,User Guide", + "title":"Resetting the SASL_SSL Password", + "githuburl":"" + }, + { + "uri":"kafka_ug_0026.html", + "product_code":"dms", + "code":"43", + "des":"This section describes how to delete a SASL_SSL user.Select the region where your Kafka instance is located.On the Users tab page, click Delete in the row that contains t", + "doc_type":"usermanual", + "kw":"Deleting a SASL_SSL User,Managing Users,User Guide", + "title":"Deleting a SASL_SSL User", + "githuburl":"" + }, + { + "uri":"kafka-ug-0011.html", + "product_code":"dms", + "code":"44", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Managing Consumer Groups", + "title":"Managing Consumer Groups", + "githuburl":"" + }, + { + "uri":"kafka_ug_0021.html", + "product_code":"dms", + "code":"45", + "des":"View the consumer group list, consumer list, and consumer offsets.The consumer list can be viewed only when consumers in a consumer group are connected to the Kafka insta", + "doc_type":"usermanual", + "kw":"Querying Consumer Group Details,Managing Consumer Groups,User Guide", + "title":"Querying Consumer Group Details", + "githuburl":"" + }, + { + "uri":"kafka-ug-0012.html", + "product_code":"dms", + "code":"46", + "des":"You can delete a consumer group using either of the following methods:Method 1: Delete a consumer group on the console.Method 2: Use Kafka CLI to delete a consumer group.", + "doc_type":"usermanual", + "kw":"Deleting a Consumer Group,Managing Consumer Groups,User Guide", + "title":"Deleting a Consumer Group", + "githuburl":"" + }, + { + "uri":"kafka-ug-0014.html", + "product_code":"dms", + "code":"47", + "des":"Resetting the consumer offset is to change the retrieval position of a consumer.Messages may be retrieved more than once after the offset is reset. Exercise caution when ", + "doc_type":"usermanual", + "kw":"Resetting the Consumer Offset,Managing Consumer Groups,User Guide", + "title":"Resetting the Consumer Offset", + "githuburl":"" + }, + { + "uri":"kafka-ug-0015.html", + "product_code":"dms", + "code":"48", + "des":"View consumer connection addresses on the DMS console.The connection address of a consumer can be viewed only when the consumer is connected to a Kafka instance.Select th", + "doc_type":"usermanual", + "kw":"Viewing Consumer Connection Addresses,Managing Consumer Groups,User Guide", + "title":"Viewing Consumer Connection Addresses", + "githuburl":"" + }, + { + "uri":"kafka-ug-0007.html", + "product_code":"dms", + "code":"49", + "des":"Your Kafka instances, topics, and consumers come with default configuration parameter settings. You can modify common parameters on the DMS console. For details about par", + "doc_type":"usermanual", + "kw":"Modifying Kafka Parameters,User Guide", + "title":"Modifying Kafka Parameters", + "githuburl":"" + }, + { + "uri":"kafka-ug-180413001.html", + "product_code":"dms", + "code":"50", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Monitoring", + "title":"Monitoring", + "githuburl":"" + }, + { + "uri":"kafka-ug-190605001.html", + "product_code":"dms", + "code":"51", + "des":"Cloud Eye monitors Kafka instance metrics in real time. You can view these metrics on the Cloud Eye console.At least one Kafka instance has been created. The instance has", + "doc_type":"usermanual", + "kw":"Viewing Metrics,Monitoring,User Guide", + "title":"Viewing Metrics", + "githuburl":"" + }, + { + "uri":"dms-ug-180413002.html", + "product_code":"dms", + "code":"52", + "des":"This section describes DMS metrics reported to Cloud Eye as well as their namespace and dimensions. You can use the Cloud Eye console to query the Kafka metrics and alarm", + "doc_type":"usermanual", + "kw":"Kafka Metrics,Monitoring,User Guide", + "title":"Kafka Metrics", + "githuburl":"" + }, + { + "uri":"kafka-ug-180418001.html", + "product_code":"dms", + "code":"53", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Auditing", + "title":"Auditing", + "githuburl":"" + }, + { + "uri":"kafka-ug-180418002.html", + "product_code":"dms", + "code":"54", + "des":"With Cloud Trace Service (CTS), you can record operations associated with DMS for later query, audit, and backtrack operations.", + "doc_type":"usermanual", + "kw":"Operations Logged by CTS,Auditing,User Guide", + "title":"Operations Logged by CTS", + "githuburl":"" + }, + { + "uri":"kafka-ug-180418003.html", + "product_code":"dms", + "code":"55", + "des":"This section describes how to view operation records of the last 7 days on the CTS console.Select the region where your Kafka instance is located.Trace Source: Select DMS", + "doc_type":"usermanual", + "kw":"Viewing Audit Logs,Auditing,User Guide", + "title":"Viewing Audit Logs", + "githuburl":"" + }, + { + "uri":"kafka-ug-0723004.html", + "product_code":"dms", + "code":"56", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"FAQs", + "title":"FAQs", + "githuburl":"" + }, + { + "uri":"kafka-faq-191030002.html", + "product_code":"dms", + "code":"57", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Instances", + "title":"Instances", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426002.html", + "product_code":"dms", + "code":"58", + "des":"To improve the reliability of a Kafka instance, you are advised to select three AZs or more when creating the instance. You cannot select two AZs.Each Kafka instance cont", + "doc_type":"usermanual", + "kw":"Why Can't I Select Two AZs?,Instances,User Guide", + "title":"Why Can't I Select Two AZs?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426003.html", + "product_code":"dms", + "code":"59", + "des":"This may be because you do not have the Server Administrator and VPC Administrator permissions. For details about how to add permissions to a user group, see \"User and Us", + "doc_type":"usermanual", + "kw":"Why Can't I View the Subnet and Security Group Information When Creating a DMS Instance?,Instances,U", + "title":"Why Can't I View the Subnet and Security Group Information When Creating a DMS Instance?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426005.html", + "product_code":"dms", + "code":"60", + "des":"The storage space is the space for storing messages (including messages in replicas), logs and metadata. When specifying storage space, specify the disk type and disk siz", + "doc_type":"usermanual", + "kw":"How Do I Select Storage Space for a Kafka Instance?,Instances,User Guide", + "title":"How Do I Select Storage Space for a Kafka Instance?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426006.html", + "product_code":"dms", + "code":"61", + "des":"High I/O: The average latency is 6 to 10 ms, and the maximum bandwidth is 120 MB/s (read + write).Ultra-high I/O: The average latency is 1 to 3 ms, and the maximum bandwi", + "doc_type":"usermanual", + "kw":"How Do I Choose Between High I/O and Ultra-high I/O?,Instances,User Guide", + "title":"How Do I Choose Between High I/O and Ultra-high I/O?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426007.html", + "product_code":"dms", + "code":"62", + "des":"The following policies are supported:Stop productionWhen the memory usage reaches the disk capacity threshold (95%), new messages will no longer be created, but existing ", + "doc_type":"usermanual", + "kw":"Which Capacity Threshold Policy Should I Use?,Instances,User Guide", + "title":"Which Capacity Threshold Policy Should I Use?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426008.html", + "product_code":"dms", + "code":"63", + "des":"Kafka v1.1.0, v2.3.0, and v2.7.", + "doc_type":"usermanual", + "kw":"Which Kafka Versions Are Supported?,Instances,User Guide", + "title":"Which Kafka Versions Are Supported?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426009.html", + "product_code":"dms", + "code":"64", + "des":"Kafka instances are managed using ZooKeeper. Opening ZooKeeper may cause misoperations and service losses. ZooKeeper is used only within Kafka clusters and does not provi", + "doc_type":"usermanual", + "kw":"What Is the ZooKeeper Address of a Kafka Instance?,Instances,User Guide", + "title":"What Is the ZooKeeper Address of a Kafka Instance?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426010.html", + "product_code":"dms", + "code":"65", + "des":"Yes. A Kafka instance is a cluster that consists of three or more brokers.", + "doc_type":"usermanual", + "kw":"Are Kafka Instances in Cluster Mode?,Instances,User Guide", + "title":"Are Kafka Instances in Cluster Mode?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426011.html", + "product_code":"dms", + "code":"66", + "des":"No. You must access a Kafka instance through one of the following ports:Accessing a Kafka instance without SASL:The port varies with the access mode:Intra-VPC access: por", + "doc_type":"usermanual", + "kw":"Can I Modify the Connection Address for Accessing a Kafka Instance?,Instances,User Guide", + "title":"Can I Modify the Connection Address for Accessing a Kafka Instance?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426012.html", + "product_code":"dms", + "code":"67", + "des":"The certificates are valid for more than 15 years. You do not need to worry about certificate expiration. The certificates are used for one-way authentication when enabli", + "doc_type":"usermanual", + "kw":"How Long Are Kafka SSL Certificates Valid for?,Instances,User Guide", + "title":"How Long Are Kafka SSL Certificates Valid for?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426013.html", + "product_code":"dms", + "code":"68", + "des":"Unfortunately, you cannot synchronize two Kafka instances in real time. To migrate services from one instance to another, create messages to both instances. After all mes", + "doc_type":"usermanual", + "kw":"How Do I Synchronize Data from One Kafka Instance to Another?,Instances,User Guide", + "title":"How Do I Synchronize Data from One Kafka Instance to Another?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426014.html", + "product_code":"dms", + "code":"69", + "des":"The SASL_SSL setting cannot be changed once the instance has been created. Be careful when configuring this setting during instance creation. If you need to change the se", + "doc_type":"usermanual", + "kw":"How Do I Change the SASL_SSL Setting of a Kafka Instance?,Instances,User Guide", + "title":"How Do I Change the SASL_SSL Setting of a Kafka Instance?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0015.html", + "product_code":"dms", + "code":"70", + "des":"Kafka brokers and ZooKeeper are deployed on the same VM.", + "doc_type":"usermanual", + "kw":"Are Kafka Brokers and ZooKeeper Deployed on the Same VM or on Different VMs?,Instances,User Guide", + "title":"Are Kafka Brokers and ZooKeeper Deployed on the Same VM or on Different VMs?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0020.html", + "product_code":"dms", + "code":"71", + "des":"For security purposes, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 is supported.", + "doc_type":"usermanual", + "kw":"Which Cipher Suites Are Supported by Kafka?,Instances,User Guide", + "title":"Which Cipher Suites Are Supported by Kafka?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0023.html", + "product_code":"dms", + "code":"72", + "des":"No. The AZ configuration cannot be changed once the instance is created. To use multiple AZs, create another instance.", + "doc_type":"usermanual", + "kw":"Can I Change an Instance from Single-AZ Deployment to Multi-AZ Deployment?,Instances,User Guide", + "title":"Can I Change an Instance from Single-AZ Deployment to Multi-AZ Deployment?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0025.html", + "product_code":"dms", + "code":"73", + "des":"DMS for Kafka supports cross-AZ disaster recovery. If you select multiple AZs when creating an instance, cross-AZ disaster recovery will be available.You can view the AZs", + "doc_type":"usermanual", + "kw":"Does DMS for Kafka Support Cross-AZ Disaster Recovery? Where Can I View the AZs Configured for an Ex", + "title":"Does DMS for Kafka Support Cross-AZ Disaster Recovery? Where Can I View the AZs Configured for an Existing Instance?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0030.html", + "product_code":"dms", + "code":"74", + "des":"Yes.", + "doc_type":"usermanual", + "kw":"Do Kafka Instances Support Disk Encryption?,Instances,User Guide", + "title":"Do Kafka Instances Support Disk Encryption?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0036.html", + "product_code":"dms", + "code":"75", + "des":"No. Once an instance is created, its VPC and subnet cannot be changed.", + "doc_type":"usermanual", + "kw":"Can I Change the VPC and Subnet After a Kafka Instance Is Created?,Instances,User Guide", + "title":"Can I Change the VPC and Subnet After a Kafka Instance Is Created?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0037.html", + "product_code":"dms", + "code":"76", + "des":"You can find Kafka Streams use cases on the official Kafka website.", + "doc_type":"usermanual", + "kw":"Where Can I Find Kafka Streams Use Cases?,Instances,User Guide", + "title":"Where Can I Find Kafka Streams Use Cases?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0040.html", + "product_code":"dms", + "code":"77", + "des":"No. Kafka instances cannot be upgraded once they are created. To use a higher Kafka version, create another Kafka instance.", + "doc_type":"usermanual", + "kw":"Can I Upgrade Kafka Instances?,Instances,User Guide", + "title":"Can I Upgrade Kafka Instances?", + "githuburl":"" + }, + { + "uri":"kafka_faq_0046.html", + "product_code":"dms", + "code":"78", + "des":"On the DMS console, click the name of the target Kafka instance. Disable Public Access in the Connection section on the Basic Information tab page, and then enable it aga", + "doc_type":"usermanual", + "kw":"How Do I Bind an EIP Again?,Instances,User Guide", + "title":"How Do I Bind an EIP Again?", + "githuburl":"" + }, + { + "uri":"kafka-faq-191030001.html", + "product_code":"dms", + "code":"79", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Connections", + "title":"Connections", + "githuburl":"" + }, + { + "uri":"kafka-faq-0604001.html", + "product_code":"dms", + "code":"80", + "des":"This section describes how to troubleshoot Kafka connection problems.If the connection to a Kafka instance is abnormal, perform the following operations to troubleshoot t", + "doc_type":"usermanual", + "kw":"Troubleshooting Kafka Connection Exceptions,Connections,User Guide", + "title":"Troubleshooting Kafka Connection Exceptions", + "githuburl":"" + }, + { + "uri":"kafka-faq-180604024.html", + "product_code":"dms", + "code":"81", + "des":"Kafka instances can be accessed within a VPC, across VPCs, through DNAT, or over public networks. Before accessing a Kafka instance, configure a security group.If they us", + "doc_type":"usermanual", + "kw":"How Do I Select and Configure a Security Group?,Connections,User Guide", + "title":"How Do I Select and Configure a Security Group?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426015.html", + "product_code":"dms", + "code":"82", + "des":"Yes. For details, see the instance access instructions.", + "doc_type":"usermanual", + "kw":"Can I Access a Kafka Instance Over a Public Network?,Connections,User Guide", + "title":"Can I Access a Kafka Instance Over a Public Network?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426016.html", + "product_code":"dms", + "code":"83", + "des":"The number of connection addresses of a Kafka instance is the same as the number of brokers of the instance.", + "doc_type":"usermanual", + "kw":"How Many Connection Addresses Does a Kafka Instance Have by Default?,Connections,User Guide", + "title":"How Many Connection Addresses Does a Kafka Instance Have by Default?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426017.html", + "product_code":"dms", + "code":"84", + "des":"Yes. You can access a Kafka instance across regions over a public network or by using direct connections.", + "doc_type":"usermanual", + "kw":"Do Kafka Instances Support Cross-Region Access?,Connections,User Guide", + "title":"Do Kafka Instances Support Cross-Region Access?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426019.html", + "product_code":"dms", + "code":"85", + "des":"Yes. You can use one of the following methods to access a Kafka instance across VPCs:Establish a VPC peering connection to allow two VPCs to communicate with each other. ", + "doc_type":"usermanual", + "kw":"Do Kafka Instances Support Cross-VPC Access?,Connections,User Guide", + "title":"Do Kafka Instances Support Cross-VPC Access?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426020.html", + "product_code":"dms", + "code":"86", + "des":"Yes.If the client and the instance are in the same VPC, cross-subnet access is supported.", + "doc_type":"usermanual", + "kw":"Do Kafka Instances Support Cross-Subnet Access?,Connections,User Guide", + "title":"Do Kafka Instances Support Cross-Subnet Access?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426023.html", + "product_code":"dms", + "code":"87", + "des":"No, Kerberos authentication is not supported. Kafka supports client authentication with SASL and API calling authentication using tokens and AK/SK.To access an instance i", + "doc_type":"usermanual", + "kw":"Does DMS for Kafka Support Authentication with Kerberos?,Connections,User Guide", + "title":"Does DMS for Kafka Support Authentication with Kerberos?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200708002.html", + "product_code":"dms", + "code":"88", + "des":"Yes. No password is required for accessing a Kafka instance with SASL disabled. For details, see Accessing a Kafka Instance Without SASL.", + "doc_type":"usermanual", + "kw":"Does DMS for Kafka Support Password-Free Access?,Connections,User Guide", + "title":"Does DMS for Kafka Support Password-Free Access?", + "githuburl":"" + }, + { + "uri":"kafka-connect-other.html", + "product_code":"dms", + "code":"89", + "des":"Kafka instances are fully compatible with open-source clients. You can obtain clients in other programming languages and access your instance as instructed by the officia", + "doc_type":"usermanual", + "kw":"Obtaining Kafka Clients,Connections,User Guide", + "title":"Obtaining Kafka Clients", + "githuburl":"" + }, + { + "uri":"kafka-faq-0001.html", + "product_code":"dms", + "code":"90", + "des":"Click the name of your Kafka instance. In the Connection section on the Basic Information tab page, view Instance Address (Public Network).For details about how to connec", + "doc_type":"usermanual", + "kw":"How Do I Obtain the Public Access Address After Public Access Is Enabled?,Connections,User Guide", + "title":"How Do I Obtain the Public Access Address After Public Access Is Enabled?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0026.html", + "product_code":"dms", + "code":"91", + "des":"No.", + "doc_type":"usermanual", + "kw":"Does DMS for Kafka Support Authentication on Clients by the Server?,Connections,User Guide", + "title":"Does DMS for Kafka Support Authentication on Clients by the Server?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0027.html", + "product_code":"dms", + "code":"92", + "des":"No. You can only use JKS certificates for connecting to instances in Java.", + "doc_type":"usermanual", + "kw":"Can I Use PEM SSL Truststore When Connecting to a Kafka Instance with SASL_SSL Enabled?,Connections,", + "title":"Can I Use PEM SSL Truststore When Connecting to a Kafka Instance with SASL_SSL Enabled?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0028.html", + "product_code":"dms", + "code":"93", + "des":"JKS certificates are used for connecting to instances in Java and CRT certificates are used for connecting to instances in Python.", + "doc_type":"usermanual", + "kw":"What Are the Differences Between JKS and CRT Certificates?,Connections,User Guide", + "title":"What Are the Differences Between JKS and CRT Certificates?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0029.html", + "product_code":"dms", + "code":"94", + "des":"TLS 1.2.", + "doc_type":"usermanual", + "kw":"Which TLS Version Does DMS for Kafka Support?,Connections,User Guide", + "title":"Which TLS Version Does DMS for Kafka Support?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0033.html", + "product_code":"dms", + "code":"95", + "des":"Yes. The maximum allowed number of client connections varies by instance specifications.If the bandwidth is 100 MB/s, a maximum of 3000 client connections are allowed.If ", + "doc_type":"usermanual", + "kw":"Is There a Limit on the Number of Client Connections to a Kafka Instance?,Connections,User Guide", + "title":"Is There a Limit on the Number of Client Connections to a Kafka Instance?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0034.html", + "product_code":"dms", + "code":"96", + "des":"Each Kafka broker allows a maximum of 1000 connections from each IP address by default. Excess connections will be rejected. You can change the limit by referring to Modi", + "doc_type":"usermanual", + "kw":"How Many Connections Are Allowed from Each IP Address?,Connections,User Guide", + "title":"How Many Connections Are Allowed from Each IP Address?", + "githuburl":"" + }, + { + "uri":"kafka-faq-191030003.html", + "product_code":"dms", + "code":"97", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Topics and Partitions", + "title":"Topics and Partitions", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426024.html", + "product_code":"dms", + "code":"98", + "des":"The number of topics is related to the total number of topic partitions and the number of partitions in each topic. There is an upper limit on the aggregate number of par", + "doc_type":"usermanual", + "kw":"Is There a Limit on the Number of Topics in a Kafka Instance?,Topics and Partitions,User Guide", + "title":"Is There a Limit on the Number of Topics in a Kafka Instance?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426025.html", + "product_code":"dms", + "code":"99", + "des":"Kafka manages messages by partition. If there are too many partitions, message creation, storage, and retrieval will be fragmented, affecting the performance and stabilit", + "doc_type":"usermanual", + "kw":"Why Is Partition Quantity Limited?,Topics and Partitions,User Guide", + "title":"Why Is Partition Quantity Limited?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426101.html", + "product_code":"dms", + "code":"100", + "des":"No. If you want to use fewer partitions, delete the corresponding topic, create another one, and specify the desired number of partitions.", + "doc_type":"usermanual", + "kw":"Can I Change the Partition Quantity?,Topics and Partitions,User Guide", + "title":"Can I Change the Partition Quantity?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426026.html", + "product_code":"dms", + "code":"101", + "des":"Possible cause: The aggregate number of partitions of created topics has reached the upper limit. The maximum number of partitions varies with instance specifications. Fo", + "doc_type":"usermanual", + "kw":"Why Do I Fail to Create Topics?,Topics and Partitions,User Guide", + "title":"Why Do I Fail to Create Topics?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426027.html", + "product_code":"dms", + "code":"102", + "des":"Automatic topic creation is supported, but batch topic import is not supported. You can only export topics in batches.Enable automatic topic creation using one of the fol", + "doc_type":"usermanual", + "kw":"Do Kafka Instances Support Batch Importing Topics or Automatic Topic Creation?,Topics and Partitions", + "title":"Do Kafka Instances Support Batch Importing Topics or Automatic Topic Creation?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426028.html", + "product_code":"dms", + "code":"103", + "des":"This may be because automatic topic creation has been enabled and a consumer is connecting to the topic. If no existing topics are available for message creation, new top", + "doc_type":"usermanual", + "kw":"Why Do Deleted Topics Still Exist?,Topics and Partitions,User Guide", + "title":"Why Do Deleted Topics Still Exist?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426030.html", + "product_code":"dms", + "code":"104", + "des":"Yes. Use either of the following methods to check the disk space used by a topic:Click next to the Kafka instance name to go to the Cloud Eye console. On the Queues tab ", + "doc_type":"usermanual", + "kw":"Can I View the Disk Space Used by a Topic?,Topics and Partitions,User Guide", + "title":"Can I View the Disk Space Used by a Topic?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426032.html", + "product_code":"dms", + "code":"105", + "des":"If you have enabled SASL_SSL for your Kafka instance, you can configure ACL permissions for your topics. On the Topics tab page of the Kafka console, click Grant User Per", + "doc_type":"usermanual", + "kw":"Can I Add ACL Permissions for Topics?,Topics and Partitions,User Guide", + "title":"Can I Add ACL Permissions for Topics?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0003.html", + "product_code":"dms", + "code":"106", + "des":"Messages are not deleted immediately after being retrieved. They are deleted only when the aging time expires.You can shorten the aging time.", + "doc_type":"usermanual", + "kw":"What Should I Do If Kafka Storage Space Is Used Up Because Retrieved Messages Are Not Deleted?,Topic", + "title":"What Should I Do If Kafka Storage Space Is Used Up Because Retrieved Messages Are Not Deleted?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0010.html", + "product_code":"dms", + "code":"107", + "des":"Yes. A Kafka instance will be restarted if you enable or disable automatic topic creation for it.", + "doc_type":"usermanual", + "kw":"Will a Kafka Instance Be Restarted After Its Automatic Topic Creation Setting Is Modified?,Topics an", + "title":"Will a Kafka Instance Be Restarted After Its Automatic Topic Creation Setting Is Modified?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0014.html", + "product_code":"dms", + "code":"108", + "des":"On the Kafka console, click the name of your instance.In the Instance Information section of the Basic Information tab page, click next to Automatic Topic Creation to di", + "doc_type":"usermanual", + "kw":"How Do I Disable Automatic Topic Creation?,Topics and Partitions,User Guide", + "title":"How Do I Disable Automatic Topic Creation?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0031.html", + "product_code":"dms", + "code":"109", + "des":"Yes, just simply unsubscribe from it on the Kafka client.", + "doc_type":"usermanual", + "kw":"Can I Delete Unnecessary Topics in a Consumer Group?,Topics and Partitions,User Guide", + "title":"Can I Delete Unnecessary Topics in a Consumer Group?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0038.html", + "product_code":"dms", + "code":"110", + "des":"Symptom: Different consumers in a consumer group have different topic permissions. When a consumer attempts to retrieve messages from a topic, the error message \"Not auth", + "doc_type":"usermanual", + "kw":"What Can I Do If a Consumer Fails to Retrieve Messages from a Topic Due to Insufficient Permissions?", + "title":"What Can I Do If a Consumer Fails to Retrieve Messages from a Topic Due to Insufficient Permissions?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200423001.html", + "product_code":"dms", + "code":"111", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Consumer Groups", + "title":"Consumer Groups", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426033.html", + "product_code":"dms", + "code":"112", + "des":"No. They are generated automatically when you use the instance.For details about creating and retrieving messages after connecting to a Kafka instance, see Accessing a Ka", + "doc_type":"usermanual", + "kw":"Do I Need to Create Consumer Groups, Producers, and Consumers for Kafka Instances?,Consumer Groups,U", + "title":"Do I Need to Create Consumer Groups, Producers, and Consumers for Kafka Instances?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0032.html", + "product_code":"dms", + "code":"113", + "des":"No. You can directly delete the consumer group.", + "doc_type":"usermanual", + "kw":"Do I Need to Unsubscribe from a Topic Before Deleting a Consumer Group?,Consumer Groups,User Guide", + "title":"Do I Need to Unsubscribe from a Topic Before Deleting a Consumer Group?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0043.html", + "product_code":"dms", + "code":"114", + "des":"Yes.Kafka uses the offsets.retention.minutes parameter to control how long to keep offsets for a consumer group. If offsets are not committed within this period, they wil", + "doc_type":"usermanual", + "kw":"Will a Consumer Group Without Active Consumers Be Automatically Deleted in 14 Days?,Consumer Groups,", + "title":"Will a Consumer Group Without Active Consumers Be Automatically Deleted in 14 Days?", + "githuburl":"" + }, + { + "uri":"kafka-faq-190416001.html", + "product_code":"dms", + "code":"115", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Messages", + "title":"Messages", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426035.html", + "product_code":"dms", + "code":"116", + "des":"10 MB.", + "doc_type":"usermanual", + "kw":"What Is the Maximum Size of a Message that Can be Created?,Messages,User Guide", + "title":"What Is the Maximum Size of a Message that Can be Created?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426036.html", + "product_code":"dms", + "code":"117", + "des":"Rebalancing is a process where partitions of topics are re-allocated for a consumer group.In normal cases, rebalancing occurs inevitably when a consumer is added to or re", + "doc_type":"usermanual", + "kw":"Why Does Message Poll Often Fail During Rebalancing?,Messages,User Guide", + "title":"Why Does Message Poll Often Fail During Rebalancing?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426037.html", + "product_code":"dms", + "code":"118", + "des":"Possible cause 1: The message has been aged.Solution: Change the aging time.Solution: Change the aging time.Possible cause 2: The createTime timestamp of the message is i", + "doc_type":"usermanual", + "kw":"Why Can't I Query Messages on the Console?,Messages,User Guide", + "title":"Why Can't I Query Messages on the Console?", + "githuburl":"" + }, + { + "uri":"kafka-faq-200708001.html", + "product_code":"dms", + "code":"119", + "des":"If the aging time has been set for a topic, the value of the log.retention.hours parameter does not take effect for the topic. The value of the log.retention.hours parame", + "doc_type":"usermanual", + "kw":"Why Do Messages Still Exist After the Retention Period Elapses?,Messages,User Guide", + "title":"Why Do Messages Still Exist After the Retention Period Elapses?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0018.html", + "product_code":"dms", + "code":"120", + "des":"No.", + "doc_type":"usermanual", + "kw":"Do Kafka Instances Support Delayed Message Delivery?,Messages,User Guide", + "title":"Do Kafka Instances Support Delayed Message Delivery?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0041.html", + "product_code":"dms", + "code":"121", + "des":"View the number of accumulated messages using any of the following methods:On the Consumer Groups page of an instance, click the name of the consumer group whose accumula", + "doc_type":"usermanual", + "kw":"How Do I View the Number of Accumulated Messages?,Messages,User Guide", + "title":"How Do I View the Number of Accumulated Messages?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0045.html", + "product_code":"dms", + "code":"122", + "des":"The message creation time is specified by CreateTime when a producer creates messages. If this parameter is not set during message creation, the message creation time is ", + "doc_type":"usermanual", + "kw":"Why Is the Message Creation Time Displayed as Year 1970?,Messages,User Guide", + "title":"Why Is the Message Creation Time Displayed as Year 1970?", + "githuburl":"" + }, + { + "uri":"kafka-faq-191030004.html", + "product_code":"dms", + "code":"123", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Monitoring & Alarm", + "title":"Monitoring & Alarm", + "githuburl":"" + }, + { + "uri":"kafka-faq-200426041.html", + "product_code":"dms", + "code":"124", + "des":"The possible causes are as follows:The topic name starts with a special character, such as an underscore (_) or a number sign (#).The consumer group name starts with a sp", + "doc_type":"usermanual", + "kw":"Why Can't I View the Monitoring Data?,Monitoring & Alarm,User Guide", + "title":"Why Can't I View the Monitoring Data?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0007.html", + "product_code":"dms", + "code":"125", + "des":"Symptom: The monitoring data shows that there are 810 million accumulated messages. However, the Kafka console shows that there are 100 million messages in all six topics", + "doc_type":"usermanual", + "kw":"Why Is the Monitored Number of Accumulated Messages Inconsistent with the Message Quantity Displayed", + "title":"Why Is the Monitored Number of Accumulated Messages Inconsistent with the Message Quantity Displayed on the Kafka Console?", + "githuburl":"" + }, + { + "uri":"kafka-faq-0022.html", + "product_code":"dms", + "code":"126", + "des":"The monitoring data is reported every minute. The reported data will be displayed on the monitoring page after being sorted. This process takes less than 20 minutes. Afte", + "doc_type":"usermanual", + "kw":"Why Is a Consumer Group Still on the Monitoring Page After Being Deleted?,Monitoring & Alarm,User Gu", + "title":"Why Is a Consumer Group Still on the Monitoring Page After Being Deleted?", + "githuburl":"" + }, + { + "uri":"kafka-ug-00001.html", + "product_code":"dms", + "code":"127", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Change History,User Guide", + "title":"Change History", + "githuburl":"" + }, + { + "uri":"dms-ug-0312114.html", + "product_code":"dms", + "code":"128", + "des":"See Glossary.", + "doc_type":"usermanual", + "kw":"Glossary,User Guide", + "title":"Glossary", + "githuburl":"" + } +] \ No newline at end of file diff --git a/docs/dms/umn/CLASS.TXT.json b/docs/dms/umn/CLASS.TXT.json new file mode 100644 index 000000000..bf94028ef --- /dev/null +++ b/docs/dms/umn/CLASS.TXT.json @@ -0,0 +1,1154 @@ +[ + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Service Overview", + "uri":"kafka-ug-0723001.html", + "doc_type":"usermanual", + "p_code":"", + "code":"1" + }, + { + "desc":"Apache Kafka is distributed message middleware that features high throughput, data persistence, horizontal scalability, and stream data processing. It adopts the publish-", + "product_code":"dms", + "title":"What Is DMS?", + "uri":"kafka-pd-190605001.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"2" + }, + { + "desc":"DMS provides easy-to-use message queuing based on Apache Kafka. Services can be quickly migrated to the cloud without any change, reducing maintenance and usage costs.Rap", + "product_code":"dms", + "title":"Product Advantages", + "uri":"kafka-advantage.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"3" + }, + { + "desc":"Kafka is popular message-oriented middleware that features highly reliable, asynchronous message delivery. It is widely used for transmitting data between different syste", + "product_code":"dms", + "title":"Application Scenarios", + "uri":"kafka-scenarios.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"4" + }, + { + "desc":"Kafka instances are compatible with open-source Kafka 1.1.0, 2.3.0, and 2.7. The instance specifications are classified based on bandwidth, namely, 100 MB/s, 300 MB/s, 60", + "product_code":"dms", + "title":"Specifications", + "uri":"Kafka-specification.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"5" + }, + { + "desc":"DMS is compatible with open-source Kafka and has customized and enhanced Kafka features. In addition to the advantages of open-source Kafka, DMS for Kafka provides more r", + "product_code":"dms", + "title":"Comparing DMS for Kafka and Open-Source Kafka", + "uri":"kafka-pd-200720001.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"6" + }, + { + "desc":"This section describes the notes and constraints on DMS.", + "product_code":"dms", + "title":"Notes and Constraints", + "uri":"kafka-pd-190605003.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"7" + }, + { + "desc":"Cloud Trace Service (CTS)CTS generates traces to provide you with a history of operations performed on cloud service resources. The traces include operation requests sent", + "product_code":"dms", + "title":"Related Services", + "uri":"kafka-pd-190605002.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"8" + }, + { + "desc":"DMS for Kafka of the cloud service platform uses Kafka as the message engine. This chapter presents explanations of basic concepts of Kafka.A topic is a category for mess", + "product_code":"dms", + "title":"Basic Concepts", + "uri":"glossary-kafka.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"9" + }, + { + "desc":"This section provides recommendations on configuring common parameters for Kafka producers and consumers.", + "product_code":"dms", + "title":"Best Practices", + "uri":"dms-ug-001.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"10" + }, + { + "desc":"By default, there are two types of user permissions: user management and resource management.User management refers to the management of users, user groups, and user grou", + "product_code":"dms", + "title":"Permissions", + "uri":"dms-ug-190128001.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"11" + }, + { + "desc":"Before creating a Kafka instance, ensure the availability of resources, including a virtual private cloud (VPC), subnet, security group, and security group rules. Each Ka", + "product_code":"dms", + "title":"Preparing Required Resources", + "uri":"kafka-ug-180604012.html", + "doc_type":"usermanual", + "p_code":"", + "code":"12" + }, + { + "desc":"Kafka instances are physically isolated and exclusively occupied by each tenant. You can customize the computing capabilities and storage space of an instance based on se", + "product_code":"dms", + "title":"Creating an Instance", + "uri":"dms-ug-180604013.html", + "doc_type":"usermanual", + "p_code":"", + "code":"13" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Accessing a Kafka Instance", + "uri":"kafka-ug190605003.html", + "doc_type":"usermanual", + "p_code":"", + "code":"14" + }, + { + "desc":"This section describes how to use an open-source Kafka client to access a Kafka instance if SASL access is not enabled for the instance. There are two scenarios. For cros", + "product_code":"dms", + "title":"Accessing a Kafka Instance Without SASL", + "uri":"kafka-ug-180604020.html", + "doc_type":"usermanual", + "p_code":"14", + "code":"15" + }, + { + "desc":"If you enable SASL_SSL when creating an instance, data will be encrypted before transmission for enhanced security.For security purposes, TLS_ECDHE_RSA_WITH_AES_128_GCM_S", + "product_code":"dms", + "title":"Accessing a Kafka Instance with SASL", + "uri":"kafka-ug-180801001.html", + "doc_type":"usermanual", + "p_code":"14", + "code":"16" + }, + { + "desc":"VPCs are logically isolated from each other. If a Kafka instance and a Kafka client are in different VPCs within a region, they cannot communicate with each other. In thi", + "product_code":"dms", + "title":"Cross-VPC Access to a Kafka Instance", + "uri":"kafka-ug-0001.html", + "doc_type":"usermanual", + "p_code":"14", + "code":"17" + }, + { + "desc":"You can use destination NAT (DNAT) to access a Kafka instance so that the instance can provide services on the public network through port mapping.You have created EIPs. ", + "product_code":"dms", + "title":"Using DNAT to Access a Kafka Instance", + "uri":"kafka-dnat.html", + "doc_type":"usermanual", + "p_code":"14", + "code":"18" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Managing Instances", + "uri":"kafka-ug-180604011.html", + "doc_type":"usermanual", + "p_code":"", + "code":"19" + }, + { + "desc":"View detailed information about a Kafka instance on the DMS console, for example, the IP addresses and port numbers for accessing the instance.Select the region where you", + "product_code":"dms", + "title":"Viewing an Instance", + "uri":"kafka-ug-180604014.html", + "doc_type":"usermanual", + "p_code":"19", + "code":"20" + }, + { + "desc":"Restart one or more Kafka instances at a time on the DMS console.When a Kafka instance is being restarted, message retrieval and creation requests of clients will be reje", + "product_code":"dms", + "title":"Restarting an Instance", + "uri":"kafka-ug-180604015.html", + "doc_type":"usermanual", + "p_code":"19", + "code":"21" + }, + { + "desc":"On the DMS console, you can delete one or more Kafka instances that have been created or failed to be created.Deleting a Kafka instance will delete the data in the instan", + "product_code":"dms", + "title":"Deleting an Instance", + "uri":"kafka-ug-180604016.html", + "doc_type":"usermanual", + "p_code":"19", + "code":"22" + }, + { + "desc":"After creating a Kafka instance, you can modify some parameters of the instance based on service requirements, including the instance name, description, security group, a", + "product_code":"dms", + "title":"Modifying the Information About an Instance", + "uri":"kafka-ug-180604017.html", + "doc_type":"usermanual", + "p_code":"19", + "code":"23" + }, + { + "desc":"To access a Kafka instance over a public network, enable public access and configure EIPs for the instance.If you no longer need public access to the instance, you can di", + "product_code":"dms", + "title":"Configuring Public Access", + "uri":"kafka-ug-0319001.html", + "doc_type":"usermanual", + "p_code":"19", + "code":"24" + }, + { + "desc":"You can reset the SASL_SSL password for accessing a Kafka instance by resetting Kafka password if you forget it.You can reset the Kafka password only if Kafka SASL_SSL ha", + "product_code":"dms", + "title":"Resetting Kafka Password", + "uri":"kafka-ug-180718001.html", + "doc_type":"usermanual", + "p_code":"19", + "code":"25" + }, + { + "desc":"Tags facilitate Kafka instance identification and management.You can add tags to a Kafka instance when creating the instance or add tags on the Tags tab page of the creat", + "product_code":"dms", + "title":"Managing Instance Tags", + "uri":"TagManagement.html", + "doc_type":"usermanual", + "p_code":"19", + "code":"26" + }, + { + "desc":"After you initiate certain instance operations such as configuring public access and modifying the capacity threshold policy, a background task will start for each operat", + "product_code":"dms", + "title":"Viewing Background Tasks", + "uri":"kafka-ug-200119002.html", + "doc_type":"usermanual", + "p_code":"19", + "code":"27" + }, + { + "desc":"On the Kafka console, you can view the disk usage of each broker.Select the region where your Kafka instance is located.You can query topics that use the most disk space ", + "product_code":"dms", + "title":"Viewing Disk Usage", + "uri":"kafka-ug-0004.html", + "doc_type":"usermanual", + "p_code":"19", + "code":"28" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Managing Topics", + "uri":"kafka-ug-0720001.html", + "doc_type":"usermanual", + "p_code":"", + "code":"29" + }, + { + "desc":"A topic is a stream of messages. If automatic topic creation is not enabled during Kafka instance creation, you need to manually create topics for creating and retrieving", + "product_code":"dms", + "title":"Creating a Topic", + "uri":"dms-ug-180604018.html", + "doc_type":"usermanual", + "p_code":"29", + "code":"30" + }, + { + "desc":"Delete a topic using either of the following methods:By using the consoleBy using Kafka CLIA Kafka instance has been created, and a topic has been created in this instanc", + "product_code":"dms", + "title":"Deleting a Topic", + "uri":"kafka-ug-180604019.html", + "doc_type":"usermanual", + "p_code":"29", + "code":"31" + }, + { + "desc":"Aging time is a period that messages in the topic are retained for. Consumers must retrieve messages before this period ends. Otherwise, the messages will be deleted and ", + "product_code":"dms", + "title":"Modifying Topic Aging Time", + "uri":"kafka-ug-200506001.html", + "doc_type":"usermanual", + "p_code":"29", + "code":"32" + }, + { + "desc":"After creating a topic, you can increase the number of partitions based on service requirements.Changing the number of partitions does not affect services.Methods for cha", + "product_code":"dms", + "title":"Changing Partition Quantity", + "uri":"kafka-ug-0006.html", + "doc_type":"usermanual", + "p_code":"29", + "code":"33" + }, + { + "desc":"Synchronous replication: A message is returned to the client only after the message creation request has been received and the message has been acknowledged by all replic", + "product_code":"dms", + "title":"Modifying Synchronous Replication and Flushing Settings", + "uri":"kafka_ug_0022.html", + "doc_type":"usermanual", + "p_code":"29", + "code":"34" + }, + { + "desc":"On the console, view sample code for creating and retrieving messages in Java, Go, and Python.Select the region where your Kafka instance is located.View sample code for ", + "product_code":"dms", + "title":"Viewing Sample Code", + "uri":"kafka_ug_0024.html", + "doc_type":"usermanual", + "p_code":"29", + "code":"35" + }, + { + "desc":"Export topics on the console. Batch export is supported.A topic has been created.Select the region where your Kafka instance is located.The topic list contains the follow", + "product_code":"dms", + "title":"Exporting Topics", + "uri":"kafka_ug_0027.html", + "doc_type":"usermanual", + "p_code":"29", + "code":"36" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Managing Messages", + "uri":"kafka-ug-0720002.html", + "doc_type":"usermanual", + "p_code":"", + "code":"37" + }, + { + "desc":"You can view the offset of different partitions, the message size, creation time, and body of messages in topics.Select the region where your Kafka instance is located.If", + "product_code":"dms", + "title":"Querying Messages", + "uri":"kafka-ug-190904001.html", + "doc_type":"usermanual", + "p_code":"37", + "code":"38" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Managing Users", + "uri":"kafka-ug-0009.html", + "doc_type":"usermanual", + "p_code":"", + "code":"39" + }, + { + "desc":"DMS supports ACL permission management for topics. You can differentiate the operations that different users are allowed to perform on a topic by granting the users diffe", + "product_code":"dms", + "title":"Creating a SASL_SSL User", + "uri":"kafka-ug-0003.html", + "doc_type":"usermanual", + "p_code":"39", + "code":"40" + }, + { + "desc":"DMS supports ACL permission management for topics. You can differentiate the operations that different users are allowed to perform on a topic by granting the users diffe", + "product_code":"dms", + "title":"Granting Permissions to a SASL_SSL User", + "uri":"kafka-ug-0002.html", + "doc_type":"usermanual", + "p_code":"39", + "code":"41" + }, + { + "desc":"If you forget the password of a SASL_SSL user created on the Users tab page, you can reset the password and use the new password to connect to the Kafka instance.If you f", + "product_code":"dms", + "title":"Resetting the SASL_SSL Password", + "uri":"kafka_ug_0025.html", + "doc_type":"usermanual", + "p_code":"39", + "code":"42" + }, + { + "desc":"This section describes how to delete a SASL_SSL user.Select the region where your Kafka instance is located.On the Users tab page, click Delete in the row that contains t", + "product_code":"dms", + "title":"Deleting a SASL_SSL User", + "uri":"kafka_ug_0026.html", + "doc_type":"usermanual", + "p_code":"39", + "code":"43" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Managing Consumer Groups", + "uri":"kafka-ug-0011.html", + "doc_type":"usermanual", + "p_code":"", + "code":"44" + }, + { + "desc":"View the consumer group list, consumer list, and consumer offsets.The consumer list can be viewed only when consumers in a consumer group are connected to the Kafka insta", + "product_code":"dms", + "title":"Querying Consumer Group Details", + "uri":"kafka_ug_0021.html", + "doc_type":"usermanual", + "p_code":"44", + "code":"45" + }, + { + "desc":"You can delete a consumer group using either of the following methods:Method 1: Delete a consumer group on the console.Method 2: Use Kafka CLI to delete a consumer group.", + "product_code":"dms", + "title":"Deleting a Consumer Group", + "uri":"kafka-ug-0012.html", + "doc_type":"usermanual", + "p_code":"44", + "code":"46" + }, + { + "desc":"Resetting the consumer offset is to change the retrieval position of a consumer.Messages may be retrieved more than once after the offset is reset. Exercise caution when ", + "product_code":"dms", + "title":"Resetting the Consumer Offset", + "uri":"kafka-ug-0014.html", + "doc_type":"usermanual", + "p_code":"44", + "code":"47" + }, + { + "desc":"View consumer connection addresses on the DMS console.The connection address of a consumer can be viewed only when the consumer is connected to a Kafka instance.Select th", + "product_code":"dms", + "title":"Viewing Consumer Connection Addresses", + "uri":"kafka-ug-0015.html", + "doc_type":"usermanual", + "p_code":"44", + "code":"48" + }, + { + "desc":"Your Kafka instances, topics, and consumers come with default configuration parameter settings. You can modify common parameters on the DMS console. For details about par", + "product_code":"dms", + "title":"Modifying Kafka Parameters", + "uri":"kafka-ug-0007.html", + "doc_type":"usermanual", + "p_code":"", + "code":"49" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Monitoring", + "uri":"kafka-ug-180413001.html", + "doc_type":"usermanual", + "p_code":"", + "code":"50" + }, + { + "desc":"Cloud Eye monitors Kafka instance metrics in real time. You can view these metrics on the Cloud Eye console.At least one Kafka instance has been created. The instance has", + "product_code":"dms", + "title":"Viewing Metrics", + "uri":"kafka-ug-190605001.html", + "doc_type":"usermanual", + "p_code":"50", + "code":"51" + }, + { + "desc":"This section describes DMS metrics reported to Cloud Eye as well as their namespace and dimensions. You can use the Cloud Eye console to query the Kafka metrics and alarm", + "product_code":"dms", + "title":"Kafka Metrics", + "uri":"dms-ug-180413002.html", + "doc_type":"usermanual", + "p_code":"50", + "code":"52" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Auditing", + "uri":"kafka-ug-180418001.html", + "doc_type":"usermanual", + "p_code":"", + "code":"53" + }, + { + "desc":"With Cloud Trace Service (CTS), you can record operations associated with DMS for later query, audit, and backtrack operations.", + "product_code":"dms", + "title":"Operations Logged by CTS", + "uri":"kafka-ug-180418002.html", + "doc_type":"usermanual", + "p_code":"53", + "code":"54" + }, + { + "desc":"This section describes how to view operation records of the last 7 days on the CTS console.Select the region where your Kafka instance is located.Trace Source: Select DMS", + "product_code":"dms", + "title":"Viewing Audit Logs", + "uri":"kafka-ug-180418003.html", + "doc_type":"usermanual", + "p_code":"53", + "code":"55" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"FAQs", + "uri":"kafka-ug-0723004.html", + "doc_type":"usermanual", + "p_code":"", + "code":"56" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Instances", + "uri":"kafka-faq-191030002.html", + "doc_type":"usermanual", + "p_code":"56", + "code":"57" + }, + { + "desc":"To improve the reliability of a Kafka instance, you are advised to select three AZs or more when creating the instance. You cannot select two AZs.Each Kafka instance cont", + "product_code":"dms", + "title":"Why Can't I Select Two AZs?", + "uri":"kafka-faq-200426002.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"58" + }, + { + "desc":"This may be because you do not have the Server Administrator and VPC Administrator permissions. For details about how to add permissions to a user group, see \"User and Us", + "product_code":"dms", + "title":"Why Can't I View the Subnet and Security Group Information When Creating a DMS Instance?", + "uri":"kafka-faq-200426003.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"59" + }, + { + "desc":"The storage space is the space for storing messages (including messages in replicas), logs and metadata. When specifying storage space, specify the disk type and disk siz", + "product_code":"dms", + "title":"How Do I Select Storage Space for a Kafka Instance?", + "uri":"kafka-faq-200426005.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"60" + }, + { + "desc":"High I/O: The average latency is 6 to 10 ms, and the maximum bandwidth is 120 MB/s (read + write).Ultra-high I/O: The average latency is 1 to 3 ms, and the maximum bandwi", + "product_code":"dms", + "title":"How Do I Choose Between High I/O and Ultra-high I/O?", + "uri":"kafka-faq-200426006.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"61" + }, + { + "desc":"The following policies are supported:Stop productionWhen the memory usage reaches the disk capacity threshold (95%), new messages will no longer be created, but existing ", + "product_code":"dms", + "title":"Which Capacity Threshold Policy Should I Use?", + "uri":"kafka-faq-200426007.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"62" + }, + { + "desc":"Kafka v1.1.0, v2.3.0, and v2.7.", + "product_code":"dms", + "title":"Which Kafka Versions Are Supported?", + "uri":"kafka-faq-200426008.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"63" + }, + { + "desc":"Kafka instances are managed using ZooKeeper. Opening ZooKeeper may cause misoperations and service losses. ZooKeeper is used only within Kafka clusters and does not provi", + "product_code":"dms", + "title":"What Is the ZooKeeper Address of a Kafka Instance?", + "uri":"kafka-faq-200426009.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"64" + }, + { + "desc":"Yes. A Kafka instance is a cluster that consists of three or more brokers.", + "product_code":"dms", + "title":"Are Kafka Instances in Cluster Mode?", + "uri":"kafka-faq-200426010.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"65" + }, + { + "desc":"No. You must access a Kafka instance through one of the following ports:Accessing a Kafka instance without SASL:The port varies with the access mode:Intra-VPC access: por", + "product_code":"dms", + "title":"Can I Modify the Connection Address for Accessing a Kafka Instance?", + "uri":"kafka-faq-200426011.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"66" + }, + { + "desc":"The certificates are valid for more than 15 years. You do not need to worry about certificate expiration. The certificates are used for one-way authentication when enabli", + "product_code":"dms", + "title":"How Long Are Kafka SSL Certificates Valid for?", + "uri":"kafka-faq-200426012.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"67" + }, + { + "desc":"Unfortunately, you cannot synchronize two Kafka instances in real time. To migrate services from one instance to another, create messages to both instances. After all mes", + "product_code":"dms", + "title":"How Do I Synchronize Data from One Kafka Instance to Another?", + "uri":"kafka-faq-200426013.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"68" + }, + { + "desc":"The SASL_SSL setting cannot be changed once the instance has been created. Be careful when configuring this setting during instance creation. If you need to change the se", + "product_code":"dms", + "title":"How Do I Change the SASL_SSL Setting of a Kafka Instance?", + "uri":"kafka-faq-200426014.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"69" + }, + { + "desc":"Kafka brokers and ZooKeeper are deployed on the same VM.", + "product_code":"dms", + "title":"Are Kafka Brokers and ZooKeeper Deployed on the Same VM or on Different VMs?", + "uri":"kafka-faq-0015.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"70" + }, + { + "desc":"For security purposes, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 is supported.", + "product_code":"dms", + "title":"Which Cipher Suites Are Supported by Kafka?", + "uri":"kafka-faq-0020.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"71" + }, + { + "desc":"No. The AZ configuration cannot be changed once the instance is created. To use multiple AZs, create another instance.", + "product_code":"dms", + "title":"Can I Change an Instance from Single-AZ Deployment to Multi-AZ Deployment?", + "uri":"kafka-faq-0023.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"72" + }, + { + "desc":"DMS for Kafka supports cross-AZ disaster recovery. If you select multiple AZs when creating an instance, cross-AZ disaster recovery will be available.You can view the AZs", + "product_code":"dms", + "title":"Does DMS for Kafka Support Cross-AZ Disaster Recovery? Where Can I View the AZs Configured for an Existing Instance?", + "uri":"kafka-faq-0025.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"73" + }, + { + "desc":"Yes.", + "product_code":"dms", + "title":"Do Kafka Instances Support Disk Encryption?", + "uri":"kafka-faq-0030.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"74" + }, + { + "desc":"No. Once an instance is created, its VPC and subnet cannot be changed.", + "product_code":"dms", + "title":"Can I Change the VPC and Subnet After a Kafka Instance Is Created?", + "uri":"kafka-faq-0036.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"75" + }, + { + "desc":"You can find Kafka Streams use cases on the official Kafka website.", + "product_code":"dms", + "title":"Where Can I Find Kafka Streams Use Cases?", + "uri":"kafka-faq-0037.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"76" + }, + { + "desc":"No. Kafka instances cannot be upgraded once they are created. To use a higher Kafka version, create another Kafka instance.", + "product_code":"dms", + "title":"Can I Upgrade Kafka Instances?", + "uri":"kafka-faq-0040.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"77" + }, + { + "desc":"On the DMS console, click the name of the target Kafka instance. Disable Public Access in the Connection section on the Basic Information tab page, and then enable it aga", + "product_code":"dms", + "title":"How Do I Bind an EIP Again?", + "uri":"kafka_faq_0046.html", + "doc_type":"usermanual", + "p_code":"57", + "code":"78" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Connections", + "uri":"kafka-faq-191030001.html", + "doc_type":"usermanual", + "p_code":"56", + "code":"79" + }, + { + "desc":"This section describes how to troubleshoot Kafka connection problems.If the connection to a Kafka instance is abnormal, perform the following operations to troubleshoot t", + "product_code":"dms", + "title":"Troubleshooting Kafka Connection Exceptions", + "uri":"kafka-faq-0604001.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"80" + }, + { + "desc":"Kafka instances can be accessed within a VPC, across VPCs, through DNAT, or over public networks. Before accessing a Kafka instance, configure a security group.If they us", + "product_code":"dms", + "title":"How Do I Select and Configure a Security Group?", + "uri":"kafka-faq-180604024.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"81" + }, + { + "desc":"Yes. For details, see the instance access instructions.", + "product_code":"dms", + "title":"Can I Access a Kafka Instance Over a Public Network?", + "uri":"kafka-faq-200426015.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"82" + }, + { + "desc":"The number of connection addresses of a Kafka instance is the same as the number of brokers of the instance.", + "product_code":"dms", + "title":"How Many Connection Addresses Does a Kafka Instance Have by Default?", + "uri":"kafka-faq-200426016.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"83" + }, + { + "desc":"Yes. You can access a Kafka instance across regions over a public network or by using direct connections.", + "product_code":"dms", + "title":"Do Kafka Instances Support Cross-Region Access?", + "uri":"kafka-faq-200426017.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"84" + }, + { + "desc":"Yes. You can use one of the following methods to access a Kafka instance across VPCs:Establish a VPC peering connection to allow two VPCs to communicate with each other. ", + "product_code":"dms", + "title":"Do Kafka Instances Support Cross-VPC Access?", + "uri":"kafka-faq-200426019.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"85" + }, + { + "desc":"Yes.If the client and the instance are in the same VPC, cross-subnet access is supported.", + "product_code":"dms", + "title":"Do Kafka Instances Support Cross-Subnet Access?", + "uri":"kafka-faq-200426020.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"86" + }, + { + "desc":"No, Kerberos authentication is not supported. Kafka supports client authentication with SASL and API calling authentication using tokens and AK/SK.To access an instance i", + "product_code":"dms", + "title":"Does DMS for Kafka Support Authentication with Kerberos?", + "uri":"kafka-faq-200426023.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"87" + }, + { + "desc":"Yes. No password is required for accessing a Kafka instance with SASL disabled. For details, see Accessing a Kafka Instance Without SASL.", + "product_code":"dms", + "title":"Does DMS for Kafka Support Password-Free Access?", + "uri":"kafka-faq-200708002.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"88" + }, + { + "desc":"Kafka instances are fully compatible with open-source clients. You can obtain clients in other programming languages and access your instance as instructed by the officia", + "product_code":"dms", + "title":"Obtaining Kafka Clients", + "uri":"kafka-connect-other.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"89" + }, + { + "desc":"Click the name of your Kafka instance. In the Connection section on the Basic Information tab page, view Instance Address (Public Network).For details about how to connec", + "product_code":"dms", + "title":"How Do I Obtain the Public Access Address After Public Access Is Enabled?", + "uri":"kafka-faq-0001.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"90" + }, + { + "desc":"No.", + "product_code":"dms", + "title":"Does DMS for Kafka Support Authentication on Clients by the Server?", + "uri":"kafka-faq-0026.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"91" + }, + { + "desc":"No. You can only use JKS certificates for connecting to instances in Java.", + "product_code":"dms", + "title":"Can I Use PEM SSL Truststore When Connecting to a Kafka Instance with SASL_SSL Enabled?", + "uri":"kafka-faq-0027.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"92" + }, + { + "desc":"JKS certificates are used for connecting to instances in Java and CRT certificates are used for connecting to instances in Python.", + "product_code":"dms", + "title":"What Are the Differences Between JKS and CRT Certificates?", + "uri":"kafka-faq-0028.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"93" + }, + { + "desc":"TLS 1.2.", + "product_code":"dms", + "title":"Which TLS Version Does DMS for Kafka Support?", + "uri":"kafka-faq-0029.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"94" + }, + { + "desc":"Yes. The maximum allowed number of client connections varies by instance specifications.If the bandwidth is 100 MB/s, a maximum of 3000 client connections are allowed.If ", + "product_code":"dms", + "title":"Is There a Limit on the Number of Client Connections to a Kafka Instance?", + "uri":"kafka-faq-0033.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"95" + }, + { + "desc":"Each Kafka broker allows a maximum of 1000 connections from each IP address by default. Excess connections will be rejected. You can change the limit by referring to Modi", + "product_code":"dms", + "title":"How Many Connections Are Allowed from Each IP Address?", + "uri":"kafka-faq-0034.html", + "doc_type":"usermanual", + "p_code":"79", + "code":"96" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Topics and Partitions", + "uri":"kafka-faq-191030003.html", + "doc_type":"usermanual", + "p_code":"56", + "code":"97" + }, + { + "desc":"The number of topics is related to the total number of topic partitions and the number of partitions in each topic. There is an upper limit on the aggregate number of par", + "product_code":"dms", + "title":"Is There a Limit on the Number of Topics in a Kafka Instance?", + "uri":"kafka-faq-200426024.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"98" + }, + { + "desc":"Kafka manages messages by partition. If there are too many partitions, message creation, storage, and retrieval will be fragmented, affecting the performance and stabilit", + "product_code":"dms", + "title":"Why Is Partition Quantity Limited?", + "uri":"kafka-faq-200426025.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"99" + }, + { + "desc":"No. If you want to use fewer partitions, delete the corresponding topic, create another one, and specify the desired number of partitions.", + "product_code":"dms", + "title":"Can I Change the Partition Quantity?", + "uri":"kafka-faq-200426101.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"100" + }, + { + "desc":"Possible cause: The aggregate number of partitions of created topics has reached the upper limit. The maximum number of partitions varies with instance specifications. Fo", + "product_code":"dms", + "title":"Why Do I Fail to Create Topics?", + "uri":"kafka-faq-200426026.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"101" + }, + { + "desc":"Automatic topic creation is supported, but batch topic import is not supported. You can only export topics in batches.Enable automatic topic creation using one of the fol", + "product_code":"dms", + "title":"Do Kafka Instances Support Batch Importing Topics or Automatic Topic Creation?", + "uri":"kafka-faq-200426027.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"102" + }, + { + "desc":"This may be because automatic topic creation has been enabled and a consumer is connecting to the topic. If no existing topics are available for message creation, new top", + "product_code":"dms", + "title":"Why Do Deleted Topics Still Exist?", + "uri":"kafka-faq-200426028.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"103" + }, + { + "desc":"Yes. Use either of the following methods to check the disk space used by a topic:Click next to the Kafka instance name to go to the Cloud Eye console. On the Queues tab ", + "product_code":"dms", + "title":"Can I View the Disk Space Used by a Topic?", + "uri":"kafka-faq-200426030.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"104" + }, + { + "desc":"If you have enabled SASL_SSL for your Kafka instance, you can configure ACL permissions for your topics. On the Topics tab page of the Kafka console, click Grant User Per", + "product_code":"dms", + "title":"Can I Add ACL Permissions for Topics?", + "uri":"kafka-faq-200426032.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"105" + }, + { + "desc":"Messages are not deleted immediately after being retrieved. They are deleted only when the aging time expires.You can shorten the aging time.", + "product_code":"dms", + "title":"What Should I Do If Kafka Storage Space Is Used Up Because Retrieved Messages Are Not Deleted?", + "uri":"kafka-faq-0003.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"106" + }, + { + "desc":"Yes. A Kafka instance will be restarted if you enable or disable automatic topic creation for it.", + "product_code":"dms", + "title":"Will a Kafka Instance Be Restarted After Its Automatic Topic Creation Setting Is Modified?", + "uri":"kafka-faq-0010.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"107" + }, + { + "desc":"On the Kafka console, click the name of your instance.In the Instance Information section of the Basic Information tab page, click next to Automatic Topic Creation to di", + "product_code":"dms", + "title":"How Do I Disable Automatic Topic Creation?", + "uri":"kafka-faq-0014.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"108" + }, + { + "desc":"Yes, just simply unsubscribe from it on the Kafka client.", + "product_code":"dms", + "title":"Can I Delete Unnecessary Topics in a Consumer Group?", + "uri":"kafka-faq-0031.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"109" + }, + { + "desc":"Symptom: Different consumers in a consumer group have different topic permissions. When a consumer attempts to retrieve messages from a topic, the error message \"Not auth", + "product_code":"dms", + "title":"What Can I Do If a Consumer Fails to Retrieve Messages from a Topic Due to Insufficient Permissions?", + "uri":"kafka-faq-0038.html", + "doc_type":"usermanual", + "p_code":"97", + "code":"110" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Consumer Groups", + "uri":"kafka-faq-200423001.html", + "doc_type":"usermanual", + "p_code":"56", + "code":"111" + }, + { + "desc":"No. They are generated automatically when you use the instance.For details about creating and retrieving messages after connecting to a Kafka instance, see Accessing a Ka", + "product_code":"dms", + "title":"Do I Need to Create Consumer Groups, Producers, and Consumers for Kafka Instances?", + "uri":"kafka-faq-200426033.html", + "doc_type":"usermanual", + "p_code":"111", + "code":"112" + }, + { + "desc":"No. You can directly delete the consumer group.", + "product_code":"dms", + "title":"Do I Need to Unsubscribe from a Topic Before Deleting a Consumer Group?", + "uri":"kafka-faq-0032.html", + "doc_type":"usermanual", + "p_code":"111", + "code":"113" + }, + { + "desc":"Yes.Kafka uses the offsets.retention.minutes parameter to control how long to keep offsets for a consumer group. If offsets are not committed within this period, they wil", + "product_code":"dms", + "title":"Will a Consumer Group Without Active Consumers Be Automatically Deleted in 14 Days?", + "uri":"kafka-faq-0043.html", + "doc_type":"usermanual", + "p_code":"111", + "code":"114" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Messages", + "uri":"kafka-faq-190416001.html", + "doc_type":"usermanual", + "p_code":"56", + "code":"115" + }, + { + "desc":"10 MB.", + "product_code":"dms", + "title":"What Is the Maximum Size of a Message that Can be Created?", + "uri":"kafka-faq-200426035.html", + "doc_type":"usermanual", + "p_code":"115", + "code":"116" + }, + { + "desc":"Rebalancing is a process where partitions of topics are re-allocated for a consumer group.In normal cases, rebalancing occurs inevitably when a consumer is added to or re", + "product_code":"dms", + "title":"Why Does Message Poll Often Fail During Rebalancing?", + "uri":"kafka-faq-200426036.html", + "doc_type":"usermanual", + "p_code":"115", + "code":"117" + }, + { + "desc":"Possible cause 1: The message has been aged.Solution: Change the aging time.Solution: Change the aging time.Possible cause 2: The createTime timestamp of the message is i", + "product_code":"dms", + "title":"Why Can't I Query Messages on the Console?", + "uri":"kafka-faq-200426037.html", + "doc_type":"usermanual", + "p_code":"115", + "code":"118" + }, + { + "desc":"If the aging time has been set for a topic, the value of the log.retention.hours parameter does not take effect for the topic. The value of the log.retention.hours parame", + "product_code":"dms", + "title":"Why Do Messages Still Exist After the Retention Period Elapses?", + "uri":"kafka-faq-200708001.html", + "doc_type":"usermanual", + "p_code":"115", + "code":"119" + }, + { + "desc":"No.", + "product_code":"dms", + "title":"Do Kafka Instances Support Delayed Message Delivery?", + "uri":"kafka-faq-0018.html", + "doc_type":"usermanual", + "p_code":"115", + "code":"120" + }, + { + "desc":"View the number of accumulated messages using any of the following methods:On the Consumer Groups page of an instance, click the name of the consumer group whose accumula", + "product_code":"dms", + "title":"How Do I View the Number of Accumulated Messages?", + "uri":"kafka-faq-0041.html", + "doc_type":"usermanual", + "p_code":"115", + "code":"121" + }, + { + "desc":"The message creation time is specified by CreateTime when a producer creates messages. If this parameter is not set during message creation, the message creation time is ", + "product_code":"dms", + "title":"Why Is the Message Creation Time Displayed as Year 1970?", + "uri":"kafka-faq-0045.html", + "doc_type":"usermanual", + "p_code":"115", + "code":"122" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Monitoring & Alarm", + "uri":"kafka-faq-191030004.html", + "doc_type":"usermanual", + "p_code":"56", + "code":"123" + }, + { + "desc":"The possible causes are as follows:The topic name starts with a special character, such as an underscore (_) or a number sign (#).The consumer group name starts with a sp", + "product_code":"dms", + "title":"Why Can't I View the Monitoring Data?", + "uri":"kafka-faq-200426041.html", + "doc_type":"usermanual", + "p_code":"123", + "code":"124" + }, + { + "desc":"Symptom: The monitoring data shows that there are 810 million accumulated messages. However, the Kafka console shows that there are 100 million messages in all six topics", + "product_code":"dms", + "title":"Why Is the Monitored Number of Accumulated Messages Inconsistent with the Message Quantity Displayed on the Kafka Console?", + "uri":"kafka-faq-0007.html", + "doc_type":"usermanual", + "p_code":"123", + "code":"125" + }, + { + "desc":"The monitoring data is reported every minute. The reported data will be displayed on the monitoring page after being sorted. This process takes less than 20 minutes. Afte", + "product_code":"dms", + "title":"Why Is a Consumer Group Still on the Monitoring Page After Being Deleted?", + "uri":"kafka-faq-0022.html", + "doc_type":"usermanual", + "p_code":"123", + "code":"126" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dms", + "title":"Change History", + "uri":"kafka-ug-00001.html", + "doc_type":"usermanual", + "p_code":"", + "code":"127" + }, + { + "desc":"See Glossary.", + "product_code":"dms", + "title":"Glossary", + "uri":"dms-ug-0312114.html", + "doc_type":"usermanual", + "p_code":"", + "code":"128" + } +] \ No newline at end of file diff --git a/docs/dms/umn/Kafka-specification.html b/docs/dms/umn/Kafka-specification.html new file mode 100644 index 000000000..2338555bd --- /dev/null +++ b/docs/dms/umn/Kafka-specification.html @@ -0,0 +1,107 @@ + + +
Kafka instances are compatible with open-source Kafka 1.1.0, 2.3.0, and 2.7. The instance specifications are classified based on bandwidth, namely, 100 MB/s, 300 MB/s, 600 MB/s, and 1200 MB/s.
+ +Bandwidth + |
+I/O Type + |
+TPS (High-Throughput) + |
+TPS (Synchronous Replication) + |
+Maximum Partitions + |
+
|---|---|---|---|---|
100 MB/s + |
+High I/O + |
+100,000 + |
+60,000 + |
+300 + |
+
Ultra-high I/O + |
+100,000 + |
+80,000 + |
+300 + |
+|
300 MB/s + |
+High I/O + |
+300,000 + |
+150,000 + |
+900 + |
+
Ultra-high I/O + |
+300,000 + |
+200,000 + |
+900 + |
+|
600 MB/s + |
+Ultra-high I/O + |
+600,000 + |
+300,000 + |
+1800 + |
+
1200 MB/s + |
+Ultra-high I/O + |
+1,200,000 + |
+400,000 + |
+1800 + |
+
For Kafka instances, the number of transactions per second (TPS) is the maximum number of messages that can be written per second. The preceding TPS is calculated with each message being 1 KB.
+The bandwidth of a Kafka instance refers to the maximum read or write bandwidth. You are advised to select a bandwidth 30% higher than what is required.
+Recommended for up to 3000 client connections, 60 consumer groups, and 70 MB/s service traffic.
+Recommended for up to 10,000 client connections, 300 consumer groups, and 210 MB/s service traffic.
+Recommended for up to 20,000 client connections, 600 consumer groups, and 420 MB/s service traffic.
+Recommended for up to 20,000 client connections, 600 consumer groups, and 840 MB/s service traffic.
+Kafka instances support storage with 1 to 3 replicas. The storage space is consumed by all replicas. When creating an instance, specify its storage space based on the expected service message size and the number of replicas.
+For example, if the estimated message size is 100 GB, the disk capacity must be at least: 100 GB x Number of replicas + 100 GB (reserved space).
+There are limits on the topic quantity and the aggregate number of partitions in the topics. When the partition quantity limit is reached, you can no longer create topics.
+The number of topics is related to the maximum number of partitions allowed and the specified number of partitions in each topic (see Table 1).
+The maximum number of partitions for a 100 MB/s instance is 300.
+Tags facilitate Kafka instance identification and management.
+You can add tags to a Kafka instance when creating the instance or add tags on the Tags tab page of the created instance. Up to 20 tags can be added to an instance. Tags can be modified and deleted.
+A tag consists of a tag key and a tag value. Table 1 lists the tag key and value requirements.
+ +Parameter + |
+Requirements + |
+
|---|---|
Tag key + |
+
|
+
Tag value + |
+
|
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+View the tags of the instance.
+If you have predefined tags, select a predefined pair of tag key and value, and click Add.
+
next to the tag to be edited, enter the original tag key and the new tag value, and click Add.Delete a tag using either of the following methods:
+
next to the tag to be deleted and click OK.This section provides recommendations on configuring common parameters for Kafka producers and consumers.
+ +Parameter + |
+Default Value + |
+Recommended Value + |
+Description + |
+
|---|---|---|---|
acks + |
+1 + |
+all or –1 (if high reliability mode is selected) +1 (if high throughput mode is selected) + |
+Number of acknowledgments the producer requires the server to return before considering a request complete. This controls the durability of records that are sent. Options: +0: The producer will not wait for any acknowledgment from the server at all. The record will be immediately added to the socket buffer and considered sent. No guarantee can be made that the server has received the record, and the retries configuration will not take effect (as the client generally does not know of any failures). The offset given back for each record will always be set to –1. +1: The leader will write the record to its local log but will respond without waiting until receiving full acknowledgement from all followers. If the leader fails immediately after acknowledging the record but before the followers have replicated it, the record will be lost. +all or –1: The leader will wait for the full set of replicas to acknowledge the record. This is the strongest available guarantee because the record will not be lost even if there is just one replica that works. min.insync.replicas specifies the minimum number of replicas that must acknowledge a write for the write to be considered successful. + |
+
retries + |
+0 + |
+Set as required. + |
+Number of times that the client resends a message. Setting this parameter to a value greater than zero will cause the client to resend any record that failed to be sent. +Note that this retry is no different than if the client resends the record upon receiving the error. Allowing retries will potentially change the ordering of records because if two batches are sent to the same partition, and the first fails and is retried but the second succeeds, then the records in the second batch may appear first. +You are advised to configure producers so that they can be able to retry in case of network disconnections. Set retries to 3 and the retry interval retry.backoff.ms to 1000. + |
+
request.timeout.ms + |
+30000 + |
+Set as required. + |
+Maximum amount of time (in ms) the client will wait for the response of a request. If the response is not received before the timeout elapses, the client will throw a timeout exception. +Setting this parameter to a large value, for example, 127000 (127s), can prevent records from failing to be sent in high-concurrency scenarios. + |
+
block.on.buffer.full + |
+TRUE + |
+TRUE + |
+Setting this parameter to TRUE indicates that when buffer memory is exhausted, the producer must stop receiving new message records or throw an exception. +By default, this parameter is set to TRUE. However, in some cases, non-blocking usage is desired and it is better to throw an exception immediately. Setting this parameter to FALSE will cause the producer to instead throw "BufferExhaustedException" when buffer memory is exhausted. + |
+
batch.size + |
+16384 + |
+262144 + |
+Default maximum number of bytes of messages that can be processed at a time. The producer will attempt to batch records together into fewer requests whenever multiple records are being sent to the same partition. This improves performance of both the client and the server. No attempt will be made to batch records larger than this size. +Requests sent to brokers will contain multiple batches, one for each partition with data available to be sent. +A smaller batch size will make batching less common and may reduce throughput (a batch size of zero will disable batching entirely). A larger batch size may use more memory as a buffer of the specified batch size will always be allocated in anticipation of additional records. + |
+
buffer.memory + |
+33554432 + |
+67108864 + |
+Total bytes of memory the producer can use to buffer records waiting to be sent to the server. If data is generated faster than it is sent to the broker, the producer blocks or throw a "block.on.buffer.full" exception. +This setting should correspond roughly to the total memory the producer will use, but is not a rigid bound since not all memory the producer uses is used for buffering. Some additional memory will be used for compression (if compression is enabled) as well as for maintaining in-flight requests. + |
+
Parameter + |
+Default Value + |
+Recommended Value + |
+Description + |
+
|---|---|---|---|
auto.commit.enable + |
+TRUE + |
+FALSE + |
+If this parameter is set to TRUE, the offset of messages already fetched by the consumer will be periodically committed to ZooKeeper. This committed offset will be used when the process fails as the position from which the new consumer will begin. +Constraints: If this parameter is set to FALSE, to avoid message loss, an offset must be committed to ZooKeeper after the messages are successfully consumed. + |
+
auto.offset.reset + |
+latest + |
+earliest + |
+Indicates what to do when there is no initial offset in ZooKeeper or if the current offset has been deleted. Options: +earliest: Automatically reset to the smallest offset. +latest: Automatically reset to the largest offset. +none: The system throws an exception to the consumer if no offset is available. +anything else: The system throws an exception to the consumer. + |
+
connections.max.idle.ms + |
+600000 + |
+30000 + |
+Timeout interval (in ms) for an idle connection. The server closes the idle connection after this period of time ends. Setting this parameter to 30000 can reduce the server response failures when the network condition is poor. + |
+
See Glossary.
+This section describes DMS metrics reported to Cloud Eye as well as their namespace and dimensions. You can use the Cloud Eye console to query the Kafka metrics and alarms.
+SYS.DMS
+Metric ID + |
+Metric Name + |
+Description + |
+Value Range + |
+Monitored Object + |
+Monitoring Period (Raw Data) + |
+
|---|---|---|---|---|---|
current_partitions + |
+Partitions + |
+Number of used partitions in the instance +Unit: count + |
+0~1800 + |
+Kafka instance + |
+1 minute + |
+
current_topics + |
+Topics + |
+Number of created topics in the instance +Unit: count + |
+0–600 + |
+Kafka instance + |
+1 minute + |
+
group_msgs + |
+Accumulated Messages + |
+Total number of accumulated messages in all consumer groups of the instance +Unit: count + |
+0–1,000,000,000 + |
+Kafka instance + |
+1 minute + |
+
Metric ID + |
+Metric Name + |
+Description + |
+Value Range + |
+Monitored Object + |
+Monitoring Period (Raw Data) + |
+
|---|---|---|---|---|---|
broker_data_size + |
+Message Size + |
+Total size of messages in the broker +Unit: byte, KB, MB, GB, TB or PB + |
+0–5,000,000,000,000 + |
+Kafka instance broker + |
+1 minute + |
+
broker_messages_in_rate + |
+Message Creation Rate + |
+Number of messages created per second +Unit: count/s + |
+0–500,000 + |
+Kafka instance broker + |
+1 minute + |
+
broker_bytes_out_rate + |
+Message Retrieval + |
+Number of bytes retrieved per second +Unit: byte/s, KB/s, MB/s, or GB/s + |
+0–500,000,000 + |
+Kafka instance broker + |
+1 minute + |
+
broker_bytes_in_rate + |
+Message Creation + |
+Number of bytes created per second +Unit: byte/s, KB/s, MB/s, or GB/s + |
+0–500,000,000 + |
+Kafka instance broker + |
+1 minute + |
+
broker_public_bytes_in_rate + |
+Public Inbound Traffic + |
+Inbound traffic over public networks per second +Unit: byte/s, KB/s, MB/s, or GB/s + NOTE:
+You can view this metric on the EIP console if public access has been enabled and EIPs have been assigned to the instance. + |
+0–500,000,000 + |
+Kafka instance broker + |
+1 minute + |
+
broker_public_bytes_out_rate + |
+Public Outbound Traffic + |
+Outbound traffic over public networks per second +Unit: byte/s, KB/s, MB/s, or GB/s + NOTE:
+You can view this metric on the EIP console if public access has been enabled and EIPs have been assigned to the instance. + |
+0–500,000,000 + |
+Kafka instance broker + |
+1 minute + |
+
broker_fetch_mean + |
+Average Message Retrieval Processing Duration + |
+Average time that the broker spends processing message retrieval requests +Unit: ms + |
+0–10,000 + |
+Kafka instance broker + |
+1 minute + |
+
broker_produce_mean + |
+Average Message Creation Processing Duration + |
+Average time that the broker spends processing message creation requests +Unit: ms + |
+0–10,000 + |
+Kafka instance broker + |
+1 minute + |
+
broker_cpu_core_load + |
+Average Load per CPU Core + |
+Average load of each CPU core of the Kafka VM +Unit: % + |
+0–20 + |
+Kafka instance broker + |
+1 minute + |
+
broker_disk_usage + |
+Disk Capacity Usage + |
+Disk usage of the Kafka VM +Unit: % + |
+0–100 + |
+Kafka instance broker + |
+1 minute + |
+
broker_memory_usage + |
+Memory Usage + |
+Memory usage of the Kafka VM +Unit: % + |
+0–100 + |
+Kafka instance broker + |
+1 minute + |
+
broker_heap_usage + |
+JVM Heap Memory Usage of Kafka + |
+Heap memory usage of the Kafka JVM +Unit: % + |
+0–100 + |
+Kafka instance broker + |
+1 minute + |
+
broker_alive + |
+Broker Alive + |
+Whether the Kafka broker is alive + |
+1: alive +0: not alive + |
+Kafka instance broker + |
+1 minute + |
+
broker_connections + |
+Connections + |
+Total number of TCP connections on the Kafka broker +Unit: count + |
+> 0 + |
+Kafka instance broker + |
+1 minute + |
+
broker_cpu_usage + |
+CPU Usage + |
+CPU usage of the Kafka VM +Unit: % + |
+0–100 + |
+Kafka instance broker + |
+1 minute + |
+
broker_disk_read_await + |
+Average Disk Read Time + |
+Average time for each disk I/O read in the monitoring period +Unit: ms + |
+> 0 + |
+Kafka instance broker + |
+1 minute + |
+
broker_disk_write_await + |
+Average Disk Write Time + |
+Average time for each disk I/O write in the monitoring period +Unit: ms + |
+> 0 + |
+Kafka instance broker + |
+1 minute + |
+
broker_total_bytes_in_rate + |
+Inbound Traffic + |
+Inbound traffic per second +Unit: byte/s + |
+> 0 + |
+Kafka instance broker + |
+1 minute + |
+
broker_total_bytes_out_rate + |
+Outbound Traffic + |
+Outbound traffic per second +Unit: byte/s + |
+> 0 + |
+Kafka instance broker + |
+1 minute + |
+
Metric ID + |
+Metric Name + |
+Description + |
+Value Range + |
+Monitored Object + |
+Monitoring Period (Raw Data) + |
+
|---|---|---|---|---|---|
topic_bytes_in_rate + |
+Message Creation + |
+Number of bytes created per second +Unit: byte/s, KB/s, MB/s, or GB/s + NOTE:
+This metric is available only when Scope is set to Basic monitoring on the Queues tab page. + |
+0–500,000,000 + |
+Topic in a Kafka instance + |
+1 minute + |
+
topic_bytes_out_rate + |
+Message Retrieval + |
+Number of bytes retrieved per second +Unit: byte/s, KB/s, MB/s, or GB/s + NOTE:
+This metric is available only when Scope is set to Basic monitoring on the Queues tab page. + |
+0–500,000,000 + |
+Topic in a Kafka instance + |
+1 minute + |
+
topic_data_size + |
+Message Size + |
+Total size of messages in the queue +Unit: byte, KB, MB, GB, TB or PB + NOTE:
+This metric is available only when Scope is set to Basic monitoring on the Queues tab page. + |
+0–5,000,000,000,000 + |
+Topic in a Kafka instance + |
+1 minute + |
+
topic_messages + |
+Total Messages + |
+Total number of messages in the queue +Unit: count + NOTE:
+This metric is available only when Scope is set to Basic monitoring on the Queues tab page. + |
+≥ 0 + |
+Topic in a Kafka instance + |
+1 minute + |
+
topic_messages_in_rate + |
+Message Creation Rate + |
+Number of messages created per second +Unit: count/s + NOTE:
+This metric is available only when Scope is set to Basic monitoring on the Queues tab page. + |
+0–500,000 + |
+Topic in a Kafka instance + |
+1 minute + |
+
partition_messages + |
+Partition Messages + |
+Total number of messages in the partition +Unit: count + NOTE:
+This metric is available only when Scope is set to Partition monitoring on the Queues tab page. + |
+≥ 0 + |
+Topic in a Kafka instance + |
+1 minute + |
+
produced_messages + |
+Created Messages + |
+Number of messages that have been created +Unit: count + NOTE:
+This metric is available only when Scope is set to Partition monitoring on the Queues tab page. + |
+≥ 0 + |
+Topic in a Kafka instance + |
+1 minute + |
+
Metric ID + |
+Metric Name + |
+Description + |
+Value Range + |
+Monitored Object + |
+Monitoring Period (Raw Data) + |
+
|---|---|---|---|---|---|
messages_consumed + |
+Retrieved Messages + |
+Number of messages that have been retrieved in the consumer group +Unit: count + NOTE:
+This metric is available only when Queue is set to a specified topic name and Monitoring Type is set to Partition monitoring on the By Consumer Group tab page. + |
+≥ 0 + |
+Consumer group of a Kafka instance + |
+1 minute + |
+
messages_remained + |
+Available Messages + |
+Number of messages that can be retrieved in the consumer group +Unit: count + NOTE:
+This metric is available only when Queue is set to a specified topic name and Monitoring Type is set to Partition monitoring on the By Consumer Group tab page. + |
+≥ 0 + |
+Consumer group of a Kafka instance + |
+1 minute + |
+
topic_messages_remained + |
+Topic Available Messages + |
+Number of remaining messages that can be retrieved from the specified topic in the consumer group +Unit: Count + NOTE:
+This metric is available only when Queue is set to a specified topic name and Monitoring Type is set to Basic monitoring on the By Consumer Group tab page. + |
+0 to 263–1 + |
+Consumer group of a Kafka instance + |
+1 minute + |
+
topic_messages_consumed + |
+Topic Retrieved Messages + |
+Number of messages that have been retrieved from the specified topic in the consumer group +Unit: Count + NOTE:
+This metric is available only when Queue is set to a specified topic name and Monitoring Type is set to Basic monitoring on the By Consumer Group tab page. + |
+0 to 263–1 + |
+Consumer group of a Kafka instance + |
+1 minute + |
+
consumer_messages_remained + |
+Consumer Available Messages + |
+Number of remaining messages that can be retrieved in the consumer group +Unit: Count + NOTE:
+This metric is available only when Queue is set to All queues on the By Consumer Group tab page. + |
+0 to 263–1 + |
+Consumer group of a Kafka instance + |
+1 minute + |
+
consumer_messages_consumed + |
+Consumer Retrieved Messages + |
+Number of messages that have been retrieved in the consumer group +Unit: Count + NOTE:
+This metric is available only when Queue is set to All queues on the By Consumer Group tab page. + |
+0 to 263–1 + |
+Consumer group of a Kafka instance + |
+1 minute + |
+
Key + |
+Value + |
+
|---|---|
kafka_instance_id + |
+Kafka instance + |
+
kafka_broker + |
+Kafka instance broker + |
+
kafka_topics + |
+Topic in a Kafka instance + |
+
kafka_partitions + |
+Partition in a Kafka instance + |
+
kafka_groups-partitions + |
+Partition consumer group in a Kafka instance + |
+
kafka_groups_topics + |
+Topic consumer group in a Kafka instance + |
+
kafka_groups + |
+Consumer group of a Kafka instance + |
+
Kafka instances are physically isolated and exclusively occupied by each tenant. You can customize the computing capabilities and storage space of an instance based on service requirements.
+
in the upper left corner to select a region.
Select the same region as your application service.
+By default, you can create a maximum of 100 Kafka instances for each project. To create more instances, contact customer service to increase your quota.
+You can view the broker quantity and flavor, the maximum number of partitions allowed, and number of consumer groups recommended for each bandwidth option.
+The Maximum Partitions parameter indicates the maximum number of partitions that can be created for a Kafka instance. If the total number of partitions of all topics exceeds this threshold, topic creation will fail.
+The storage space is the total space to be consumed by all replicas. Specify the storage space based on the expected service message size and the number of replicas. For example, if the required disk size to store the data for the retention period is 100 GB, the disk capacity must be at least: 100 GB x Number of replicas + 100 GB (reserved space).
+Disks are formatted when an instance is created. As a result, the actual available disk space is 93% to 95% of the total disk space.
+
A VPC provides an isolated virtual network for your Kafka instances. You can configure and manage the network as required.
+
After the Kafka instance is created, its VPC and subnet cannot be changed.
+A security group is a set of rules for accessing a Kafka instance. You can click Manage Security Group to view or create security groups on the network console.
+Public access is disabled by default. You can enable or disable it as required.
+After public access is enabled, configure an IPv4 EIP for each broker.
+This parameter indicates whether to enable SSL authentication when a client connects to the instance. If you enable Kafka SASL_SSL, data will be encrypted before transmission to enhance security.
+Kafka SASL_SSL is disabled by default. You can enable or disable it as required. This setting cannot be changed after the instance is created. If you want to use a different setting, you must create a new instance.
+If you enable Kafka SASL_SSL, you must also set the username and password for accessing the instance.
+This setting is disabled by default. You can enable or disable it as required.
+If automatic topic creation is enabled, the system automatically creates a topic when a message is created in or retrieved from a topic that does not exist. This topic has the following default settings: 3 partitions, 3 replicas, aging time 72 hours, and synchronous replication and flushing disabled.
+After you change the value of the log.retention.hours, default.replication.factor, or num.partitions parameter, automatically created topics later use the new value. For example, if num.partitions is set to 5, an automatically created topic will have the following settings: 5 partitions, 3 replicas, aging time 72 hours, and synchronous replication and flushing disabled.
+Tags are used to identify cloud resources. When you have many cloud resources of the same type, you can use tags to classify them by dimension (for example, use, owner, or environment).
+Up to 20 tags can be added to each Kafka instance. For details about the requirements on tags, see Managing Instance Tags.
+It takes 3 to 15 minutes to create an instance. During this period, the instance status is Creating.
+
Instances that fail to be created do not occupy other resources.
+A topic is a stream of messages. If automatic topic creation is not enabled during Kafka instance creation, you need to manually create topics for creating and retrieving messages. If automatic topic creation has been enabled for the instance, this operation is optional.
+If automatic topic creation is enabled, the system automatically creates a topic when a message is created in or retrieved from a topic that does not exist. This topic has the following default settings: 3 partitions, 3 replicas, aging time 72 hours, and synchronous replication and flushing disabled. After you change the value of the log.retention.hours, default.replication.factor, or num.partitions parameter, automatically created topics later use the new value. For example, if num.partitions is set to 5, an automatically created topic will have the following settings: 5 partitions, 3 replicas, aging time 72 hours, and synchronous replication and flushing disabled.
+There is a limit on the total number of partitions in topics. When the partition quantity limit is reached, you can no longer create topics. The total number of partitions varies with instance specifications. For details, see Specifications.
+Methods that can be used to manually create a topic:
+ +
If an instance node is faulty, an internal service error may be reported when you query messages in a topic with only one replica. Therefore, you are not advised to use a topic with only one replica.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+The Create Topic dialog box is displayed.
++
Parameter + |
+Description + |
+
|---|---|
Topic Name + |
+When creating a topic, you can modify the automatically generated topic name. +Once the topic is created, you cannot modify its name. + |
+
Partitions + |
+A larger number of partitions for a topic indicates more messages retrieved concurrently. +If this parameter is set to 1, messages will be retrieved in the FIFO order. +Value range: 1 to 100 +Default value: 3 + |
+
Replicas + |
+A higher number of replicas delivers higher reliability. Data is automatically backed up on each replica. When one Kafka broker becomes faulty, data is still available on other brokers. +If this parameter is set to 1, only one set of data is available. +Value range: 1 to 3 +Default value: 3 + NOTE:
+If an instance node is faulty, an internal service error may be reported when you query messages in a topic with only one replica. Therefore, you are not advised to use a topic with only one replica. + |
+
Aging Time (h) + |
+The period that messages are retained for. Consumers must retrieve messages before this period ends. Otherwise, the messages will be deleted and can no longer be retrieved. +Value range: 1 to 720 +Default value: 72 + |
+
Synchronous Replication + |
+A message is returned to the client only after the message creation request has been received and the message has been acknowledged by all replicas. +After enabling synchronous replication, set acks to all or –1 on the client. Otherwise, this function will not take effect. +If there is only one replica, synchronous replication cannot be enabled. + |
+
Synchronous Flushing + |
+An indicator of whether a message is immediately flushed to disk once created. +
|
+
If your client is v2.2 or later, you can use kafka-topics.sh to create topics and manage topic parameters.
+
If a topic name starts with a special character, for example, an underscore (_) or a number sign (#), monitoring data cannot be displayed.
+./kafka-topics.sh --create --topic {topic_name} --bootstrap-server {broker_ip}:{port} --partitions {partition_num} --replication-factor {replication_num}
+Create the ssl-user-config.properties file in the /config directory of the Kafka client and add the SSL certificate configurations by referring to 3.
+./kafka-topics.sh --create --topic {topic_name} --bootstrap-server {broker_ip}:{port} --partitions {partition_num} --replication-factor {replication_num} --command-config ./config/ssl-user-config.properties
+By default, there are two types of user permissions: user management and resource management.
+For further details, see Permissions.
+DMS for Kafka of the cloud service platform uses Kafka as the message engine. This chapter presents explanations of basic concepts of Kafka.
+A topic is a category for messages. Messages are created, retrieved, and managed in the form of topics.
+Topics adopt the publish-subscribe pattern. Producers publish messages into topics. One or more consumers subscribe to the messages in the topics. The producers and consumers are not directly linked to each other.
+A producer publishes messages into topics. The messages are then delivered to other systems or modules for processing as agreed.
+A consumer subscribes to messages in topics and processes the messages. For example, a monitoring and alarm platform (a consumer) subscribing to log messages in certain topics can identify alarm logs and then send SMS or email alarm notifications.
+A broker is a Kafka process in a Kafka cluster. Each process runs on a server, so a broker includes the storage, bandwidth, and other server resources.
+A topic is divided into partitions. Messages are distributed to multiple partitions to achieve scalability and fault tolerance.
+A replica is a redundant copy of a partition in a topic. Each partition can have one or more replicas, enabling message reliability.
+Messages in each partition are fully replicated and synchronized, preventing data loss if one replica fails.
+Each partition has one replica as the leader which handles the creation and retrievals of all messages. The rest replicas are followers which replicate the leader.
+Topics and partitions are logical concepts, while replicas and brokers are physical concepts. The following diagram shows the relationships between partitions, brokers, and topics in messages streaming.
+
The period that messages are retained for. Consumers must retrieve messages before this period ends. Otherwise, the messages will be deleted and can no longer be retrieved.
+DMS provides easy-to-use message queuing based on Apache Kafka. Services can be quickly migrated to the cloud without any change, reducing maintenance and usage costs.
+Simply set instance information on the DMS for Kafka console, submit your order, and a complete Kafka instance will be automatically created and deployed.
+DMS for Kafka is compatible with open-source Kafka APIs and supports all message processing functions of open-source Kafka.
+If your application services are developed based on open-source Kafka, you can easily migrate them to DMS after specifying a few authentication configurations.
+
Kafka instances are compatible with Apache Kafka v1.1.0, v2.3.0, and v2.7. Keep the client and server versions the same.
+Operations on Kafka instances are recorded and can be audited. Messages can be encrypted before storage.
+In addition to Simple Authentication and Security Layer (SASL) authentication, Virtual Private Clouds (VPCs) and security groups also provide security controls on network access.
+Kafka instances support data persistence and replication. Messages can be synchronously or asynchronously replicated between replicas and flushed to disk.
+Kafka runs in clusters, enabling failover and fault tolerance so that services can run smoothly.
+Kafka instance brokers can be deployed across AZs to enhance service availability.
+The cloud service platform provides a whole set of monitoring and alarm services, eliminating the need for 24/7 attendance. Kafka instance metrics are monitored and reported, including the number of partitions, topics, and accumulated messages. You can configure alarm rules and receive SMS or email notifications on how your services are running in real time.
+You can customize the bandwidth and storage space for the instance and the number of partitions and replicas for topics in the instance.
+Kafka instances are fully compatible with open-source clients. You can obtain clients in other programming languages and access your instance as instructed by the official Kafka website.
+You can use destination NAT (DNAT) to access a Kafka instance so that the instance can provide services on the public network through port mapping.
+You have created EIPs. The number of EIPs is the same as the number of brokers in the Kafka instance.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+Set other parameters as required. For details, see Creating a NAT Gateway.
+For details about more parameters, see Adding a DNAT Rule.
+
View the DNAT rule status in the DNAT rule list. If Status is Running, the rule has been added successfully.
+Check whether messages can be created and retrieved by referring to Accessing a Kafka Instance Without SASL or Accessing a Kafka Instance with SASL.
+Notes:
+Click the name of your Kafka instance. In the Connection section on the Basic Information tab page, view Instance Address (Public Network).
+For details about how to connect to a Kafka instance, see Accessing a Kafka Instance.
+ +Messages are not deleted immediately after being retrieved. They are deleted only when the aging time expires.
+You can shorten the aging time.
+Symptom: The monitoring data shows that there are 810 million accumulated messages. However, the Kafka console shows that there are 100 million messages in all six topics of the instance.
+Analysis: The two statistics methods are different. The Kafka console shows the number of messages that have not been retrieved. The monitoring data shows the number of accumulated messages in the topics multiplied by the number of consumer groups.
+Yes. A Kafka instance will be restarted if you enable or disable automatic topic creation for it.
+
next to Automatic Topic Creation to disable automatic topic creation.You can view the execution status of the task on the Background Tasks tab page.
+Kafka brokers and ZooKeeper are deployed on the same VM.
+No.
+For security purposes, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 is supported.
+The monitoring data is reported every minute. The reported data will be displayed on the monitoring page after being sorted. This process takes less than 20 minutes. After deleting a consumer group, you can wait for a while before checking the latest monitoring data.
+No. The AZ configuration cannot be changed once the instance is created. To use multiple AZs, create another instance.
+DMS for Kafka supports cross-AZ disaster recovery. If you select multiple AZs when creating an instance, cross-AZ disaster recovery will be available.
+You can view the AZs configured for an instance in the Network section on the Basic Information tab page of the instance. If there are multiple AZs, cross-AZ disaster recovery is available.
+No.
+No. You can only use JKS certificates for connecting to instances in Java.
+JKS certificates are used for connecting to instances in Java and CRT certificates are used for connecting to instances in Python.
+TLS 1.2.
+Yes.
+Yes, just simply unsubscribe from it on the Kafka client.
+No. You can directly delete the consumer group.
+Yes. The maximum allowed number of client connections varies by instance specifications.
+Each Kafka broker allows a maximum of 1000 connections from each IP address by default. Excess connections will be rejected. You can change the limit by referring to Modifying Kafka Parameters.
+No. Once an instance is created, its VPC and subnet cannot be changed.
+You can find Kafka Streams use cases on the official Kafka website.
+Symptom: Different consumers in a consumer group have different topic permissions. When a consumer attempts to retrieve messages from a topic, the error message "Not authorized to access topics." is displayed, and the message retrieval fails.
+
Analysis: When assigning partitions, the leader of the consumer group does not consider the permissions of individual consumers. Instead, the leader assigns partitions based on the overall subscription of the consumer group. In this case, consumers may be assigned topics that they do not have access to.
+For example, consumers A, B, and C are in the same consumer group. Consumer A has subscribed to and has permissions to access Topics 0, 1, and 2; consumer B has subscribed to and has permissions to access Topics 3, 4, and 5; consumer C has subscribed to and has permissions to access Topics 6, 7, and 8. Assume that each topic has only one partition. Based on the partition assignment determined by the leader, consumer A may be assigned Topics 0, 3, and 6; consumer B is assigned Topics 1, 4, and 7; and consumer C is assigned Topics 2, 5, and 8. In this case, consumer A does not have permissions to access Topics 3 and 6, resulting in the error.
+
Solution:
+No. Kafka instances cannot be upgraded once they are created. To use a higher Kafka version, create another Kafka instance.
+View the number of accumulated messages using any of the following methods:
+
If SASL authentication is enabled for the Kafka instance, the --command-config {SASL authentication configuration file consumer.properties} parameter must be added to the preceding command. For details about the configuration file consumer.properties, see the CLI access instructions provided in Accessing a Kafka Instance with SASL.
+Yes.
+Kafka uses the offsets.retention.minutes parameter to control how long to keep offsets for a consumer group. If offsets are not committed within this period, they will be deleted. The default value of offsets.retention.minutes is 20,160 minutes (14 days).
+If Kafka determines that there are no active consumers in a consumer group (for example, when the consumer group is empty) and there are no offsets, Kafka will delete the consumer group.
+ +The message creation time is specified by CreateTime when a producer creates messages. If this parameter is not set during message creation, the message creation time is year 1970 by default.
+This section describes how to troubleshoot Kafka connection problems.
+If the connection to a Kafka instance is abnormal, perform the following operations to troubleshoot the fault:
+ +Before connecting to a Kafka instance, ensure that the client and the instance are interconnected. If they cannot be connected, check the network connection.
+For example, if you have enabled SASL_SSL to access the Kafka instance, run the following command:
+curl -kv {ip}:{port}
+View logs to check whether the parameters printed during the initialization of the consumer and producer are the same as those set in the configuration files.
+If they are different, check the parameters in the configuration file.
+The following error is displayed:
+
Solution: Check the consumer.properties and producer.properties files, in which the ssl.endpoint.identification.algorithm parameter must be left empty, indicating that domain name verification is disabled.
+ssl.endpoint.identification.algorithm=+
The following error is displayed:
+
Solution:
+ssl.truststore.password=dms@kafka+
The following error is displayed:
+
Solution: Create another topic or enable the automatic topic creation function.
+Kafka instances can be accessed within a VPC, across VPCs, through DNAT, or over public networks. Before accessing a Kafka instance, configure a security group.
+Direction + |
+Protocol + |
+Port + |
+Source + |
+Description + |
+
|---|---|---|---|---|
Inbound + |
+TCP + |
+9092 + |
+0.0.0.0/0 + |
+Accessing an instance within a VPC (with SSL encryption disabled) + |
+
Inbound + |
+TCP + |
+9093 + |
+0.0.0.0/0 + |
+Accessing an instance within a VPC (with SSL encryption enabled) + |
+
Assume that the security groups of the client and Kafka instance are sg-53d4 and Default_All, respectively. You can specify a security group or IP address as the destination in the following rule. A security group is used as an example.
+To ensure that your client can access the Kafka instance, add the following rule to the security group configured for the client:
+ +Direction + |
+Protocol & Port + |
+Destination + |
+
|---|---|---|
Outbound + |
+All + |
+Default_All + |
+
To ensure that your client can access the Kafka instance, add the following rule to the security group configured for the instance.
+ +Direction + |
+Protocol & Port + |
+Source + |
+
|---|---|---|
Inbound + |
+All + |
+sg-53d4 + |
+
Configure security group rules according to Table 5.
+ +Direction + |
+Protocol + |
+Port + |
+Source + |
+Description + |
+
|---|---|---|---|---|
Inbound + |
+TCP + |
+9011 + |
+198.19.128.0/17 + |
+Accessing a Kafka instance using VPC Endpoint (VPCEP) + |
+
Inbound + |
+TCP + |
+9011 + |
+0.0.0.0/0 + |
+Accessing a Kafka instance using DNAT + |
+
Configure security group rules according to Table 5.
+ +Direction + |
+Protocol + |
+Port + |
+Source + |
+Description + |
+
|---|---|---|---|---|
Inbound + |
+TCP + |
+9094 + |
+0.0.0.0/0 + |
+Access Kafka through the public network (without SSL encryption). + |
+
Inbound + |
+TCP + |
+9095 + |
+0.0.0.0/0 + |
+Access Kafka through the public network (with SSL encryption). + |
+
To improve the reliability of a Kafka instance, you are advised to select three AZs or more when creating the instance. You cannot select two AZs.
+Each Kafka instance contains three ZooKeeper nodes. The ZooKeeper cluster manages Kafka instance configurations. If the ZooKeeper cluster is faulty, the Kafka instance cannot run properly. At least two ZooKeepers are required for the cluster to run properly.
+Assume that you select only two AZs. AZ 1 has one ZooKeeper node, and AZ 2 has two. If AZ 1 is faulty, the instance can be used properly. If AZ 2 is faulty, the cluster cannot be used. In this case, the availability rate of the Kafka instance is just 50%. Therefore, do not select 2 AZs.
+This may be because you do not have the Server Administrator and VPC Administrator permissions. For details about how to add permissions to a user group, see "User and User Group Management" > "Viewing and Modifying User Group Information" in the Identity and Access Management User Guide.
+The storage space is the space for storing messages (including messages in replicas), logs and metadata. When specifying storage space, specify the disk type and disk size. Different bandwidth configurations support different disk types. For details, see Table 1.For more information about disks, see "Overview" > "Disk Types and Performance" in the Elastic Volume Service User Guide.
+For example, if the required disk size to store data for the retention period is 100 GB, the disk capacity must be at least: 100 GB x Number of replicas + 100 GB (reserved space). In a Kafka cluster, each node uses a 33 GB disk to store logs and ZooKeeper data. Therefore, the actual available storage space is less than the created storage space.
+The number of replicas (3 by default) can be configured when you create a topic. If automatic topic creation has been enabled, each automatically created topic has three replicas by default. You can change this quantity by setting default.replication.factor on the Parameters tab page.
+You are advised to select ultra-high I/O, because ultra-high I/O disks deliver much higher bandwidth than high I/O.
+Different bandwidth configurations support different disk types. For details, see Table 1.
+The following policies are supported:
+When the memory usage reaches the disk capacity threshold (95%), new messages will no longer be created, but existing messages can still be retrieved until they are discarded. The default retention time is three days. This policy is suitable for scenarios where no data losses can be tolerated.
+When the memory usage reaches the disk capacity threshold (95%), messages can be created and retrieved, but 10% of the earliest messages will be deleted to ensure sufficient disk space. This policy is suitable for scenarios where no service interruption can be tolerated. Data may be lost.
+Select a proper policy based on requirements for data and service reliability. Both policies are only used for handling extreme scenarios. To avoid extreme scenarios, create sufficient disk space in the first place.
+Kafka v1.1.0, v2.3.0, and v2.7.
+Kafka instances are managed using ZooKeeper. Opening ZooKeeper may cause misoperations and service losses. ZooKeeper is used only within Kafka clusters and does not provide services externally.
+You can use open-source Kafka clients to connect to Kafka instances and call the native APIs to create and retrieve messages.
+Yes. A Kafka instance is a cluster that consists of three or more brokers.
+No. You must access a Kafka instance through one of the following ports:
+The port varies with the access mode:
+The port varies with the access mode:
+Ensure that correct rules have been configured for the security group of the instance. For details, see How Do I Select and Configure a Security Group?
+The certificates are valid for more than 15 years. You do not need to worry about certificate expiration. The certificates are used for one-way authentication when enabling SASL for Kafka instances.
+Unfortunately, you cannot synchronize two Kafka instances in real time. To migrate services from one instance to another, create messages to both instances. After all messages in the original instance have been retrieved or aged, you can migrate services to the new instance.
+The SASL_SSL setting cannot be changed once the instance has been created. Be careful when configuring this setting during instance creation. If you need to change the setting, you must create another instance.
+Yes. For details, see the instance access instructions.
+The number of connection addresses of a Kafka instance is the same as the number of brokers of the instance.
+Yes. You can access a Kafka instance across regions over a public network or by using direct connections.
+Yes. You can use one of the following methods to access a Kafka instance across VPCs:
+Yes.
+If the client and the instance are in the same VPC, cross-subnet access is supported.
+No, Kerberos authentication is not supported. Kafka supports client authentication with SASL and API calling authentication using tokens and AK/SK.
+To access an instance in SASL mode, you need the certificates provided by DMS. For details, see Accessing a Kafka Instance with SASL.
+The number of topics is related to the total number of topic partitions and the number of partitions in each topic. There is an upper limit on the aggregate number of partitions of topics. When this limit is reached, no more topics can be created.
+The partition limit varies depending on the flavor, as shown in the following table.
+ +Bandwidth + |
+I/O Type + |
+TPS (High-Throughput) + |
+TPS (Synchronous Replication) + |
+Maximum Partitions + |
+
|---|---|---|---|---|
100 MB/s + |
+High I/O + |
+100,000 + |
+60,000 + |
+300 + |
+
Ultra-high I/O + |
+100,000 + |
+80,000 + |
+300 + |
+|
300 MB/s + |
+High I/O + |
+300,000 + |
+150,000 + |
+900 + |
+
Ultra-high I/O + |
+300,000 + |
+200,000 + |
+900 + |
+|
600 MB/s + |
+Ultra-high I/O + |
+600,000 + |
+300,000 + |
+1800 + |
+
1200 MB/s + |
+Ultra-high I/O + |
+1,200,000 + |
+400,000 + |
+1800 + |
+
Kafka manages messages by partition. If there are too many partitions, message creation, storage, and retrieval will be fragmented, affecting the performance and stability. If the total number of partitions of topics reaches the upper limit, you cannot create more topics.
+The partition limit varies depending on the flavor, as shown in the following table.
+ +Bandwidth + |
+I/O Type + |
+TPS (High-Throughput) + |
+TPS (Synchronous Replication) + |
+Maximum Partitions + |
+
|---|---|---|---|---|
100 MB/s + |
+High I/O + |
+100,000 + |
+60,000 + |
+300 + |
+
Ultra-high I/O + |
+100,000 + |
+80,000 + |
+300 + |
+|
300 MB/s + |
+High I/O + |
+300,000 + |
+150,000 + |
+900 + |
+
Ultra-high I/O + |
+300,000 + |
+200,000 + |
+900 + |
+|
600 MB/s + |
+Ultra-high I/O + |
+600,000 + |
+300,000 + |
+1800 + |
+
1200 MB/s + |
+Ultra-high I/O + |
+1,200,000 + |
+400,000 + |
+1800 + |
+
Possible cause: The aggregate number of partitions of created topics has reached the upper limit. The maximum number of partitions varies with instance specifications. For details, see Specifications.
+Solution: Delete unnecessary topics.
+Automatic topic creation is supported, but batch topic import is not supported. You can only export topics in batches.
+Enable automatic topic creation using one of the following methods:
+This may be because automatic topic creation has been enabled and a consumer is connecting to the topic. If no existing topics are available for message creation, new topics will be automatically created.
+To solve this problem, disable automatic topic creation.
+Yes. Use either of the following methods to check the disk space used by a topic:
+
next to the Kafka instance name to go to the Cloud Eye console. On the Queues tab page, set Queue to the name of the topic whose disk space you want to view and Scope to Basic monitoring. The Message Size metric reflects the message size of the selected topic.If you have enabled SASL_SSL for your Kafka instance, you can configure ACL permissions for your topics. On the Topics tab page of the Kafka console, click Grant User Permission in the row that contains the topic for which you want to configure user permissions.
+For details, see Granting Permissions to a SASL_SSL User.
+No. They are generated automatically when you use the instance.
+For details about creating and retrieving messages after connecting to a Kafka instance, see Accessing a Kafka Instance.
+10 MB.
+Rebalancing is a process where partitions of topics are re-allocated for a consumer group.
+In normal cases, rebalancing occurs inevitably when a consumer is added to or removed from a consumer group. However, if a consumer is regarded as abnormal and removed from the consumer group, message retrieval may fail.
+This may happen in the following scenarios:
+A consumer sends heartbeat requests to the broker at the interval specified by heartbeat.interval.ms. If the broker does not receive any heartbeat request from the consumer within the period specified by session.timeout.ms, the broker considers that the consumer is abnormal and removes the consumer from the consumer group, triggering rebalancing.
+The maximum number of messages that a consumer can retrieve at a time is specified by max.poll.records. In most cases, a client processes the retrieved data before starting the next retrieval. The processing may be prolonged when a large number of messages are retrieved at a time and cannot be processed within the time specified by max.poll.interval.ms, or when an exception occurs during the process (for example, data needs to be written to the backend database, but the backend database pressure is too high, resulting in high latency). If the consumer does not send the next retrieval request within the time specified by max.poll.interval.ms, the broker considers that the consumer is inactive and removes it from the consumer group, triggering rebalancing.
+Scenario 1: Heartbeat requests are not sent in time.
+Solution: Set the value of session.timeout.ms to three times the value of heartbeat.interval.ms.
+Scenario 2: The interval between retrievals is too long.
+Troubleshooting methods:
+Solution: Change the aging time.
+On the console, messages are queried based on the timestamp, which is generated by the client. Different clients have different processing policies. The default value may be 0 or -1. As a result, message may fail to be queried.
+Solution: Check whether the value of createTime is correctly configured.
+If Capacity Threshold Policy is set to Automatically delete, the earliest 10% of messages will be deleted when 95% of the disk capacity is used, to ensure sufficient disk space. In this case, the messages that do not reach the aging time are also deleted and cannot be queried.
+Solution: Modify the capacity threshold policy. If Capacity Threshold Policy is set to Stop production, new messages will no longer be created when the disk usage reaches the disk capacity threshold (95%), but existing messages can still be retrieved until the aging time arrives. This policy is suitable for scenarios where no data losses can be tolerated.
+The possible causes are as follows:
+To solve the problem, delete topics and consumer groups whose names contain the special character.
+No. If you want to use fewer partitions, delete the corresponding topic, create another one, and specify the desired number of partitions.
+If the aging time has been set for a topic, the value of the log.retention.hours parameter does not take effect for the topic. The value of the log.retention.hours parameter takes effect only if the aging time has not been set for the topic.
+Possible cause 1: The segment files are in use.
+Solution: Wait until the segment is no longer in use or delete the topic where messages have reached their retention period.
+Possible cause 2: In a topic, there is a message whose CreateTime is a future time. For example, assume that it is January 1, and the CreateTime is February 1. The message will not be aged after 72 hours from now. As a result, messages created subsequently will also not be aged.
+Solution: Delete the topic where the CreateTime of a message is a future time.
+Yes. No password is required for accessing a Kafka instance with SASL disabled. For details, see Accessing a Kafka Instance Without SASL.
+ +Apache Kafka is distributed message middleware that features high throughput, data persistence, horizontal scalability, and stream data processing. It adopts the publish-subscribe pattern and is widely used for log collection, data streaming, online/offline system analytics, and real-time monitoring.
+DMS is a message queuing service based on Apache Kafka. It provides Kafka instances with isolated compute, storage, and bandwidth resources. DMS allows you to apply and configure resources based on service requirements. It can be used out of the box and frees you from deployment and O&M so that you can focus on the agile development of your applications.
+ +This documentation introduces DMS for Kafka and its differences from Apache Kafka. You will learn about the detailed information about the specifications, console operations, and client access to instances of DMS for Kafka.
+For more information about the basic knowledge of Kafka or technical details about creating and retrieving messages, please go to the official Apache Kafka website.
+CTS generates traces to provide you with a history of operations performed on cloud service resources. The traces include operation requests sent using the management console or open APIs, as well as the operation results. You can view all generated traces to query, audit, and backtrack performed operations.
+For details about the operations recorded by CTS, see Operations Logged by CTS.
+Kafka instances run in VPCs and use the IP addresses and bandwidth of VPC. Security groups of VPCs enhance the security of network access to the Kafka instances.
+An ECS is a basic computing unit that consists of vCPUs, memory, OS, and EVS disks. Kafka instances run on ECSs. A broker corresponds to an ECS.
+EVS provides block storage services for ECSs. All Kafka data, such as messages, metadata, and logs, is stored in EVS disks.
+Cloud Eye is an open platform that provides monitoring, alarm reporting, and alarm notification for your resources in real time.
+For details about DMS metrics monitored by Cloud Eye, see Kafka Metrics.
+
The values of all Kafka instance metrics are reported to Cloud Eye every minute.
+The EIP service provides independent public IP addresses and bandwidth for Internet access. Kafka instances bound with EIPs can be accessed over public networks.
+TMS is a visualized service for fast and unified cross-region tagging and categorization of cloud services.
+Tags facilitate Kafka instance identification and management.
+When creating a Kafka instance, you can specify whether to enable disk encryption. Enabling disk encryption improves data security. Disk encryption depends on the keys provided by KMS.
+This section describes the notes and constraints on DMS.
+Item + |
+Notes and Constraints + |
+
|---|---|
Kafka ZooKeeper + |
+Kafka clusters are managed using ZooKeeper. Opening ZooKeeper may cause misoperations and service losses. Currently, ZooKeeper is used only within Kafka clusters and does not provide services externally. + |
+
Version + |
+
|
+
Logging in to the VM where the Kafka brokers reside + |
+Not supported + |
+
VPC and subnet + |
+After an instance is created, its VPC and subnet cannot be modified. + |
+
Kerberos authentication + |
+Not supported + |
+
Item + |
+Notes and Constraints + |
+
|---|---|
Total number of topic partitions + |
+The total number of topic partitions is related to the instance specifications. For details, see Specifications. +Kafka manages messages by partition. If there are too many partitions, message creation, storage, and retrieval will be fragmented, affecting the performance and stability. If the total number of partitions of topics reaches the upper limit, you cannot create more topics. + |
+
Number of partitions in a topic + |
+The number of partitions in a topic can be increased but cannot be decreased. + |
+
Topic quantity + |
+The topic quantity is related to the total number of topic partitions and number of partitions in each topic. For details, see Specifications. + |
+
Automatic topic creation + |
+Supported. If automatic topic creation is enabled, the system automatically creates a topic when a message is created in or retrieved from a topic that does not exist. This topic has the following default settings: 3 partitions, 3 replicas, aging time 72 hours, and synchronous replication and flushing disabled. +After you change the value of the log.retention.hours, default.replication.factor, or num.partitions parameter, automatically created topics later use the new value. For example, if num.partitions is set to 5, an automatically created topic will have the following settings: 5 partitions, 3 replicas, aging time 72 hours, and synchronous replication and flushing disabled. + |
+
Decreasing partition quantity + |
+The partition quantity cannot be decreased due to the limitations of Apache Kafka. + |
+
Synchronous replication + |
+If a topic has only one replica, synchronous replication cannot be enabled. + |
+
Replica quantity + |
+Single-replica topics are not recommended. If an instance node is faulty, an internal service error may be reported when you query messages in a topic with only one replica. Therefore, you are not advised to use a topic with only one replica. + |
+
Aging time + |
+The value of the log.retention.hours parameter takes effect only if the aging time has not been set for the topic. +For example, if the aging time of Topic01 is set to 60 hours and log.retention.hours is set to 72 hours, the actual aging time of Topic01 is 60 hours. + |
+
Batch importing and exporting topics + |
+Batch export is supported, but batch import is not supported. + |
+
Topic name + |
+If a topic name starts with a special character, for example, an underscore (_) or a number sign (#), monitoring data cannot be displayed. + |
+
Delay queues + |
+Not supported + |
+
Item + |
+Notes and Constraints + |
+
|---|---|
Creating consumer groups, consumers, and producers + |
+Consumer groups, consumers, and producers are generated automatically when you use the instance. + |
+
Resetting the consumer offset + |
+Messages may be retrieved more than once after the offset is reset. + |
+
Consumer group name + |
+If a consumer group name starts with a special character, for example, an underscore (_) or a number sign (#), monitoring data cannot be displayed. + |
+
Item + |
+Notes and Constraints + |
+
|---|---|
Message size + |
+The maximum length of a message is 10 MB. If the length exceeds 10 MB, the production fails. + |
+
DMS is compatible with open-source Kafka and has customized and enhanced Kafka features. In addition to the advantages of open-source Kafka, DMS for Kafka provides more reliable and useful features.
+ +Category + |
+Item + |
+DMS + |
+Open-source Kafka + |
+
|---|---|---|---|
Ease of use + |
+Readily available + |
+Instances can be created intuitively within minutes and used right out of the box with visualized operations and real-time monitoring. + |
+Preparing server resources and installing and configuring the software is time-consuming and prone to mistakes. + |
+
APIs + |
+Instances can be managed easily by calling RESTful APIs. + |
+N/A + |
+|
Costs + |
+On-demand use + |
+Multiple specifications are available to suit different needs. + |
+Expenses are incurred for setting up a message service and occupying underlying resources. + |
+
Fully managed + |
+Services are readily available without requiring additional hardware resources or expenses. + |
+Users must prepare hardware resources and set up the service by themselves, and bear high usage and maintenance costs. + |
+|
Proven success + + |
+Mature + |
+DMS has been deployed in many cloud products and proven successful in large e-commerce events. It is also used in the clouds of carrier-grade customers across the world, and meets strict carrier-grade reliability standards. DMS closely follows up with community updates to continuously fix known open-source vulnerabilities and add support for new features. + |
+Using open-source software requires lengthy self-development and verification and has had few successful cases. + |
+
Feature-rich + |
+While maintaining 100% open-source compatibility, DMS further optimizes open-source code to improve performance and reliability, and provides message querying, and many other features. + |
+Functionality is limited and requires self-development. + |
+|
Reliability + |
+Highly available + |
+DMS supports cross-AZ deployment to improve reliability. In addition, automatic fault detection and alarms ensure reliable operations of key services. + |
+High availability requires self-development or open-source code implementation, which are costly and cannot guarantee reliability. + |
+
Simple O&M + |
+O&M is entirely transparent to tenants with a full set of monitoring and alarm functions. O&M personnel will be informed of any exceptions, eliminating the need for 24/7 attending. + |
+Users need to develop and optimize O&M functions, especially alarm notification functions. Otherwise, manual attendance is required. + |
+|
Secure + |
+DMS uses VPC isolation, disk encryption, and SSL channel encryption. + |
+Security must be hardened by users themselves. + |
+
Kafka is popular message-oriented middleware that features highly reliable, asynchronous message delivery. It is widely used for transmitting data between different systems in many industries, including enterprise application, payment, telecommunications, e-commerce, social networking, instant messaging, video, Internet of Things, and Internet of Vehicle.
+Non-core or less important messages are sent asynchronously to receiving systems, so that the main service process is not kept waiting for the results of other systems, allowing for faster responses.
+For example, Kafka can be used to send a notification email and SMS message after a user has registered with a website, providing fast responses throughout the registration process.
+

In e-commerce systems or large-scale websites, there is a processing capability gap between upstream and downstream systems. Traffic bursts from upstream systems with high processing capabilities may have a large impact on downstream systems with lower processing capabilities. For example, online sales promotions involve a huge amount of traffic flooding into e-commerce systems. Kafka provides a three-day buffer by default for hundreds of millions of messages, such as orders and other information. In this way, message consumption systems can process the messages during off-peak periods.
+In addition, flash sale traffic bursts originating from frontend systems can be handled with Kafka, keeping the backend systems from crashing.
+
In large-scale service systems, logs of different applications are collected for quick troubleshooting, full-link tracing, and real-time monitoring.
+Kafka is originally designed for this scenario. Applications asynchronously send log messages to message queues over reliable transmission channels. Other components can read the log messages from message queues for further analysis, either in real time or offline. In addition, Kafka can collect key log information to monitor applications.
+Log synchronization involves three major components: log collection clients, Kafka, and backend log processing applications.
+Kafka clients receive and compress messages in batches. This only has a minor impact on the service performance.
+
Logstash is for log analytics, Elasticsearch is for log search, and Hadoop is for big data analytics. They are all open-source tools.
+Released On + |
+What's New + |
+
|---|---|
2022-10-30 + |
+This release incorporates the following changes: +
|
+
2020-08-11 + |
+This release incorporates the following changes: +
|
+
2020-07-29 + |
+This release incorporates the following changes: +
|
+
2020-02-20 + |
+This release incorporates the following changes: +
|
+
2019-12-16 + |
+Accepted for RM-2128. + |
+
2019-11-30 + |
+This release incorporates the following changes: +
|
+
2019-11-08 + |
+This release incorporates the following changes: +
|
+
2018-11-07 + |
+Accepted in OTC 3.2. + |
+
2018-05-25 + |
+Accepted in OTC 3.1. + |
+
2018-04-28 + |
+Added the following content: +
|
+
2018-04-03 + |
+Added the following content: +
|
+
2018-01-19 + |
+Modified the following content: +
|
+
2017-12-05 + |
+Modified the following content: +
|
+
2017-10-25 + |
+Modified the following content: +
|
+
2017-07-28 + |
+Modified the following content: +
|
+
2017-03-30 + |
+Modified the following content: +
|
+
2017-01-19 + |
+Modified the following content: +
|
+
2017-01-13 + |
+Modified the following content: +
|
+
2017-01-09 + |
+Modified the following content: +
|
+
2016-12-19 + |
+This issue is the first official release. + |
+
VPCs are logically isolated from each other. If a Kafka instance and a Kafka client are in different VPCs within a region, they cannot communicate with each other. In this case, you can use one of the following methods to access a Kafka instance across VPCs:
+The following describes how to use VPCEP to implement cross-VPC access.
+VPCEP provides two types of resources: VPC endpoint services and VPC endpoints.
+

in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+


curl -i -k -H 'Accept:application/json' -H 'Content-Type:application/json;charset=utf8' -X POST -H "X-Auth-Token:$token" -d '{"port_id":"38axxxeac","vpc_id":"706xxx888","ports":[{"protocol":"TCP","client_port":9011,"server_port":9011 }],"approval_enabled":false,"service_type":"interface","server_type":"VM"}' https://{endpoint}/v1/{project_id}/vpc-endpoint-services
+Parameter description:
+Record the value of service_name in the response. This parameter indicates the name of the VPC endpoint service.
+If the Kafka client and Kafka instance belong to different accounts, add the ID of the account to which the Kafka client belongs to the whitelist of the endpoint service. For details, see Add a Whitelist Record.
+Retain the default values for other parameters. For details, see Creating a VPC Endpoint.
+
You can use the private IP address to access the VPC endpoint service.
+
Each IP address must match the corresponding port ID. Otherwise, the network will be disconnected.
+Check whether messages can be created and retrieved by referring to Accessing a Kafka Instance Without SASL or Accessing a Kafka Instance with SASL.
+Notes:
+
198.19.128.0/17 is the network segment allocated to the VPCEP service. To use VPCEP, allow access from this network segment.
+DMS supports ACL permission management for topics. You can differentiate the operations that different users are allowed to perform on a topic by granting the users different permissions.
+This section describes how to grant topic permissions to a SASL_SSL user. For details about how to create a SASL_SSL user, see Creating a SASL_SSL User.
+If no SASL_SSL user is granted any permission for a topic, all users can subscribe to or publish messages to the topic.
+If one or more SASL_SSL users are granted permissions for a topic, only the authorized users can subscribe to or publish messages to the topic.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+In the upper part of the Grant User Permission dialog box, the topic information is displayed, including the topic name, number of partitions, aging time, number of replicas, and whether synchronous flushing is enabled. In the middle part, you can use the search box to search for a user if there are many SASL_SSL users. In the Users area, the list of created SASL_SSL users is displayed. In the Selected area, you can grant permissions to the SASL_SSL users.
+As shown in Figure 1, only the test, send, and receive users can subscribe to or publish messages to topic-01. The send_receive user cannot subscribe to or publish messages to topic-01.
+On the Topics tab page, click
next to the topic name to view the authorized users and their permissions.

in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+DMS supports ACL permission management for topics. You can differentiate the operations that different users are allowed to perform on a topic by granting the users different permissions.
+This section describes how to create a SASL_SSL user after SASL_SSL is enabled for a Kafka instance. For details about how to grant user permissions, see Granting Permissions to a SASL_SSL User.
+SASL_SSL has been enabled when you create the Kafka instance.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+After the SASL_SSL user is created, grant permissions to the user by referring to Granting Permissions to a SASL_SSL User.
+On the Kafka console, you can view the disk usage of each broker.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
You can query topics that use the most disk space or topics that have used a specified amount or percentage of disk space.
+In the upper right corner of the page, click View Metric. On the displayed Cloud Eye page, you can view metrics of Kafka instances.
+After creating a topic, you can increase the number of partitions based on service requirements.
+
Changing the number of partitions does not affect services.
+Methods for changing the partition quantity:
+ +
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
If your Kafka client version is later than 2.2, you can use kafka-topics.sh to change the partition quantity.
+./kafka-topics.sh --bootstrap-server {broker_ip}:{port} --topic {topic_name} --alter --partitions {partition_num}
+Create the ssl-user-config.properties file in the /config directory of the Kafka client and add the SSL certificate configurations by referring to 3.
+./kafka-topics.sh --bootstrap-server {broker_ip}:{port} --topic {topic_name} --alter --partitions {partition_num} --command-config ./config/ssl-user-config.properties
+Your Kafka instances, topics, and consumers come with default configuration parameter settings. You can modify common parameters on the DMS console. For details about parameters that are not listed on the console, see the Kafka official website.
+Parameters of v1.1.0 instances are all static parameters. v2.3.0/2.7 instances have both dynamic and static parameters.
+
Configuration parameters of some old instances cannot be modified. Check whether your instance parameters can be modified on the console. If they cannot be modified, contact customer service.
+You can modify configuration parameters of a Kafka instance when the instance is in the Running state.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
++
Parameter + |
+Description + |
+Value Range + |
+Default Value + |
+
|---|---|---|---|
min.insync.replicas + |
+If a producer sets the acks parameter to all (or -1), the min.insync.replicas parameter specifies the minimum number of replicas that must acknowledge a write for the write to be considered successful. + |
+1–3 + |
+1 + |
+
message.max.bytes + |
+Maximum length of a single message, in bytes. + |
+0–10,485,760 + |
+10,485,760 + |
+
unclean.leader.election.enable + |
+Indicates whether to allow replicas not in the ISR set to be elected as the leader as a last resort, even though doing so may result in data loss. + |
+true or false + |
+true + |
+
connections.max.idle.ms + |
+Idle connection timeout (in ms). Connections that are idle for the duration specified by this parameter will be closed. + |
+5000–600,000 + |
+600,000 + |
+
log.retention.hours + |
+Duration (in hours) for retaining a log file. +This parameter takes effect only for topics that have no aging time configured. If there is aging time configured for topics, it overrides this parameter. + |
+1–168 + |
+72 + |
+
max.connections.per.ip + |
+The maximum number of connections allowed from each IP address. Request for new connections will be rejected once the limit is reached. The limit set using this parameter will be replaced if there are overrides configured using the max.connections.per.ip.overrides parameter. + |
+100–20,000 + |
+1000 + |
+
group.max.session.timeout.ms + |
+The maximum session timeout (in ms) for consumers. A longer timeout gives consumers more time to process messages between heartbeats but results in a longer time to detect failures. + |
+6000–1,800,000 + |
+1,800,000 + |
+
default.replication.factor + |
+The default number of replicas configured for an automatically created topic. + |
+1–3 + |
+3 + |
+
num.partitions + |
+The default number of partitions configured for each automatically created topic. + |
+1–100 + |
+3 + |
+
group.min.session.timeout.ms + |
+The minimum session timeout (in ms) for consumers. A shorter timeout enables quicker failure detection but results in more frequent consumer heartbeating, which can overwhelm broker resources. + |
+6000–300,000 + |
+6000 + |
+
Parameter + |
+Description + |
+Value Range + |
+Default Value + |
+
|---|---|---|---|
min.insync.replicas + |
+If a producer sets the acks parameter to all (or -1), the min.insync.replicas parameter specifies the minimum number of replicas that must acknowledge a write for the write to be considered successful. + |
+1–3 + |
+1 + |
+
message.max.bytes + |
+Maximum length of a single message, in bytes. + |
+0–10,485,760 + |
+10,485,760 + |
+
unclean.leader.election.enable + |
+Indicates whether to allow replicas not in the ISR set to be elected as the leader as a last resort, even though doing so may result in data loss. + |
+true or false + |
+true + |
+
Parameter + |
+Description + |
+Value Range + |
+Default Value + |
+
|---|---|---|---|
connections.max.idle.ms + |
+Idle connection timeout (in ms). Connections that are idle for the duration specified by this parameter will be closed. + |
+5000–600,000 + |
+600,000 + |
+
log.retention.hours + |
+Duration (in hours) for retaining a log file. +This parameter takes effect only for topics that have no aging time configured. If there is aging time configured for topics, it overrides this parameter. + |
+1–168 + |
+72 + |
+
max.connections.per.ip + |
+The maximum number of connections allowed from each IP address. Request for new connections will be rejected once the limit is reached. The limit set using this parameter will be replaced if there are overrides configured using the max.connections.per.ip.overrides parameter. + |
+100–20,000 + |
+1000 + |
+
group.max.session.timeout.ms + |
+The maximum session timeout (in ms) for consumers. A longer timeout gives consumers more time to process messages between heartbeats but results in a longer time to detect failures. + |
+6000–1,800,000 + |
+1,800,000 + |
+
default.replication.factor + |
+The default number of replicas configured for an automatically created topic. + |
+1–3 + |
+3 + |
+
num.partitions + |
+The default number of partitions configured for each automatically created topic. + |
+1–100 + |
+3 + |
+
group.min.session.timeout.ms + |
+The minimum session timeout (in ms) for consumers. A shorter timeout enables quicker failure detection but results in more frequent consumer heartbeating, which can overwhelm broker resources. + |
+6000–300,000 + |
+6000 + |
+
Modifying dynamic parameters will not restart the instance. Static parameter modification requires manual restart of the instance.
+You can delete a consumer group using either of the following methods:
+The status of the consumer group to be deleted is EMPTY.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
A consumer group can be deleted only when its status is EMPTY.
+Consumer group statuses include:
+The following uses Linux as an example.
+kafka-consumer-groups.sh --bootstrap-server {Kafka instance connection address} --delete --group {consumer group name}
+[root@zk-server-1 bin]# ./kafka-consumer-groups.sh --bootstrap-server 192.168.1.245:9091,192.168.1.86:9091,192.168.1.128:9091 --delete --group bbbb
+Note: This will not show information about old Zookeeper-based consumers.
+Deletion of requested consumer groups ('bbbb') was successful.
+
If SASL authentication is enabled for the Kafka instance, the --command-config {consumer.properties file with SASL authentication} parameter must be added to the preceding commands. For details about the consumer.properties file, see Accessing a Kafka Instance with SASL.
+Resetting the consumer offset is to change the retrieval position of a consumer.
+
Messages may be retrieved more than once after the offset is reset. Exercise caution when performing this operation.
+The consumer offset cannot be reset on the fly. You must first stop retrieval of the desired consumer group.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
++
Parameter + |
+Description + |
+
|---|---|
Reset By + |
+You can reset an offset by: +
|
+
Time + |
+Set this parameter if Reset By is set to Time. +Select a time point. After the reset is complete, retrieval starts from this time point. +
|
+
Offset + |
+Set this parameter if Reset By is set to Offset. +Enter an offset, which is greater than or equal to 0. After the reset is complete, retrieval starts from this offset. + |
+
View consumer connection addresses on the DMS console.
+
The connection address of a consumer can be viewed only when the consumer is connected to a Kafka instance.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+To access a Kafka instance over a public network, enable public access and configure EIPs for the instance.
+If you no longer need public access to the instance, you can disable it as required.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
Enabling public access
+Click
next to Public Access to enable public access. For Elastic IP Address, select an EIP for each broker and then click
.
You can view the operation progress on the Background Tasks page. If the task status is Successful, the modification has succeeded.
+
After public access is enabled, configure security group rules listed in Table 1 before attempting to access Kafka. For details about accessing Kafka, see Accessing a Kafka Instance.
+ +Direction + |
+Protocol + |
+Port + |
+Source + |
+Description + |
+
|---|---|---|---|---|
Inbound + |
+TCP + |
+9094 + |
+0.0.0.0/0 + |
+Access Kafka through the public network (without SSL encryption). + |
+
Inbound + |
+TCP + |
+9095 + |
+0.0.0.0/0 + |
+Access Kafka through the public network (with SSL encryption). + |
+
Disabling public access
+Click
next to Public Access.
You can view the operation progress on the Background Tasks page. If the task status is Successful, the modification has succeeded.
+With Cloud Trace Service (CTS), you can record operations associated with DMS for later query, audit, and backtrack operations.
+ +Operation + |
+Resource Type + |
+Trace Name + |
+
|---|---|---|
Successfully deleting a background task + |
+kafka + |
+deleteDMSBackendJobSuccess + |
+
Failing to delete a background task + |
+kafka + |
+deleteDMSBackendJobFailure + |
+
Successfully creating an order for creating an instance + |
+kafka + |
+createDMSInstanceOrderSuccess + |
+
Failing to create an order for creating an instance + |
+kafka + |
+createDMSInstanceOrderFailure + |
+
Successfully submitting a request to modify an instance order + |
+kafka + |
+modifyDMSInstanceOrderSuccess + |
+
Failing to submit a request to modify an instance order + |
+kafka + |
+modifyDMSInstanceOrderFailure + |
+
Successfully submitting a request to scale up an instance + |
+kafka + |
+extendDMSInstanceSuccess + |
+
Failing to submit a request to scale up an instance + |
+kafka + |
+extendDMSInstanceFailure + |
+
Successfully submitting a request to reset instance password + |
+kafka + |
+resetDMSInstancePasswordSuccess + |
+
Failing to submit a request to reset instance password + |
+kafka + |
+resetDMSInstancePasswordFailure + |
+
Successfully creating a topic for a Kafka instance + |
+kafka + |
+Kafka_platinum_create_topicSuccess + |
+
Failing to create a topic for a Kafka instance + |
+kafka + |
+Kafka_platinum_create_topicFailure + |
+
Successfully deleting a topic from a Kafka instance + |
+kafka + |
+Kafka_platinum_delete_topicsSuccess + |
+
Failing to delete a topic for a Kafka instance + |
+kafka + |
+Kafka_platinum_delete_topicsFailure + |
+
Successfully deleting an instance that failed to be created + |
+kafka + |
+deleteDMSCreateFailureInstancesSuccess + |
+
Failing to delete an instance that failed to be created + |
+kafka + |
+deleteDMSCreateFailureInstancesFailure + |
+
Successfully submitting a request to restart an instance + |
+kafka + |
+restartDMSInstanceSuccess + |
+
Failing to submit a request to restart an instance + |
+kafka + |
+restartDMSInstanceFailure + |
+
Successfully submitting a request to delete multiple instances at a time + |
+kafka + |
+batchDeleteDMSInstanceSuccess + |
+
Failing to submit a request to delete multiple instances at a time + |
+kafka + |
+batchDeleteDMSInstanceFailure + |
+
Successfully submitting a request to restart multiple instances at a time + |
+kafka + |
+batchRestartDMSInstanceSuccess + |
+
Failing to submit a request to restart multiple instances at a time + |
+kafka + |
+batchRestartDMSInstanceFailure + |
+
Successfully submitting a request to modify instance information + |
+kafka + |
+modifyDMSInstanceInfoSuccess + |
+
Failing to submit a request to modify instance information + |
+kafka + |
+modifyDMSInstanceInfoFailure + |
+
Deleting multiple instance tasks at a time + |
+kafka + |
+batchDeleteDMSInstanceTask + |
+
Successfully deleting an instance + |
+kafka + |
+deleteDMSInstanceTaskSuccess + |
+
Failing to delete an instance + |
+kafka + |
+deleteDMSInstanceTaskFailure + |
+
Successfully creating an instance + |
+kafka + |
+createDMSInstanceTaskSuccess + |
+
Failing to create an instance + |
+kafka + |
+createDMSInstanceTaskFailure + |
+
Successfully scaling up an instance + |
+kafka + |
+extendDMSInstanceTaskSuccess + |
+
Failing to scale up an instance + |
+kafka + |
+extendDMSInstanceTaskFailure + |
+
Successfully restarting an instance + |
+kafka + |
+restartDMSInstanceTaskSuccess + |
+
Failing to restart an instance + |
+kafka + |
+restartDMSInstanceTaskFailure + |
+
Successfully restarting multiple instances at a time + |
+kafka + |
+batchRestartDMSInstanceTaskSuccess + |
+
Failing to restart multiple instances at a time + |
+kafka + |
+batchRestartDMSInstanceTaskFailure + |
+
Successfully modifying instance information + |
+kafka + |
+modifyDMSInstanceInfoTaskSuccess + |
+
Failing to modify instance information + |
+kafka + |
+modifyDMSInstanceInfoTaskFailure + |
+
This section describes how to view operation records of the last 7 days on the CTS console.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
on the left of a trace to expand its details.
Before creating a Kafka instance, ensure the availability of resources, including a virtual private cloud (VPC), subnet, security group, and security group rules. Each Kafka instance is deployed in a VPC and bound to a specific subnet and security group. In this way, Kafka provides an isolated virtual network environment and security protection policies that you can easily configure and manage.
+To access a Kafka instance over a public network, prepare an elastic IP address (EIP) in advance.
+To encrypt the disk, prepare a KMS key in advance.
+Table 1 lists the resources required by a Kafka instance.
+ +Resource + |
+Requirement + |
+Operations + |
+
|---|---|---|
VPC and subnet + |
+Different Kafka instances can use the same or different VPCs and subnets based on site requirements. Note the following when creating a VPC and a subnet: +
|
+For details about how to create a VPC and subnet, see the Virtual Private Cloud User Guide. + |
+
Security group + |
+Different Kafka instances can use the same or different security groups. Note the following when creating a security group: +
|
+For details about how to create a security group and configure security group rules, see the Virtual Private Cloud User Guide. + |
+
EIP + |
+Note the following when creating EIPs: +
|
+For details about how to create an EIP, see "Assigning an EIP" in Elastic IP User Guide. + |
+
KMS key + |
+To encrypt the disk for a Kafka instance, prepare a KMS key in advance. +The KMS key must be created in the same region as the Kafka instance. + |
+For details about how to create a KMS key, see "Creating a Key" in the Key Management Service User Guide. + |
+
Direction + |
+Protocol + |
+Port + |
+Source + |
+Description + |
+
|---|---|---|---|---|
Inbound + |
+TCP + |
+9094 + |
+0.0.0.0/0 + |
+Access a Kafka instance through the public network (without SSL encryption). + |
+
Inbound + |
+TCP + |
+9092 + |
+0.0.0.0/0 + |
+Access a Kafka instance within a VPC (without SSL encryption). + |
+
Inbound + |
+TCP + |
+9095 + |
+0.0.0.0/0 + |
+Access a Kafka instance through the public network (with SSL encryption). + |
+
Inbound + |
+TCP + |
+9093 + |
+0.0.0.0/0 + |
+Access a Kafka instance within a VPC (with SSL encryption). + |
+
Inbound + |
+TCP + |
+9011 + |
+198.19.128.0/17 + |
+Access a Kafka instance using VPC Endpoint (VPCEP). + |
+
Inbound + |
+TCP + |
+9011 + |
+0.0.0.0/0 + |
+Access a Kafka instance using DNAT. + |
+
View detailed information about a Kafka instance on the DMS console, for example, the IP addresses and port numbers for accessing the instance.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
++
Status + |
+Description + |
+
|---|---|
Creating + |
+The instance is being created. + |
+
Running + |
+The instance is running properly. +Only instances in the Running state can provide services. + |
+
Faulty + |
+The instance is not running properly. + |
+
Starting + |
+The status between Frozen and Running. + |
+
Restarting + |
+The instance is being restarted. + |
+
Changing + |
+The instance specifications or public access configurations are being modified. + |
+
Change failed + |
+The instance specifications or public access configurations failed to be modified. + |
+
Frozen + |
+The instance is frozen. + |
+
Freezing + |
+The status between Running and Frozen. + |
+
Upgrading + |
+The instance is being upgraded. + |
+
Rolling back + |
+The instance is being rolled back. + |
+
Restart one or more Kafka instances at a time on the DMS console.
+
When a Kafka instance is being restarted, message retrieval and creation requests of clients will be rejected.
+The status of the Kafka instance you want to restart is either Running or Faulty.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+It takes 3 to 15 minutes to restart a Kafka instance. After the instance is successfully restarted, its status should be Running.
+
Restarting a Kafka instance only restarts the instance process and does not restart the VM where the instance is located.
+On the DMS console, you can delete one or more Kafka instances that have been created or failed to be created.
+
Deleting a Kafka instance will delete the data in the instance without any backup. Exercise caution when performing this operation.
+The status of the Kafka instance you want to delete is Running or Faulty.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
Kafka instances in the Creating, Starting, Changing, Change failed, or Restarting state cannot be deleted.
+It takes 1 to 60 seconds to delete a Kafka instance.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
Instances that fail to be created do not occupy other resources.
+After creating a Kafka instance, you can modify some parameters of the instance based on service requirements, including the instance name, description, security group, and capacity threshold policy.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+After the parameters are modified, view the modification result in one of the following ways:
+Delete a topic using either of the following methods:
+ +
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+If your Kafka client version is later than 2.2, you can use kafka-topics.sh to delete topics.
+./kafka-topics.sh --bootstrap-server {broker_ip}:{port} --delete --topic {topic_name}
+Create the ssl-user-config.properties file in the /config directory of the Kafka client and add the SSL certificate configurations by referring to 3.
+./kafka-topics.sh --bootstrap-server {broker_ip}:{port} --delete --topic {topic_name} --command-config ./config/ssl-user-config.properties
+This section describes how to use an open-source Kafka client to access a Kafka instance if SASL access is not enabled for the instance. There are two scenarios. For cross-VPC access, see Cross-VPC Access to a Kafka Instance. For DNAT-based access, see Using DNAT to Access a Kafka Instance.
+For details on how to use Kafka clients in different languages, visit https://cwiki.apache.org/confluence/display/KAFKA/Clients.
+
To access a Kafka instance with SASL disabled, configure correct security group rules. For details about security group configuration requirements, see Table 2.
+

Add the following lines to the .bash_profile file in the home directory as an authorized user: In this command, /opt/java/jdk1.8.0_151 is the JDK installation path. Change it to the path where you install JDK.
+export JAVA_HOME=/opt/java/jdk1.8.0_151 +export PATH=$JAVA_HOME/bin:$PATH+
Run the source .bash_profile command for the modification to take effect.
+The following uses Linux as an example.
+Access the directory where the CLI package is stored and run the following command to decompress the package:
+tar -zxf [kafka_tar]
+In the preceding command, [kafka_tar] indicates the name of the CLI package.
+For example:
+tar -zxf kafka_2.12-2.7.2.tgz
+In Windows, you need to access the /bin/windows directory.
+./kafka-console-producer.sh --broker-list ${connection-address} --topic ${topic-name}
+Parameter description:
+The following example uses connection addresses 10.3.196.45:9094,10.78.42.127:9094,10.4.49.103:9094. After running the preceding command, you can send a message to the Kafka instance by writing it and pressing Enter. Each line of content is sent as a message.
+[root@ecs-kafka bin]# ./kafka-console-producer.sh --broker-list 10.3.196.45:9094,10.78.42.127:9094,10.4.49.103:9094 --topic topic-demo +>Hello +>DMS +>Kafka! +>^C[root@ecs-kafka bin]#+
To stop creating messages, press Ctrl+C to exit.
+./kafka-console-consumer.sh --bootstrap-server ${connection-address} --topic ${topic-name} --group ${consumer-group-name} --from-beginning
+Parameter description:
+Example:
+[root@ecs-kafka bin]# ./kafka-console-consumer.sh --bootstrap-server 10.3.196.45:9094,10.78.42.127:9094,10.4.49.103:9094 --topic topic-demo --group order-test --from-beginning +Kafka! +DMS +Hello +^CProcessed a total of 3 messages +[root@ecs-kafka bin]#+
To stop retrieving messages, press Ctrl+C to exit.
+You can reset the SASL_SSL password for accessing a Kafka instance by resetting Kafka password if you forget it.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
The system will display a success message only after the password is successfully reset on all brokers.
+If you enable SASL_SSL when creating an instance, data will be encrypted before transmission for enhanced security.
+For security purposes, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 is supported.
+This section describes how to use an open-source Kafka client to access a Kafka instance if SASL has been enabled for the instance. There are two scenarios. For cross-VPC access, see Cross-VPC Access to a Kafka Instance. For DNAT-based access, see Using DNAT to Access a Kafka Instance.
+
Each Kafka broker allows a maximum of 1000 connections from each IP address by default. Excess connections will be rejected. You can change the limit by referring to Modifying Kafka Parameters.
+A Kafka instance with SASL enabled can be accessed within a VPC or over public networks. Ensure that security group rules have been correctly configured for the instance. For details about security group configuration requirements, see Table 2.
+

Add the following lines to the .bash_profile file in the home directory as an authorized user: In this command, /opt/java/jdk1.8.0_151 is the JDK installation path. Change it to the path where you install JDK.
+export JAVA_HOME=/opt/java/jdk1.8.0_151 +export PATH=$JAVA_HOME/bin:$PATH+
Run the source .bash_profile command for the modification to take effect.
+The following uses Linux as an example.
+Set IP addresses to the instance connection addresses obtained in Prerequisites. Set hosts to the names of instance hosts. Specify a unique name for each host.
+For example:
+10.154.48.120 server01
+10.154.48.121 server02
+10.154.48.122 server03
+Access the directory where the CLI package is stored and run the following command to decompress the package:
+tar -zxf [kafka_tar]
+In the preceding command, [kafka_tar] indicates the name of the CLI package.
+For example:
+tar -zxf kafka_2.12-2.7.2.tgz
+Find the consumer.properties and producer.properties files in the /config directory of the Kafka CLI and add the following content to the files:
+sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
+username="**********" \
+password="**********";
+sasl.mechanism=PLAIN
+
+security.protocol=SASL_SSL
+ssl.truststore.location={ssl_truststore_path}
+ssl.truststore.password=dms@kafka
+ssl.endpoint.identification.algorithm=
+Parameter description:
+In Windows, you need to access the /bin/windows directory.
+./kafka-console-producer.sh --broker-list ${connection-address} --topic ${topic-name} --producer.config ../config/producer.properties
+Parameter description:
+The following example uses connection addresses 10.3.196.45:9095,10.78.42.127:9095,10.4.49.103:9095.
+After running the preceding command, you can send a message to the Kafka instance by writing it and pressing Enter. Each line of content is sent as a message.
+[root@ecs-kafka bin]#./kafka-console-producer.sh --broker-list 10.3.196.45:9095,10.78.42.127:9095,10.4.49.103:9095 --topic topic-demo --producer.config ../config/producer.properties +>Hello +>DMS +>Kafka! +>^C[root@ecs-kafka bin]#+
To stop creating messages, press Ctrl+C to exit.
+./kafka-console-consumer.sh --bootstrap-server ${connection-address} --topic ${topic-name} --group ${consumer-group-name} --from-beginning --consumer.config ../config/consumer.properties
+Parameter description:
+Example:
+[root@ecs-kafka bin]# ./kafka-console-consumer.sh --bootstrap-server 10.3.196.45:9095,10.78.42.127:9095,10.4.49.103:9095 --topic topic-demo --group order-test --from-beginning --consumer.config ../config/consumer.properties +Hello +DMS +Kafka! +^CProcessed a total of 3 messages +[root@ecs-kafka bin]#+
To stop retrieving messages, press Ctrl+C to exit.
+Cloud Eye monitors Kafka instance metrics in real time. You can view these metrics on the Cloud Eye console.
+At least one Kafka instance has been created. The instance has at least one available message.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
next to a Kafka instance name. On the Cloud Eye console, view the metrics of the instance, nodes, topics, and consumer groups. Metric data is reported to Cloud Eye every minute.You can view the offset of different partitions, the message size, creation time, and body of messages in topics.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+If no partition is specified, messages in all partitions of the topic are displayed.
+You can search by the following methods:
+
If a topic contains a large amount of data, an internal service error may be reported when you query messages in a topic with only one replica. You can shorten the time range for query based on the data volume.
+Parameter description:
+After you initiate certain instance operations such as configuring public access and modifying the capacity threshold policy, a background task will start for each operation. On the console, you can view the background task status and clear task information by deleting task records.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+A list of background tasks is displayed.
+
to refresh the task status.
You can only delete the records of tasks in the Successful or Failed state.
+Aging time is a period that messages in the topic are retained for. Consumers must retrieve messages before this period ends. Otherwise, the messages will be deleted and can no longer be retrieved.
+After creating a topic, you can change its aging time based on service requirements. Changing the aging time does not affect services. The default aging time is 72 hours.
+You can change the aging time in either of the following ways:
+
The log.retention.hours parameter takes effect only for topics that have no aging time configured. If there is aging time configured for a topic, it overrides the log.retention.hours parameter. For example, if the aging time of Topic01 is set to 60 hours and log.retention.hours is set to 72 hours, the actual aging time of Topic01 is 60 hours.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+On the DMS console, click the name of the target Kafka instance. Disable Public Access in the Connection section on the Basic Information tab page, and then enable it again. Select the EIP to be bound.
+View the consumer group list, consumer list, and consumer offsets.
+The consumer list can be viewed only when consumers in a consumer group are connected to the Kafka instance (that is, the consumer group is in the STABLE state).
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+The consumer group name, status, and Coordinator are displayed. Coordinator indicates the broker where the coordinator component is located. The consumer group status can be:
+
.
in the upper right corner../kafka-consumer-groups.sh --bootstrap-server {broker_ip}:{port} --list
+Create the ssl-user-config.properties file in the /config directory of the Kafka client and add the SSL certificate configurations by referring to 3.
+./kafka-consumer-groups.sh --bootstrap-server {broker_ip}:{port} --list --command-config ./config/ssl-user-config.properties
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+In the consumer list, you can view the consumer ID, consumer address, and client ID.
+
../kafka-consumer-groups.sh --bootstrap-server {broker_ip}:{port} --group {group_name} --members --describe
+Create the ssl-user-config.properties file in the /config directory of the Kafka client and add the SSL certificate configurations by referring to 3.
+./kafka-consumer-groups.sh --bootstrap-server {broker_ip}:{port} --group {group_name} --members --describe --command-config ./config/ssl-user-config.properties
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
../kafka-consumer-groups.sh --bootstrap-server {broker_ip}:{port} --offsets --describe --all-groups
+Create the ssl-user-config.properties file in the /config directory of the Kafka client and add the SSL certificate configurations by referring to 3.
+./kafka-consumer-groups.sh --bootstrap-server {broker_ip}:{port} --offsets --describe --all-groups --command-config ./config/ssl-user-config.properties
+Synchronous replication: A message is returned to the client only after the message creation request has been received and the message has been acknowledged by all replicas.
+Synchronous flushing: A message is immediately flushed to disk once created.
+The following procedure describes how to modify synchronous replication and synchronous flushing settings on the console.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
.
.
On the console, view sample code for creating and retrieving messages in Java, Go, and Python.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+View sample code for creating and retrieving messages in Java, Go, and Python. In the sample code in Go and Python, you can see whether SASL_SSL authentication is enabled. If Access By is PLAINTEXT, SASL_SSL authentication is disabled. If Access By is SASL_SSL, SASL_SSL authentication is enabled.
+If you forget the password of a SASL_SSL user created on the Users tab page, you can reset the password and use the new password to connect to the Kafka instance.
+If you forget the SASL_SSL password set during instance creation, reset the password by following the instructions provided in Resetting Kafka Password.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
The system will display a success message only after the password is successfully reset on all brokers.
+This section describes how to delete a SASL_SSL user.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
The SASL_SSL user configured during the creation of a Kafka instance cannot be deleted.
+Export topics on the console. Batch export is supported.
+A topic has been created.
+
in the upper left corner to select a region.
Select the region where your Kafka instance is located.
+
in the upper right corner to export the topic list.The topic list contains the following information: topic name, number of partitions, number of replicas, aging time, and whether synchronous replication and flushing are enabled.
+