forked from docs/doc-exports
Su, Xiaomeng
76a5b1ee83
Reviewed-by: Pruthi, Vineet <vineet.pruthi@t-systems.com> Co-authored-by: Su, Xiaomeng <suxiaomeng1@huawei.com> Co-committed-by: Su, Xiaomeng <suxiaomeng1@huawei.com>
147 lines
14 KiB
HTML
147 lines
14 KiB
HTML
<a name="dli_08_0413"></a><a name="dli_08_0413"></a>
|
|
|
|
<h1 class="topictitle1">JSON</h1>
|
|
<div id="body8662426"><div class="section" id="dli_08_0413__en-us_topic_0000001262335938_section18194105273210"><h4 class="sectiontitle">Function</h4><p id="dli_08_0413__en-us_topic_0000001262335938_p1352145614326">The JSON format allows you to read and write JSON data based on a JSON schema. Currently, the JSON schema is derived from table schema.</p>
|
|
</div>
|
|
<div class="section" id="dli_08_0413__en-us_topic_0000001262335938_section340211663317"><h4 class="sectiontitle">Supported Connectors</h4><ul id="dli_08_0413__en-us_topic_0000001262335938_ul1741845193412"><li id="dli_08_0413__en-us_topic_0000001262335938_li147412457348">Kafka</li><li id="dli_08_0413__en-us_topic_0000001262335938_li574124553416">Upsert Kafka</li><li id="dli_08_0413__en-us_topic_0000001262335938_li774184583413">Elasticsearch</li></ul>
|
|
</div>
|
|
<div class="section" id="dli_08_0413__en-us_topic_0000001262335938_section196716720333"><h4 class="sectiontitle">Parameters</h4>
|
|
<div class="tablenoborder"><table cellpadding="4" cellspacing="0" summary="" id="dli_08_0413__en-us_topic_0000001262335938_table1800141133610" frame="border" border="1" rules="all"><caption><b>Table 1 </b></caption><thead align="left"><tr id="dli_08_0413__en-us_topic_0000001262335938_row980114133620"><th align="left" class="cellrowborder" valign="top" width="20%" id="mcps1.3.3.2.2.6.1.1"><p id="dli_08_0413__en-us_topic_0000001262335938_p38010113364">Parameter</p>
|
|
</th>
|
|
<th align="left" class="cellrowborder" valign="top" width="11.81%" id="mcps1.3.3.2.2.6.1.2"><p id="dli_08_0413__en-us_topic_0000001262335938_p4801131173614">Mandatory</p>
|
|
</th>
|
|
<th align="left" class="cellrowborder" valign="top" width="10.03%" id="mcps1.3.3.2.2.6.1.3"><p id="dli_08_0413__en-us_topic_0000001262335938_p1380113133611">Default Value</p>
|
|
</th>
|
|
<th align="left" class="cellrowborder" valign="top" width="14.34%" id="mcps1.3.3.2.2.6.1.4"><p id="dli_08_0413__en-us_topic_0000001262335938_p138011813366">Type</p>
|
|
</th>
|
|
<th align="left" class="cellrowborder" valign="top" width="43.82%" id="mcps1.3.3.2.2.6.1.5"><p id="dli_08_0413__en-us_topic_0000001262335938_p108012115361">Description</p>
|
|
</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody><tr id="dli_08_0413__en-us_topic_0000001262335938_row78019117363"><td class="cellrowborder" valign="top" width="20%" headers="mcps1.3.3.2.2.6.1.1 "><p id="dli_08_0413__en-us_topic_0000001262335938_p234420229377">format</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="11.81%" headers="mcps1.3.3.2.2.6.1.2 "><p id="dli_08_0413__en-us_topic_0000001262335938_p193441922133712">Yes</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="10.03%" headers="mcps1.3.3.2.2.6.1.3 "><p id="dli_08_0413__en-us_topic_0000001262335938_p134452243717">None</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="14.34%" headers="mcps1.3.3.2.2.6.1.4 "><p id="dli_08_0413__en-us_topic_0000001262335938_p934452293712">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="43.82%" headers="mcps1.3.3.2.2.6.1.5 "><p id="dli_08_0413__en-us_topic_0000001262335938_p7344122210373">Format to be used. Set this parameter to <strong id="dli_08_0413__en-us_topic_0000001262335938_b1028295616250">json</strong>.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0413__en-us_topic_0000001262335938_row9801171183617"><td class="cellrowborder" valign="top" width="20%" headers="mcps1.3.3.2.2.6.1.1 "><p id="dli_08_0413__en-us_topic_0000001262335938_p193447229372">json.fail-on-missing-field</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="11.81%" headers="mcps1.3.3.2.2.6.1.2 "><p id="dli_08_0413__en-us_topic_0000001262335938_p7344722123711">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="10.03%" headers="mcps1.3.3.2.2.6.1.3 "><p id="dli_08_0413__en-us_topic_0000001262335938_p3344322193717">false</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="14.34%" headers="mcps1.3.3.2.2.6.1.4 "><p id="dli_08_0413__en-us_topic_0000001262335938_p1734417228374">Boolean</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="43.82%" headers="mcps1.3.3.2.2.6.1.5 "><p id="dli_08_0413__en-us_topic_0000001262335938_p19344152273718">Whether to skip the field or row or throws an error when a field to be parsed is missing. The default value is <strong id="dli_08_0413__en-us_topic_0000001262335938_b4721171092614">false</strong>, indicating that no error will be thrown.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0413__en-us_topic_0000001262335938_row198012183611"><td class="cellrowborder" valign="top" width="20%" headers="mcps1.3.3.2.2.6.1.1 "><p id="dli_08_0413__en-us_topic_0000001262335938_p2344192273710">json.ignore-parse-errors</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="11.81%" headers="mcps1.3.3.2.2.6.1.2 "><p id="dli_08_0413__en-us_topic_0000001262335938_p23441122113710">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="10.03%" headers="mcps1.3.3.2.2.6.1.3 "><p id="dli_08_0413__en-us_topic_0000001262335938_p183441122163710">false</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="14.34%" headers="mcps1.3.3.2.2.6.1.4 "><p id="dli_08_0413__en-us_topic_0000001262335938_p153441622123711">Boolean</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="43.82%" headers="mcps1.3.3.2.2.6.1.5 "><p id="dli_08_0413__en-us_topic_0000001262335938_p03442022103711">Whether fields and rows with parse errors will be skipped or failed. The default value is <strong id="dli_08_0413__en-us_topic_0000001262335938_b1864941412717">false</strong>, indicating that an error will be thrown. Fields are set to null in case of errors.</p>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0413__en-us_topic_0000001262335938_row380112183611"><td class="cellrowborder" valign="top" width="20%" headers="mcps1.3.3.2.2.6.1.1 "><p id="dli_08_0413__en-us_topic_0000001262335938_p93445222373">json.timestamp-format.standard</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="11.81%" headers="mcps1.3.3.2.2.6.1.2 "><p id="dli_08_0413__en-us_topic_0000001262335938_p1634462213374">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="10.03%" headers="mcps1.3.3.2.2.6.1.3 "><p id="dli_08_0413__en-us_topic_0000001262335938_p111861849154716">'SQL'</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="14.34%" headers="mcps1.3.3.2.2.6.1.4 "><p id="dli_08_0413__en-us_topic_0000001262335938_p16344222143717">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="43.82%" headers="mcps1.3.3.2.2.6.1.5 "><p id="dli_08_0413__en-us_topic_0000001262335938_p16344122123713">Input and output timestamp format for TIMESTAMP and TIMESTAMP WITH LOCAL TIME ZONE.</p>
|
|
<p id="dli_08_0413__en-us_topic_0000001262335938_p1344142233710">Currently supported values are <strong id="dli_08_0413__en-us_topic_0000001262335938_b1194163294712">SQL</strong> and <strong id="dli_08_0413__en-us_topic_0000001262335938_b209471232104715">ISO-8601</strong>:</p>
|
|
<ul id="dli_08_0413__en-us_topic_0000001262335938_ul1344132283711"><li id="dli_08_0413__en-us_topic_0000001262335938_li11344102233710"><strong id="dli_08_0413__en-us_topic_0000001262335938_b149131940135119">SQL</strong> will parse the input TIMESTAMP values in "yyyy-MM-dd HH:mm:ss.s{precision}" format, for example, <strong id="dli_08_0413__en-us_topic_0000001262335938_b189701856135117">2020-12-30 12:13:14.123</strong>, parse TIMESTAMP WITH LOCAL TIME ZONE values in "yyyy-MM-dd HH:mm:ss.s{precision}'Z'" format, for example, <strong id="dli_08_0413__en-us_topic_0000001262335938_b19826181817529">2020-12-30 12:13:14.123Z</strong> and output timestamp in the same format.</li><li id="dli_08_0413__en-us_topic_0000001262335938_li13344162214371"><strong id="dli_08_0413__en-us_topic_0000001262335938_b360517309527">ISO-8601</strong> will parse the input TIMESTAMP values in "yyyy-MM-ddTHH:mm:ss.s{precision}" format, for example, <strong id="dli_08_0413__en-us_topic_0000001262335938_b654011440524">2020-12-30T12:13:14.123</strong> parse TIMESTAMP WITH LOCAL TIME ZONE values in "yyyy-MM-ddTHH:mm:ss.s{precision}'Z'" format, for example, <strong id="dli_08_0413__en-us_topic_0000001262335938_b107931059155211">2020-12-30T12:13:14.123Z</strong> and output timestamp in the same format.</li></ul>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0413__en-us_topic_0000001262335938_row8863131112372"><td class="cellrowborder" valign="top" width="20%" headers="mcps1.3.3.2.2.6.1.1 "><p id="dli_08_0413__en-us_topic_0000001262335938_p103441422153713">json.map-null-key.mode</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="11.81%" headers="mcps1.3.3.2.2.6.1.2 "><p id="dli_08_0413__en-us_topic_0000001262335938_p6344722153710">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="10.03%" headers="mcps1.3.3.2.2.6.1.3 "><p id="dli_08_0413__en-us_topic_0000001262335938_p849282114476">'FALL'</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="14.34%" headers="mcps1.3.3.2.2.6.1.4 "><p id="dli_08_0413__en-us_topic_0000001262335938_p034422211375">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="43.82%" headers="mcps1.3.3.2.2.6.1.5 "><p id="dli_08_0413__en-us_topic_0000001262335938_p4344132243712">Handling mode when serializing null keys for map data. Available values are as follows:</p>
|
|
<ul id="dli_08_0413__en-us_topic_0000001262335938_ul83441822133717"><li id="dli_08_0413__en-us_topic_0000001262335938_li1134402214377"><strong id="dli_08_0413__en-us_topic_0000001262335938_b378121118534">FAIL</strong> will throw exception when encountering map value with null key.</li><li id="dli_08_0413__en-us_topic_0000001262335938_li103441022163715"><strong id="dli_08_0413__en-us_topic_0000001262335938_b754131316538">DROP</strong> will drop null key entries for map data.</li><li id="dli_08_0413__en-us_topic_0000001262335938_li103441022193712"><strong id="dli_08_0413__en-us_topic_0000001262335938_b261319146531">LITERAL</strong> replaces the empty key value in the map with a string constant. The string literal is defined by <strong id="dli_08_0413__en-us_topic_0000001262335938_b67337157531">json.map-null-key.literal</strong> option.</li></ul>
|
|
</td>
|
|
</tr>
|
|
<tr id="dli_08_0413__en-us_topic_0000001262335938_row107485503619"><td class="cellrowborder" valign="top" width="20%" headers="mcps1.3.3.2.2.6.1.1 "><p id="dli_08_0413__en-us_topic_0000001262335938_p23444222378">json.map-null-key.literal</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="11.81%" headers="mcps1.3.3.2.2.6.1.2 "><p id="dli_08_0413__en-us_topic_0000001262335938_p6344622103713">No</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="10.03%" headers="mcps1.3.3.2.2.6.1.3 "><p id="dli_08_0413__en-us_topic_0000001262335938_p16344142283712">'null'</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="14.34%" headers="mcps1.3.3.2.2.6.1.4 "><p id="dli_08_0413__en-us_topic_0000001262335938_p153442022113711">String</p>
|
|
</td>
|
|
<td class="cellrowborder" valign="top" width="43.82%" headers="mcps1.3.3.2.2.6.1.5 "><p id="dli_08_0413__en-us_topic_0000001262335938_p13344162223713">String literal to replace null key when <strong id="dli_08_0413__en-us_topic_0000001262335938_b144592695317">json.map-null-key.mode</strong> is <strong id="dli_08_0413__en-us_topic_0000001262335938_b174692614532">LITERAL</strong>.</p>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
<div class="section" id="dli_08_0413__en-us_topic_0000001262335938_section15187174103312"><h4 class="sectiontitle">Example</h4><p id="dli_08_0413__en-us_topic_0000001262335938_p1326131081119">In this example, data is read from a topic and written to another using a Kafka sink.</p>
|
|
<ol id="dli_08_0413__en-us_topic_0000001262335938_ol6699140201216"><li id="dli_08_0413__en-us_topic_0000001262335938_li8699120181210"><span>Create a datasource connection for the communication with the VPC and subnet where Kafka locates and bind the connection to the queue. Set an inbound rule for the security group to allow access of the queue and test the connectivity using the Kafka address. If the connection is successful, the datasource is bound to the queue. Otherwise, the binding fails.</span></li><li id="dli_08_0413__en-us_topic_0000001262335938_li9843102821212"><span>Create a Flink OpenSource SQL job, select Flink 1.12, and allow DLI to save job logs in OBS. Use the following statement in the job and submit it:</span><p><pre class="screen" id="dli_08_0413__en-us_topic_0000001262335938_screen550720113134">CREATE TABLE kafkaSource (
|
|
order_id string,
|
|
order_channel string,
|
|
order_time string,
|
|
pay_amount double,
|
|
real_pay double,
|
|
pay_time string,
|
|
user_id string,
|
|
user_name string,
|
|
area_id string
|
|
) WITH (
|
|
'connector' = 'kafka',
|
|
'topic' = '<yourSourceTopic>',
|
|
'properties.bootstrap.servers' = '<yourKafkaAddress>:<yourKafkaPort>',
|
|
'properties.group.id' = '<yourGroupId>',
|
|
'scan.startup.mode' = 'latest-offset',
|
|
"format" = "json"
|
|
);
|
|
|
|
CREATE TABLE kafkaSink (
|
|
order_id string,
|
|
order_channel string,
|
|
order_time string,
|
|
pay_amount double,
|
|
real_pay double,
|
|
pay_time string,
|
|
user_id string,
|
|
user_name string,
|
|
area_id string
|
|
) WITH (
|
|
'connector' = 'kafka',
|
|
'topic' = '<yourSinkTopic>',
|
|
'properties.bootstrap.servers' = '<yourKafkaAddress>:<yourKafkaPort>',
|
|
"format" = "json"
|
|
);
|
|
|
|
insert into kafkaSink select * from kafkaSource;</pre>
|
|
</p></li><li id="dli_08_0413__en-us_topic_0000001262335938_li3301119171318"><span>Insert the following data into the source Kafka topic:</span><p><pre class="screen" id="dli_08_0413__en-us_topic_0000001262335938_screen1643116512133">{"order_id":"202103241000000001","order_channel":"webShop","order_time":"2021-03-24 10:00:00","pay_amount":100.0,"real_pay":100.0,"pay_time":"2021-03-24 10:02:03","user_id":"0001","user_name":"Alice","area_id":"330106"}
|
|
|
|
{"order_id":"202103241606060001","order_channel":"appShop","order_time":"2021-03-24 16:06:06","pay_amount":200.0,"real_pay":180.0,"pay_time":"2021-03-24 16:10:06","user_id":"0001","user_name":"Alice","area_id":"330106"}</pre>
|
|
</p></li><li id="dli_08_0413__en-us_topic_0000001262335938_li482216494138"><span>Read data from the sink topic. The result is as follows:</span><p><pre class="screen" id="dli_08_0413__en-us_topic_0000001262335938_screen11212991415">{"order_id":"202103241000000001","order_channel":"webShop","order_time":"2021-03-24 10:00:00","pay_amount":100.0,"real_pay":100.0,"pay_time":"2021-03-24 10:02:03","user_id":"0001","user_name":"Alice","area_id":"330106"}
|
|
|
|
{"order_id":"202103241606060001","order_channel":"appShop","order_time":"2021-03-24 16:06:06","pay_amount":200.0,"real_pay":180.0,"pay_time":"2021-03-24 16:10:06","user_id":"0001","user_name":"Alice","area_id":"330106"}</pre>
|
|
</p></li></ol>
|
|
</div>
|
|
</div>
|
|
<div>
|
|
<div class="familylinks">
|
|
<div class="parentlink"><strong>Parent topic:</strong> <a href="dli_08_0407.html">Format</a></div>
|
|
</div>
|
|
</div>
|
|
|