From 3a07c4072987c2a4fee58d0669b6dbeb465e2918 Mon Sep 17 00:00:00 2001 From: "Su, Xiaomeng" Date: Mon, 24 Feb 2025 08:11:08 +0000 Subject: [PATCH] dli_umn_20250212 Reviewed-by: Pruthi, Vineet Co-authored-by: Su, Xiaomeng Co-committed-by: Su, Xiaomeng --- docs/dli/umn/ALL_META.TXT.json | 9352 +++++++++-------- docs/dli/umn/CLASS.TXT.json | 3656 +++---- docs/dli/umn/dli_01_00006.html | 27 +- docs/dli/umn/dli_01_0001.html | 83 +- docs/dli/umn/dli_01_0002.html | 215 +- docs/dli/umn/dli_01_0003.html | 43 +- docs/dli/umn/dli_01_0004.html | 14 +- docs/dli/umn/dli_01_0005.html | 147 +- docs/dli/umn/dli_01_0006.html | 49 +- docs/dli/umn/dli_01_0007.html | 4 +- docs/dli/umn/dli_01_0008.html | 4 +- docs/dli/umn/dli_01_0010.html | 4 +- docs/dli/umn/dli_01_0011.html | 13 +- docs/dli/umn/dli_01_0012.html | 33 - docs/dli/umn/dli_01_0013.html | 2 +- docs/dli/umn/dli_01_0014.html | 4 +- docs/dli/umn/dli_01_0015.html | 28 +- docs/dli/umn/dli_01_0016.html | 4 +- docs/dli/umn/dli_01_0017.html | 111 +- docs/dli/umn/dli_01_0018.html | 2 +- docs/dli/umn/dli_01_0019.html | 6 +- docs/dli/umn/dli_01_0020.html | 441 + docs/dli/umn/dli_01_0021.html | 121 +- docs/dli/umn/dli_01_0022.html | 6 +- docs/dli/umn/dli_01_0220.html | 10 +- docs/dli/umn/dli_01_0228.html | 154 +- docs/dli/umn/dli_01_0253.html | 8 +- docs/dli/umn/dli_01_0318.html | 485 +- docs/dli/umn/dli_01_0320.html | 345 +- docs/dli/umn/dli_01_0363.html | 79 +- docs/dli/umn/dli_01_0366.html | 16 +- docs/dli/umn/dli_01_0367.html | 9 +- docs/dli/umn/dli_01_0369.html | 39 +- docs/dli/umn/dli_01_0375.html | 201 +- docs/dli/umn/dli_01_0376.html | 10 +- docs/dli/umn/dli_01_0377.html | 32 +- docs/dli/umn/dli_01_0378.html | 41 +- docs/dli/umn/dli_01_0379.html | 31 +- docs/dli/umn/dli_01_0384.html | 146 +- docs/dli/umn/dli_01_0385.html | 53 +- docs/dli/umn/dli_01_0389.html | 24 +- docs/dli/umn/dli_01_0390.html | 28 +- docs/dli/umn/dli_01_0397.html | 4 +- docs/dli/umn/dli_01_0402.html | 141 +- docs/dli/umn/dli_01_0403.html | 83 +- docs/dli/umn/dli_01_0407.html | 70 +- docs/dli/umn/dli_01_0408.html | 2 - docs/dli/umn/dli_01_0410.html | 8 +- docs/dli/umn/dli_01_0418.html | 7 +- docs/dli/umn/dli_01_0419.html | 93 + docs/dli/umn/dli_01_0420.html | 19 - docs/dli/umn/dli_01_0421.html | 2 +- docs/dli/umn/dli_01_0422.html | 6 +- docs/dli/umn/dli_01_0426.html | 26 +- docs/dli/umn/dli_01_0427.html | 2 +- docs/dli/umn/dli_01_0440.html | 68 +- docs/dli/umn/dli_01_0443.html | 9 +- docs/dli/umn/dli_01_0445.html | 14 +- docs/dli/umn/dli_01_0447.html | 12 +- docs/dli/umn/dli_01_0448.html | 22 +- docs/dli/umn/dli_01_0454.html | 28 - docs/dli/umn/dli_01_0455.html | 181 - docs/dli/umn/dli_01_0457.html | 183 +- docs/dli/umn/dli_01_0461.html | 74 +- docs/dli/umn/dli_01_0462.html | 366 +- docs/dli/umn/dli_01_0463.html | 8 +- docs/dli/umn/dli_01_0464.html | 131 +- docs/dli/umn/dli_01_0465.html | 16 +- docs/dli/umn/dli_01_0476.html | 44 +- docs/dli/umn/dli_01_0477.html | 60 +- docs/dli/umn/dli_01_0478.html | 29 +- docs/dli/umn/dli_01_0479.html | 12 +- docs/dli/umn/dli_01_0480.html | 2 +- docs/dli/umn/dli_01_0485.html | 17 - docs/dli/umn/dli_01_0486.html | 16 + docs/dli/umn/dli_01_0487.html | 12 +- docs/dli/umn/dli_01_0488.html | 18 +- docs/dli/umn/dli_01_0489.html | 16 +- docs/dli/umn/dli_01_0494.html | 62 + docs/dli/umn/dli_01_0498.html | 102 +- docs/dli/umn/dli_01_0504.html | 151 +- docs/dli/umn/dli_01_0505.html | 206 +- docs/dli/umn/dli_01_0506.html | 82 +- docs/dli/umn/dli_01_0507.html | 2 +- docs/dli/umn/dli_01_0508.html | 16 +- docs/dli/umn/dli_01_0509.html | 119 +- docs/dli/umn/dli_01_05110.html | 15 +- docs/dli/umn/dli_01_05111.html | 13 +- docs/dli/umn/dli_01_0512.html | 320 +- docs/dli/umn/dli_01_0513.html | 21 - docs/dli/umn/dli_01_0514.html | 34 +- docs/dli/umn/dli_01_0515.html | 27 +- docs/dli/umn/dli_01_0516.html | 40 +- docs/dli/umn/dli_01_0524.html | 10 +- docs/dli/umn/dli_01_0525.html | 6 +- docs/dli/umn/dli_01_0526.html | 2 +- docs/dli/umn/dli_01_0528.html | 26 +- docs/dli/umn/dli_01_0529.html | 12 +- docs/dli/umn/dli_01_0530.html | 2 +- docs/dli/umn/dli_01_0531.html | 556 +- docs/dli/umn/dli_01_0532.html | 4 +- docs/dli/umn/dli_01_0533.html | 41 - docs/dli/umn/dli_01_0534.html | 7 +- docs/dli/umn/dli_01_0535.html | 34 +- docs/dli/umn/dli_01_0536.html | 20 + docs/dli/umn/dli_01_0538.html | 4 +- docs/dli/umn/dli_01_0539.html | 28 +- docs/dli/umn/dli_01_0550.html | 4 +- docs/dli/umn/dli_01_0551.html | 4 +- docs/dli/umn/dli_01_0552.html | 10 +- docs/dli/umn/dli_01_0553.html | 2 +- docs/dli/umn/dli_01_0556.html | 4 +- docs/dli/umn/dli_01_0558.html | 2 +- docs/dli/umn/dli_01_0559.html | 2 +- docs/dli/umn/dli_01_0560.html | 2 +- docs/dli/umn/dli_01_0561.html | 35 +- docs/dli/umn/dli_01_0563.html | 4 +- docs/dli/umn/dli_01_0565.html | 10 +- docs/dli/umn/dli_01_0566.html | 8 +- docs/dli/umn/dli_01_0567.html | 22 +- docs/dli/umn/dli_01_0611.html | 15 + docs/dli/umn/dli_01_0616.html | 132 + docs/dli/umn/dli_01_0617.html | 152 + docs/dli/umn/dli_01_0618.html | 70 + docs/dli/umn/dli_01_0621.html | 102 + docs/dli/umn/dli_01_0622.html | 55 + docs/dli/umn/dli_01_0623.html | 17 + docs/dli/umn/dli_01_0624.html | 71 - docs/dli/umn/dli_01_0625.html | 26 + docs/dli/umn/dli_01_0626.html | 20 + docs/dli/umn/dli_01_0627.html | 13 + docs/dli/umn/dli_01_0634.html | 27 + docs/dli/umn/dli_01_0635.html | 17 + docs/dli/umn/dli_01_0636.html | 32 + docs/dli/umn/dli_01_0637.html | 23 + docs/dli/umn/dli_01_0638.html | 18 + docs/dli/umn/dli_01_0639.html | 19 + docs/dli/umn/dli_01_0650.html | 33 + docs/dli/umn/dli_01_0651.html | 29 + docs/dli/umn/dli_01_0652.html | 21 + docs/dli/umn/dli_01_0661.html | 17 + docs/dli/umn/dli_01_0662.html | 16 + docs/dli/umn/dli_01_0663.html | 16 + docs/dli/umn/dli_01_0664.html | 16 + docs/dli/umn/dli_01_0665.html | 118 + docs/dli/umn/dli_01_0666.html | 110 + docs/dli/umn/dli_01_0667.html | 436 + docs/dli/umn/dli_03_0001.html | 18 +- docs/dli/umn/dli_03_0008.html | 10 +- docs/dli/umn/dli_03_0009.html | 8 +- docs/dli/umn/dli_03_0010.html | 11 - docs/dli/umn/dli_03_0011.html | 12 - docs/dli/umn/dli_03_0013.html | 9 +- docs/dli/umn/dli_03_0014.html | 9 +- docs/dli/umn/dli_03_0017.html | 12 +- docs/dli/umn/dli_03_0020.html | 12 +- docs/dli/umn/dli_03_0021.html | 12 +- docs/dli/umn/dli_03_0022.html | 32 +- docs/dli/umn/dli_03_0025.html | 12 - docs/dli/umn/dli_03_0028.html | 7 +- docs/dli/umn/dli_03_0029.html | 14 +- docs/dli/umn/dli_03_0031.html | 6 +- docs/dli/umn/dli_03_0032.html | 14 +- docs/dli/umn/dli_03_0036.html | 10 +- docs/dli/umn/dli_03_0037.html | 10 +- docs/dli/umn/dli_03_0040.html | 8 +- docs/dli/umn/dli_03_0043.html | 11 - docs/dli/umn/dli_03_0044.html | 15 +- docs/dli/umn/dli_03_0045.html | 13 - docs/dli/umn/dli_03_0046.html | 9 +- docs/dli/umn/dli_03_0047.html | 17 +- docs/dli/umn/dli_03_0048.html | 9 +- docs/dli/umn/dli_03_0049.html | 16 +- docs/dli/umn/dli_03_0053.html | 6 +- docs/dli/umn/dli_03_0054.html | 18 +- docs/dli/umn/dli_03_0056.html | 9 +- docs/dli/umn/dli_03_0057.html | 11 +- docs/dli/umn/dli_03_0058.html | 17 - docs/dli/umn/dli_03_0060.html | 10 +- docs/dli/umn/dli_03_0061.html | 11 - docs/dli/umn/dli_03_0064.html | 12 +- docs/dli/umn/dli_03_0065.html | 10 +- docs/dli/umn/dli_03_0066.html | 12 +- docs/dli/umn/dli_03_0067.html | 12 +- docs/dli/umn/dli_03_0068.html | 10 +- docs/dli/umn/dli_03_0069.html | 14 +- docs/dli/umn/dli_03_0071.html | 9 +- docs/dli/umn/dli_03_0072.html | 8 +- docs/dli/umn/dli_03_0073.html | 12 - docs/dli/umn/dli_03_0075.html | 61 - docs/dli/umn/dli_03_0076.html | 10 +- docs/dli/umn/dli_03_0077.html | 8 +- docs/dli/umn/dli_03_0080.html | 12 +- docs/dli/umn/dli_03_0082.html | 14 +- docs/dli/umn/dli_03_0083.html | 14 - docs/dli/umn/dli_03_0085.html | 11 - docs/dli/umn/dli_03_0086.html | 13 +- docs/dli/umn/dli_03_0087.html | 12 - docs/dli/umn/dli_03_0088.html | 12 - docs/dli/umn/dli_03_0089.html | 18 +- docs/dli/umn/dli_03_0090.html | 8 +- docs/dli/umn/dli_03_0091.html | 8 +- docs/dli/umn/dli_03_0092.html | 10 +- docs/dli/umn/dli_03_0093.html | 30 +- docs/dli/umn/dli_03_0095.html | 8 +- docs/dli/umn/dli_03_0096.html | 8 +- docs/dli/umn/dli_03_0098.html | 8 +- docs/dli/umn/dli_03_0099.html | 12 +- docs/dli/umn/dli_03_0100.html | 73 - docs/dli/umn/dli_03_0102.html | 16 +- docs/dli/umn/dli_03_0103.html | 8 +- docs/dli/umn/dli_03_0105.html | 6 +- docs/dli/umn/dli_03_0106.html | 58 +- docs/dli/umn/dli_03_0107.html | 10 +- docs/dli/umn/dli_03_0108.html | 9 +- docs/dli/umn/dli_03_0109.html | 11 - docs/dli/umn/dli_03_0110.html | 28 - docs/dli/umn/dli_03_0111.html | 13 - docs/dli/umn/dli_03_0112.html | 20 - docs/dli/umn/dli_03_0115.html | 10 +- docs/dli/umn/dli_03_0116.html | 8 +- docs/dli/umn/dli_03_0117.html | 11 - docs/dli/umn/dli_03_0118.html | 10 +- docs/dli/umn/dli_03_0119.html | 9 +- docs/dli/umn/dli_03_0120.html | 16 + docs/dli/umn/dli_03_0121.html | 30 + docs/dli/umn/dli_03_0125.html | 11 - docs/dli/umn/dli_03_0126.html | 12 +- docs/dli/umn/dli_03_0127.html | 12 - docs/dli/umn/dli_03_0128.html | 9 +- docs/dli/umn/dli_03_0129.html | 6 +- docs/dli/umn/dli_03_0130.html | 11 - docs/dli/umn/dli_03_0131.html | 26 +- docs/dli/umn/dli_03_0132.html | 8 +- docs/dli/umn/dli_03_0133.html | 16 +- docs/dli/umn/dli_03_0135.html | 32 - docs/dli/umn/dli_03_0136.html | 48 +- docs/dli/umn/dli_03_0137.html | 14 +- docs/dli/umn/dli_03_0138.html | 16 - docs/dli/umn/dli_03_0139.html | 12 +- docs/dli/umn/dli_03_0140.html | 13 - docs/dli/umn/dli_03_0145.html | 16 +- docs/dli/umn/dli_03_0156.html | 11 +- docs/dli/umn/dli_03_0157.html | 9 +- docs/dli/umn/dli_03_0159.html | 14 - docs/dli/umn/dli_03_0160.html | 8 +- docs/dli/umn/dli_03_0161.html | 16 +- docs/dli/umn/dli_03_0162.html | 9 +- docs/dli/umn/dli_03_0163.html | 16 - docs/dli/umn/dli_03_0164.html | 10 +- docs/dli/umn/dli_03_0165.html | 17 - docs/dli/umn/dli_03_0166.html | 11 - docs/dli/umn/dli_03_0167.html | 13 +- docs/dli/umn/dli_03_0168.html | 16 +- docs/dli/umn/dli_03_0169.html | 14 +- docs/dli/umn/dli_03_0170.html | 11 - docs/dli/umn/dli_03_0171.html | 10 +- docs/dli/umn/dli_03_0172.html | 25 - docs/dli/umn/dli_03_0173.html | 10 +- docs/dli/umn/dli_03_0174.html | 8 +- docs/dli/umn/dli_03_0175.html | 12 +- docs/dli/umn/dli_03_0176.html | 17 - docs/dli/umn/dli_03_0177.html | 10 +- docs/dli/umn/dli_03_0178.html | 10 +- docs/dli/umn/dli_03_0179.html | 46 +- docs/dli/umn/dli_03_0180.html | 10 +- docs/dli/umn/dli_03_0181.html | 16 +- docs/dli/umn/dli_03_0182.html | 9 +- docs/dli/umn/dli_03_0183.html | 10 +- docs/dli/umn/dli_03_0184.html | 12 +- docs/dli/umn/dli_03_0186.html | 14 +- docs/dli/umn/dli_03_0187.html | 22 +- docs/dli/umn/dli_03_0188.html | 7 +- docs/dli/umn/dli_03_0189.html | 8 +- docs/dli/umn/dli_03_0190.html | 10 +- docs/dli/umn/dli_03_0191.html | 6 +- docs/dli/umn/dli_03_0192.html | 8 +- docs/dli/umn/dli_03_0193.html | 11 - docs/dli/umn/dli_03_0195.html | 10 +- docs/dli/umn/dli_03_0196.html | 44 +- docs/dli/umn/dli_03_0200.html | 16 +- docs/dli/umn/dli_03_0201.html | 12 +- docs/dli/umn/dli_03_0204.html | 27 +- docs/dli/umn/dli_03_0206.html | 10 +- docs/dli/umn/dli_03_0207.html | 10 +- docs/dli/umn/dli_03_0208.html | 7 +- docs/dli/umn/dli_03_0209.html | 7 +- docs/dli/umn/dli_03_0210.html | 11 +- docs/dli/umn/dli_03_0211.html | 45 +- docs/dli/umn/dli_03_0212.html | 8 +- docs/dli/umn/dli_03_0213.html | 6 +- docs/dli/umn/dli_03_0214.html | 12 +- docs/dli/umn/dli_03_0215.html | 9 +- docs/dli/umn/dli_03_0216.html | 16 - docs/dli/umn/dli_03_0217.html | 16 +- docs/dli/umn/dli_03_0218.html | 18 +- docs/dli/umn/dli_03_0219.html | 18 - docs/dli/umn/dli_03_0220.html | 12 +- docs/dli/umn/dli_03_0221.html | 30 - docs/dli/umn/dli_03_0222.html | 18 - docs/dli/umn/dli_03_0223.html | 17 - docs/dli/umn/dli_03_0225.html | 71 + docs/dli/umn/dli_03_0226.html | 26 - docs/dli/umn/dli_03_0227.html | 10 +- docs/dli/umn/dli_03_0228.html | 11 +- docs/dli/umn/dli_03_0229.html | 22 - docs/dli/umn/dli_03_0230.html | 30 - docs/dli/umn/dli_03_0231.html | 25 +- docs/dli/umn/dli_03_0232.html | 12 +- docs/dli/umn/dli_03_0233.html | 7 +- docs/dli/umn/dli_03_0234.html | 12 +- docs/dli/umn/dli_03_0235.html | 16 - docs/dli/umn/dli_03_0236.html | 12 +- docs/dli/umn/dli_03_0237.html | 18 - docs/dli/umn/dli_03_0238.html | 18 +- docs/dli/umn/dli_03_0239.html | 20 +- docs/dli/umn/dli_03_0250.html | 14 +- docs/dli/umn/dli_03_0251.html | 18 +- docs/dli/umn/dli_03_0252.html | 12 +- docs/dli/umn/dli_03_0253.html | 16 +- docs/dli/umn/dli_03_0254.html | 12 +- docs/dli/umn/dli_03_0255.html | 12 - docs/dli/umn/dli_03_0256.html | 32 - docs/dli/umn/dli_03_0257.html | 8 +- docs/dli/umn/dli_03_0259.html | 11 - docs/dli/umn/dli_03_0260.html | 9 +- docs/dli/umn/dli_03_0261.html | 16 - docs/dli/umn/dli_03_0263.html | 8 +- docs/dli/umn/dli_03_0264.html | 21 - docs/dli/umn/dli_03_0265.html | 7 +- docs/dli/umn/dli_03_0266.html | 11 - docs/dli/umn/dli_03_0272.html | 16 + docs/dli/umn/dli_03_0275.html | 17 + docs/dli/umn/dli_03_0276.html | 32 +- docs/dli/umn/dli_07_0003.html | 64 +- docs/dli/umn/dli_07_0005.html | 264 +- docs/dli/umn/dli_07_0027.html | 49 + docs/dli/umn/en-us_image_0000001336444477.png | Bin 0 -> 280 bytes docs/dli/umn/en-us_image_0000001437320657.png | Bin 221 -> 0 bytes docs/dli/umn/en-us_image_0000001437640221.png | Bin 1004 -> 0 bytes docs/dli/umn/en-us_image_0000001452097001.png | Bin 0 -> 480 bytes docs/dli/umn/en-us_image_0000001452099393.png | Bin 0 -> 399 bytes docs/dli/umn/en-us_image_0000001529468213.png | Bin 494 -> 0 bytes docs/dli/umn/en-us_image_0000001594371725.png | Bin 0 -> 53328 bytes docs/dli/umn/en-us_image_0000001742695104.png | Bin 0 -> 12089 bytes docs/dli/umn/en-us_image_0000001838073572.png | Bin 0 -> 413 bytes docs/dli/umn/en-us_image_0000001842062193.png | Bin 0 -> 11597 bytes docs/dli/umn/en-us_image_0000001891931040.png | Bin 0 -> 432 bytes ...1.png => en-us_image_0000001897113369.png} | Bin docs/dli/umn/en-us_image_0000001934973209.png | Bin 0 -> 432 bytes docs/dli/umn/en-us_image_0000001940346350.png | Bin 0 -> 11678 bytes ...3.png => en-us_image_0000001956063772.png} | Bin ...1.png => en-us_image_0000001992752425.png} | Bin ...2.png => en-us_image_0000001994614536.png} | Bin docs/dli/umn/en-us_image_0000001995626574.png | Bin 0 -> 399 bytes docs/dli/umn/en-us_image_0000001995786314.png | Bin 0 -> 480 bytes docs/dli/umn/en-us_image_0000001995796834.png | Bin 0 -> 480 bytes ...1.png => en-us_image_0000002031849645.png} | Bin ...6.png => en-us_image_0000002031969193.png} | Bin docs/dli/umn/en-us_image_0000002032113853.png | Bin 0 -> 399 bytes docs/dli/umn/en-us_image_0000002032233425.png | Bin 0 -> 480 bytes docs/dli/umn/en-us_image_0000002032236253.png | Bin 0 -> 399 bytes docs/dli/umn/en-us_image_0000002079726669.png | Bin 0 -> 83972 bytes docs/dli/umn/en-us_image_0000002112672174.png | Bin 0 -> 213 bytes docs/dli/umn/en-us_image_0000002112684630.png | Bin 0 -> 432 bytes docs/dli/umn/en-us_image_0000002129969110.png | Bin 0 -> 185 bytes docs/dli/umn/en-us_image_0000002132498398.png | Bin 0 -> 227 bytes docs/dli/umn/en-us_image_0276441461.png | Bin 636 -> 0 bytes 368 files changed, 13610 insertions(+), 11149 deletions(-) delete mode 100644 docs/dli/umn/dli_01_0012.html create mode 100644 docs/dli/umn/dli_01_0020.html create mode 100644 docs/dli/umn/dli_01_0419.html delete mode 100644 docs/dli/umn/dli_01_0420.html delete mode 100644 docs/dli/umn/dli_01_0454.html delete mode 100644 docs/dli/umn/dli_01_0455.html delete mode 100644 docs/dli/umn/dli_01_0485.html create mode 100644 docs/dli/umn/dli_01_0486.html create mode 100644 docs/dli/umn/dli_01_0494.html delete mode 100644 docs/dli/umn/dli_01_0513.html delete mode 100644 docs/dli/umn/dli_01_0533.html create mode 100644 docs/dli/umn/dli_01_0536.html create mode 100644 docs/dli/umn/dli_01_0611.html create mode 100644 docs/dli/umn/dli_01_0616.html create mode 100644 docs/dli/umn/dli_01_0617.html create mode 100644 docs/dli/umn/dli_01_0618.html create mode 100644 docs/dli/umn/dli_01_0621.html create mode 100644 docs/dli/umn/dli_01_0622.html create mode 100644 docs/dli/umn/dli_01_0623.html delete mode 100644 docs/dli/umn/dli_01_0624.html create mode 100644 docs/dli/umn/dli_01_0625.html create mode 100644 docs/dli/umn/dli_01_0626.html create mode 100644 docs/dli/umn/dli_01_0627.html create mode 100644 docs/dli/umn/dli_01_0634.html create mode 100644 docs/dli/umn/dli_01_0635.html create mode 100644 docs/dli/umn/dli_01_0636.html create mode 100644 docs/dli/umn/dli_01_0637.html create mode 100644 docs/dli/umn/dli_01_0638.html create mode 100644 docs/dli/umn/dli_01_0639.html create mode 100644 docs/dli/umn/dli_01_0650.html create mode 100644 docs/dli/umn/dli_01_0651.html create mode 100644 docs/dli/umn/dli_01_0652.html create mode 100644 docs/dli/umn/dli_01_0661.html create mode 100644 docs/dli/umn/dli_01_0662.html create mode 100644 docs/dli/umn/dli_01_0663.html create mode 100644 docs/dli/umn/dli_01_0664.html create mode 100644 docs/dli/umn/dli_01_0665.html create mode 100644 docs/dli/umn/dli_01_0666.html create mode 100644 docs/dli/umn/dli_01_0667.html delete mode 100644 docs/dli/umn/dli_03_0010.html delete mode 100644 docs/dli/umn/dli_03_0011.html delete mode 100644 docs/dli/umn/dli_03_0025.html delete mode 100644 docs/dli/umn/dli_03_0043.html delete mode 100644 docs/dli/umn/dli_03_0045.html delete mode 100644 docs/dli/umn/dli_03_0058.html delete mode 100644 docs/dli/umn/dli_03_0061.html delete mode 100644 docs/dli/umn/dli_03_0073.html delete mode 100644 docs/dli/umn/dli_03_0075.html delete mode 100644 docs/dli/umn/dli_03_0083.html delete mode 100644 docs/dli/umn/dli_03_0085.html delete mode 100644 docs/dli/umn/dli_03_0087.html delete mode 100644 docs/dli/umn/dli_03_0088.html delete mode 100644 docs/dli/umn/dli_03_0100.html delete mode 100644 docs/dli/umn/dli_03_0109.html delete mode 100644 docs/dli/umn/dli_03_0110.html delete mode 100644 docs/dli/umn/dli_03_0111.html delete mode 100644 docs/dli/umn/dli_03_0112.html delete mode 100644 docs/dli/umn/dli_03_0117.html create mode 100644 docs/dli/umn/dli_03_0120.html create mode 100644 docs/dli/umn/dli_03_0121.html delete mode 100644 docs/dli/umn/dli_03_0125.html delete mode 100644 docs/dli/umn/dli_03_0127.html delete mode 100644 docs/dli/umn/dli_03_0130.html delete mode 100644 docs/dli/umn/dli_03_0135.html delete mode 100644 docs/dli/umn/dli_03_0138.html delete mode 100644 docs/dli/umn/dli_03_0140.html delete mode 100644 docs/dli/umn/dli_03_0159.html delete mode 100644 docs/dli/umn/dli_03_0163.html delete mode 100644 docs/dli/umn/dli_03_0165.html delete mode 100644 docs/dli/umn/dli_03_0166.html delete mode 100644 docs/dli/umn/dli_03_0170.html delete mode 100644 docs/dli/umn/dli_03_0172.html delete mode 100644 docs/dli/umn/dli_03_0176.html delete mode 100644 docs/dli/umn/dli_03_0193.html delete mode 100644 docs/dli/umn/dli_03_0216.html delete mode 100644 docs/dli/umn/dli_03_0219.html delete mode 100644 docs/dli/umn/dli_03_0221.html delete mode 100644 docs/dli/umn/dli_03_0222.html delete mode 100644 docs/dli/umn/dli_03_0223.html create mode 100644 docs/dli/umn/dli_03_0225.html delete mode 100644 docs/dli/umn/dli_03_0226.html delete mode 100644 docs/dli/umn/dli_03_0229.html delete mode 100644 docs/dli/umn/dli_03_0230.html delete mode 100644 docs/dli/umn/dli_03_0235.html delete mode 100644 docs/dli/umn/dli_03_0237.html delete mode 100644 docs/dli/umn/dli_03_0255.html delete mode 100644 docs/dli/umn/dli_03_0256.html delete mode 100644 docs/dli/umn/dli_03_0259.html delete mode 100644 docs/dli/umn/dli_03_0261.html delete mode 100644 docs/dli/umn/dli_03_0264.html delete mode 100644 docs/dli/umn/dli_03_0266.html create mode 100644 docs/dli/umn/dli_03_0272.html create mode 100644 docs/dli/umn/dli_03_0275.html create mode 100644 docs/dli/umn/dli_07_0027.html create mode 100644 docs/dli/umn/en-us_image_0000001336444477.png delete mode 100644 docs/dli/umn/en-us_image_0000001437320657.png delete mode 100644 docs/dli/umn/en-us_image_0000001437640221.png create mode 100644 docs/dli/umn/en-us_image_0000001452097001.png create mode 100644 docs/dli/umn/en-us_image_0000001452099393.png delete mode 100644 docs/dli/umn/en-us_image_0000001529468213.png create mode 100644 docs/dli/umn/en-us_image_0000001594371725.png create mode 100644 docs/dli/umn/en-us_image_0000001742695104.png create mode 100644 docs/dli/umn/en-us_image_0000001838073572.png create mode 100644 docs/dli/umn/en-us_image_0000001842062193.png create mode 100644 docs/dli/umn/en-us_image_0000001891931040.png rename docs/dli/umn/{en-us_image_0000001835430521.png => en-us_image_0000001897113369.png} (100%) create mode 100644 docs/dli/umn/en-us_image_0000001934973209.png create mode 100644 docs/dli/umn/en-us_image_0000001940346350.png rename docs/dli/umn/{en-us_image_0207655173.png => en-us_image_0000001956063772.png} (100%) rename docs/dli/umn/{en-us_image_0000001597283981.png => en-us_image_0000001992752425.png} (100%) rename docs/dli/umn/{en-us_image_0237983632.png => en-us_image_0000001994614536.png} (100%) create mode 100644 docs/dli/umn/en-us_image_0000001995626574.png create mode 100644 docs/dli/umn/en-us_image_0000001995786314.png create mode 100644 docs/dli/umn/en-us_image_0000001995796834.png rename docs/dli/umn/{en-us_image_0000001254369651.png => en-us_image_0000002031849645.png} (100%) rename docs/dli/umn/{en-us_image_0000001265889586.png => en-us_image_0000002031969193.png} (100%) create mode 100644 docs/dli/umn/en-us_image_0000002032113853.png create mode 100644 docs/dli/umn/en-us_image_0000002032233425.png create mode 100644 docs/dli/umn/en-us_image_0000002032236253.png create mode 100644 docs/dli/umn/en-us_image_0000002079726669.png create mode 100644 docs/dli/umn/en-us_image_0000002112672174.png create mode 100644 docs/dli/umn/en-us_image_0000002112684630.png create mode 100644 docs/dli/umn/en-us_image_0000002129969110.png create mode 100644 docs/dli/umn/en-us_image_0000002132498398.png delete mode 100644 docs/dli/umn/en-us_image_0276441461.png diff --git a/docs/dli/umn/ALL_META.TXT.json b/docs/dli/umn/ALL_META.TXT.json index e09a1d5bc..4b3478871 100644 --- a/docs/dli/umn/ALL_META.TXT.json +++ b/docs/dli/umn/ALL_META.TXT.json @@ -14,7 +14,10 @@ "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Service Overview", @@ -27,17 +30,18 @@ "code":"2", "des":"Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark and Apache Flink ecosystems. It frees you from managing an", "doc_type":"usermanual", - "kw":"What Is Data Lake Insight?,Service Overview,User Guide", + "kw":"What Is Data Lake Insight,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"What Is Data Lake Insight?", + "title":"What Is Data Lake Insight", "githuburl":"" }, { @@ -52,9 +56,10 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Advantages", @@ -72,9 +77,10 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Application Scenarios", @@ -85,26 +91,48 @@ "node_id":"dli_07_0005.xml", "product_code":"dli", "code":"5", - "des":"Only the latest 100 jobs are displayed on DLI's SparkUI.A maximum of 1,000 job results can be displayed on the console. To view more or all jobs, export the job data to O", + "des":"For more notes and constraints on elastic resource pools, see Notes and Constraints on Using an Elastic Resource Pool.For more notes and constraints on using a queue, see", "doc_type":"usermanual", "kw":"Notes and Constraints,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Notes and Constraints", "githuburl":"" }, + { + "uri":"dli_07_0027.html", + "node_id":"dli_07_0027.xml", + "product_code":"dli", + "code":"6", + "des":"An elastic resource pool provides compute resources (CPU and memory) for running DLI jobs. The unit is CU. One CU contains one CPU and 4 GB memory.You can create multiple", + "doc_type":"usermanual", + "kw":"Specifications,Service Overview,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Specifications", + "githuburl":"" + }, { "uri":"dli_07_0006.html", "node_id":"dli_07_0006.xml", "product_code":"dli", - "code":"6", + "code":"7", "des":"If you need to assign different permissions to employees in your enterprise to access your DLI resources, IAM is a good choice for fine-grained permissions management. IA", "doc_type":"usermanual", "kw":"Permissions Management,Service Overview,User Guide", @@ -112,9 +140,10 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Permissions Management", @@ -124,7 +153,7 @@ "uri":"dli_07_0009.html", "node_id":"dli_07_0009.xml", "product_code":"dli", - "code":"7", + "code":"8", "des":"A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.You can also request for an increased quota if your e", "doc_type":"usermanual", "kw":"Quotas,Service Overview,User Guide", @@ -132,9 +161,10 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Quotas", @@ -144,17 +174,18 @@ "uri":"dli_07_0003.html", "node_id":"dli_07_0003.xml", "product_code":"dli", - "code":"8", - "des":"DLI allows multiple organizations, departments, or applications to share resources. A logical entity, also called a tenant, is provided to use diverse resources and servi", + "code":"9", + "des":"Dedicated computing resources. They are isolated by resource pools and can only be shared by queues in the same elastic resource pool. You can set scaling policies of dif", "doc_type":"usermanual", "kw":"Basic Concepts,Service Overview,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Basic Concepts", @@ -164,7 +195,7 @@ "uri":"dli_01_0220.html", "node_id":"dli_01_0220.xml", "product_code":"dli", - "code":"9", + "code":"10", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Getting Started", @@ -172,7 +203,10 @@ "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Getting Started", @@ -182,60 +216,42 @@ "uri":"dli_01_0002.html", "node_id":"dli_01_0002.xml", "product_code":"dli", - "code":"10", - "des":"DLI can query data stored in OBS. This section describes how to us a Spark SQL job on DLI to query OBS data.You can use DLI to submit a Spark SQL job to query data. The g", - "doc_type":"usermanual", - "kw":"Creating and Submitting a Spark SQL Job,Getting Started,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Creating and Submitting a Spark SQL Job", - "githuburl":"" - }, - { - "uri":"dli_01_0512.html", - "node_id":"dli_01_0512.xml", - "product_code":"dli", "code":"11", - "des":"DLI allows you to customize query templates or save frequently used SQL statements as templates to facilitate SQL operations. After templates are saved, you do not need t", + "des":"DLI can query data stored in OBS. This section describes how to use DLI to submit a SQL job to query OBS data.To illustrate, create a new file called sampledata.csv and u", "doc_type":"usermanual", - "kw":"Developing and Submitting a Spark SQL Job Using the TPC-H Sample Template,Getting Started,User Guide", + "kw":"Using DLI to Submit a SQL Job to Query OBS Data,Getting Started,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Developing and Submitting a Spark SQL Job Using the TPC-H Sample Template", + "title":"Using DLI to Submit a SQL Job to Query OBS Data", "githuburl":"" }, { - "uri":"dli_01_0375.html", - "node_id":"dli_01_0375.xml", + "uri":"dli_01_0020.html", + "node_id":"dli_01_0020.xml", "product_code":"dli", "code":"12", - "des":"DLI can query data stored in OBS. This section describes how to use a Spark Jar job on DLI to query OBS data in real time.You can use DLI to submit Spark jobs for real-ti", + "des":"DLI can query data stored in RDS. This section describes how to use DLI to submit a SQL job to query RDS for MySQL data.In this example, we will create an RDS for MySQL D", "doc_type":"usermanual", - "kw":"Creating and Submitting a Spark Jar Job,Getting Started,User Guide", + "kw":"Using DLI to Submit a SQL Job to Query RDS for MySQL Data,Getting Started,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Creating and Submitting a Spark Jar Job", + "title":"Using DLI to Submit a SQL Job to Query RDS for MySQL Data", "githuburl":"" }, { @@ -243,59 +259,62 @@ "node_id":"dli_01_0531.xml", "product_code":"dli", "code":"13", - "des":"DLI Flink jobs can use other cloud services as data sources and sink streams for real-time compute. This example describes how to create and submit a Flink Opensource SQL", + "des":"DLI Flink jobs can use other cloud services as data sources and sink streams for real-time compute.This example describes how to create and submit a Flink OpenSource SQL ", "doc_type":"usermanual", - "kw":"Creating and Submitting a Flink OpenSource SQL Job,Getting Started,User Guide", + "kw":"Using DLI to Submit a Flink OpenSource SQL Job to Query RDS for MySQL Data,Getting Started,User Guid", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Creating and Submitting a Flink OpenSource SQL Job", + "title":"Using DLI to Submit a Flink OpenSource SQL Job to Query RDS for MySQL Data", "githuburl":"" }, { - "uri":"dli_01_0377.html", - "node_id":"dli_01_0377.xml", + "uri":"dli_01_0512.html", + "node_id":"dli_01_0512.xml", "product_code":"dli", "code":"14", - "des":"The Overview page of the DLI console provides you with the DLI workflow and resource usage.The process of using DLI is as follows:Create a queue.Queues are DLI's compute ", + "des":"Flink Jar jobs are suitable for data analysis scenarios that require custom stream processing logic, complex state management, or integration with specific libraries. You", "doc_type":"usermanual", - "kw":"DLI Console Overview,User Guide", + "kw":"Using DLI to Submit a Flink Jar Job,Getting Started,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"DLI Console Overview", + "title":"Using DLI to Submit a Flink Jar Job", "githuburl":"" }, { - "uri":"dli_01_0320.html", - "node_id":"dli_01_0320.xml", + "uri":"dli_01_0375.html", + "node_id":"dli_01_0375.xml", "product_code":"dli", "code":"15", - "des":"You can edit and run SQL statements in the SQL job editor to execute data query.The editor supports SQL:2003 and is compatible with Spark SQL. For details about the synta", + "des":"DLI allows you to submit Spark jobs compiled as JAR files, which contain the necessary code and dependency information for executing the job. These files are used for spe", "doc_type":"usermanual", - "kw":"SQL Editor,User Guide", + "kw":"Using DLI to Submit a Spark Jar Job,Getting Started,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"SQL Editor", + "title":"Using DLI to Submit a Spark Jar Job", "githuburl":"" }, { @@ -303,834 +322,269 @@ "node_id":"dli_01_0001.xml", "product_code":"dli", "code":"16", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "des":"This chapter walks you through on how to develop a DLI job.To manage fine-grained permissions for your DLI resources using IAM, create an IAM user and grant them permissi", "doc_type":"usermanual", - "kw":"Job Management", + "kw":"DLI Job Development Process,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Job Management", + "title":"DLI Job Development Process", "githuburl":"" }, { - "uri":"dli_01_0567.html", - "node_id":"dli_01_0567.xml", + "uri":"dli_01_0611.html", + "node_id":"dli_01_0611.xml", "product_code":"dli", "code":"17", - "des":"DLI provides the following job types:SQL job: SQL jobs provide you with standard SQL statements and are compatible with Spark SQL and Presto SQL (based on Presto). You ca", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Overview,Job Management,User Guide", + "kw":"Preparations", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Overview", + "title":"Preparations", "githuburl":"" }, { - "uri":"dli_01_0017.html", - "node_id":"dli_01_0017.xml", + "uri":"dli_01_0618.html", + "node_id":"dli_01_0618.xml", "product_code":"dli", "code":"18", - "des":"SQL jobs allow you to execute SQL statements in the SQL job editing window, import data, and export data.SQL job management provides the following functions:Searching for", + "des":"To use DLI, you must first configure permissions.This section applies to the following scenarios:If you use DLI for the first time, configure DLI agency permissions by re", "doc_type":"usermanual", - "kw":"SQL Job Management,Job Management,User Guide", + "kw":"Configuring DLI Agency Permissions,Preparations,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"SQL Job Management", + "title":"Configuring DLI Agency Permissions", "githuburl":"" }, { - "uri":"dli_01_0389.html", - "node_id":"dli_01_0389.xml", + "uri":"dli_01_0418.html", + "node_id":"dli_01_0418.xml", "product_code":"dli", "code":"19", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "des":"You can use Identity and Access Management (IAM) to implement fine-grained permissions control on DLI resources. For details, see Overview.If your cloud account does not ", "doc_type":"usermanual", - "kw":"Flink Job Management", + "kw":"Creating an IAM User and Granting Permissions,Preparations,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Flink Job Management", + "title":"Creating an IAM User and Granting Permissions", "githuburl":"" }, { - "uri":"dli_01_0403.html", - "node_id":"dli_01_0403.xml", + "uri":"dli_01_0536.html", + "node_id":"dli_01_0536.xml", "product_code":"dli", "code":"20", - "des":"On the Job Management page of Flink jobs, you can submit a Flink job. Currently, the following job types are supported:Flink SQL uses SQL statements to define jobs and ca", + "des":"Before using DLI, you need to configure a DLI job bucket. The bucket is used to store temporary data generated during DLI job running, such as job logs and results.Config", "doc_type":"usermanual", - "kw":"Overview,Flink Job Management,User Guide", + "kw":"Configuring a DLI Job Bucket,Preparations,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Overview", - "githuburl":"" - }, - { - "uri":"dli_01_0479.html", - "node_id":"dli_01_0479.xml", - "product_code":"dli", - "code":"21", - "des":"You can isolate Flink jobs allocated to different users by setting permissions to ensure data query performance.The administrator and job creator have all permissions, wh", - "doc_type":"usermanual", - "kw":"Managing Flink Job Permissions,Flink Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Managing Flink Job Permissions", - "githuburl":"" - }, - { - "uri":"dli_01_0454.html", - "node_id":"dli_01_0454.xml", - "product_code":"dli", - "code":"22", - "des":"To create a Flink job, you need to enter the data source and data output channel, that is, source and sink. To use another service as the source or sink stream, you need ", - "doc_type":"usermanual", - "kw":"Preparing Flink Job Data,Flink Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Preparing Flink Job Data", - "githuburl":"" - }, - { - "uri":"dli_01_0498.html", - "node_id":"dli_01_0498.xml", - "product_code":"dli", - "code":"23", - "des":"This section describes how to create a Flink OpenSource SQL job.DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink 1.10 and 1.12 provided by the ", - "doc_type":"usermanual", - "kw":"(Recommended) Creating a Flink OpenSource SQL Job,Flink Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"(Recommended) Creating a Flink OpenSource SQL Job", - "githuburl":"" - }, - { - "uri":"dli_01_0455.html", - "node_id":"dli_01_0455.xml", - "product_code":"dli", - "code":"24", - "des":"This section describes how to create a Flink SQL job. You can use Flink SQLs to develop jobs to meet your service requirements. Using SQL statements simplifies logic impl", - "doc_type":"usermanual", - "kw":"Creating a Flink SQL Job,Flink Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Creating a Flink SQL Job", - "githuburl":"" - }, - { - "uri":"dli_01_0457.html", - "node_id":"dli_01_0457.xml", - "product_code":"dli", - "code":"25", - "des":"This section describes how to create a Flink Jar job. You can perform secondary development based on Flink APIs, build your own JAR file, and submit the JAR file to DLI q", - "doc_type":"usermanual", - "kw":"Creating a Flink Jar Job,Flink Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Creating a Flink Jar Job", - "githuburl":"" - }, - { - "uri":"dli_01_0461.html", - "node_id":"dli_01_0461.xml", - "product_code":"dli", - "code":"26", - "des":"After a job is created, you can perform operations on the job as required.Editing a JobStarting a JobStopping a JobDeleting a JobExporting a JobImporting a JobModifying N", - "doc_type":"usermanual", - "kw":"Performing Operations on a Flink Job,Flink Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Performing Operations on a Flink Job", - "githuburl":"" - }, - { - "uri":"dli_01_0462.html", - "node_id":"dli_01_0462.xml", - "product_code":"dli", - "code":"27", - "des":"After creating a job, you can view the job details to learn about the following information:Viewing Job DetailsChecking Job Monitoring InformationViewing the Task List of", - "doc_type":"usermanual", - "kw":"Flink Job Details,Flink Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Flink Job Details", - "githuburl":"" - }, - { - "uri":"dli_01_0463.html", - "node_id":"dli_01_0463.xml", - "product_code":"dli", - "code":"28", - "des":"A tag is a key-value pair customized by users and used to identify cloud resources. It helps users to classify and search for cloud resources. A tag consists of a tag key", - "doc_type":"usermanual", - "kw":"Tag Management,Flink Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Tag Management", - "githuburl":"" - }, - { - "uri":"dli_01_0534.html", - "node_id":"dli_01_0534.xml", - "product_code":"dli", - "code":"29", - "des":"In actual job operations, the compute resources required by a job vary depending on the data volume. As a result, compute resources are wasted when the volume is small an", - "doc_type":"usermanual", - "kw":"Enabling Dynamic Scaling for Flink Jobs,Flink Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Enabling Dynamic Scaling for Flink Jobs", - "githuburl":"" - }, - { - "uri":"dli_01_0465.html", - "node_id":"dli_01_0465.xml", - "product_code":"dli", - "code":"30", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Spark Job Management", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Spark Job Management", - "githuburl":"" - }, - { - "uri":"dli_01_0385.html", - "node_id":"dli_01_0385.xml", - "product_code":"dli", - "code":"31", - "des":"Based on the open-source Spark, DLI optimizes performance and reconstructs services to be compatible with the Apache Spark ecosystem and interfaces, and executes batch pr", - "doc_type":"usermanual", - "kw":"Spark Job Management,Spark Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Spark Job Management", - "githuburl":"" - }, - { - "uri":"dli_01_0384.html", - "node_id":"dli_01_0384.xml", - "product_code":"dli", - "code":"32", - "des":"DLI provides fully-managed Spark computing services by allowing you to execute Spark jobs.On the Overview page, click Create Job in the upper right corner of the Spark Jo", - "doc_type":"usermanual", - "kw":"Creating a Spark Job,Spark Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Creating a Spark Job", - "githuburl":"" - }, - { - "uri":"dli_01_0535.html", - "node_id":"dli_01_0535.xml", - "product_code":"dli", - "code":"33", - "des":"In actual job operations, jobs have different importance and urgency levels. So, the compute resources required for normal operations of important and urgent jobs need to", - "doc_type":"usermanual", - "kw":"Setting the Priority for a Job,Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Setting the Priority for a Job", - "githuburl":"" - }, - { - "uri":"dli_01_0012.html", - "node_id":"dli_01_0012.xml", - "product_code":"dli", - "code":"34", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Queue Management", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Queue Management", - "githuburl":"" - }, - { - "uri":"dli_01_0402.html", - "node_id":"dli_01_0402.xml", - "product_code":"dli", - "code":"35", - "des":"Queues in DLI are computing resources, which are the basis for using DLI. All executed jobs require computing resources.Currently, DLI provides two types of queues: For S", - "doc_type":"usermanual", - "kw":"Overview,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Overview", - "githuburl":"" - }, - { - "uri":"dli_01_0015.html", - "node_id":"dli_01_0015.xml", - "product_code":"dli", - "code":"36", - "des":"You can isolate queues allocated to different users by setting permissions to ensure data query performance.The administrator and queue owner have all permissions, which ", - "doc_type":"usermanual", - "kw":"Queue Permission Management,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Queue Permission Management", - "githuburl":"" - }, - { - "uri":"dli_01_0363.html", - "node_id":"dli_01_0363.xml", - "product_code":"dli", - "code":"37", - "des":"Before executing a job, you need to create a queue.If you use a sub-account to create a queue for the first time, log in to the DLI management console using the main acco", - "doc_type":"usermanual", - "kw":"Creating a Queue,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Creating a Queue", - "githuburl":"" - }, - { - "uri":"dli_01_0016.html", - "node_id":"dli_01_0016.xml", - "product_code":"dli", - "code":"38", - "des":"You can delete a queue based on actual conditions.This operation will fail if there are jobs in the Submitting or Running state on this queue.Deleting a queue does not ca", - "doc_type":"usermanual", - "kw":"Deleting a Queue,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Deleting a Queue", - "githuburl":"" - }, - { - "uri":"dli_01_0565.html", - "node_id":"dli_01_0565.xml", - "product_code":"dli", - "code":"39", - "des":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", - "doc_type":"usermanual", - "kw":"Allocating a Queue to an Enterprise Project,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Allocating a Queue to an Enterprise Project", - "githuburl":"" - }, - { - "uri":"dli_01_0443.html", - "node_id":"dli_01_0443.xml", - "product_code":"dli", - "code":"40", - "des":"If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.If the queue whose CIDR block is to be modified", - "doc_type":"usermanual", - "kw":"Modifying the CIDR Block,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Modifying the CIDR Block", - "githuburl":"" - }, - { - "uri":"dli_01_0487.html", - "node_id":"dli_01_0487.xml", - "product_code":"dli", - "code":"41", - "des":"Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.Queues with 16 CUs do not support scale-out or scale-in.Queues ", - "doc_type":"usermanual", - "kw":"Elastic Scaling of Queues,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Elastic Scaling of Queues", - "githuburl":"" - }, - { - "uri":"dli_01_0488.html", - "node_id":"dli_01_0488.xml", - "product_code":"dli", - "code":"42", - "des":"When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. ", - "doc_type":"usermanual", - "kw":"Scheduling CU Changes,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Scheduling CU Changes", - "githuburl":"" - }, - { - "uri":"dli_01_0489.html", - "node_id":"dli_01_0489.xml", - "product_code":"dli", - "code":"43", - "des":"It can be used to test the connectivity between the DLI queue and the peer IP address specified by the user in common scenarios, or the connectivity between the DLI queue", - "doc_type":"usermanual", - "kw":"Testing Address Connectivity,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Testing Address Connectivity", - "githuburl":"" - }, - { - "uri":"dli_01_0421.html", - "node_id":"dli_01_0421.xml", - "product_code":"dli", - "code":"44", - "des":"Once you have created an SMN topic, you can easily subscribe to it by going to the Topic Management > Topics page of the SMN console. You can choose to receive notificati", - "doc_type":"usermanual", - "kw":"Creating an SMN Topic,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Creating an SMN Topic", - "githuburl":"" - }, - { - "uri":"dli_01_0022.html", - "node_id":"dli_01_0022.xml", - "product_code":"dli", - "code":"45", - "des":"A tag is a key-value pair that you can customize to identify cloud resources. It helps you to classify and search for cloud resources. A tag consists of a tag key and a t", - "doc_type":"usermanual", - "kw":"Managing Queue Tags,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Managing Queue Tags", - "githuburl":"" - }, - { - "uri":"dli_01_0563.html", - "node_id":"dli_01_0563.xml", - "product_code":"dli", - "code":"46", - "des":"DLI allows you to set properties for queues.You can set Spark driver parameters to improve the scheduling efficiency of queues.This section describes how to set queue pro", - "doc_type":"usermanual", - "kw":"Setting Queue Properties,Queue Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Setting Queue Properties", + "title":"Configuring a DLI Job Bucket", "githuburl":"" }, { "uri":"dli_01_0508.html", "node_id":"dli_01_0508.xml", "product_code":"dli", - "code":"47", + "code":"21", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Elastic Resource Pool", + "kw":"Creating an Elastic Resource Pool and Queues Within It", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Elastic Resource Pool", - "githuburl":"" - }, - { - "uri":"dli_01_0528.html", - "node_id":"dli_01_0528.xml", - "product_code":"dli", - "code":"48", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Before You Start", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Before You Start", + "title":"Creating an Elastic Resource Pool and Queues Within It", "githuburl":"" }, { "uri":"dli_01_0504.html", "node_id":"dli_01_0504.xml", "product_code":"dli", - "code":"49", - "des":"An elastic resource pool provides compute resources (CPU and memory) for running DLI jobs. The unit is CU. One CU contains one CPU and 4 GB memory.You can create multiple", + "code":"22", + "des":"DLI compute resources are the foundation to run jobs. This section describes the modes of DLI compute resources and queue types.Before we dive into the compute resource m", "doc_type":"usermanual", - "kw":"Overview,Before You Start,User Guide", + "kw":"Overview of DLI Elastic Resource Pools and Queues,Creating an Elastic Resource Pool and Queues Withi", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Overview", - "githuburl":"" - }, - { - "uri":"dli_01_0515.html", - "node_id":"dli_01_0515.xml", - "product_code":"dli", - "code":"50", - "des":"This section walks you through the procedure of adding a queue to an elastic resource pool and binding an enhanced datasource connection to the elastic resource pool.Proc", - "doc_type":"usermanual", - "kw":"Creating an Elastic Resource Pool and Running a Job,Before You Start,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Creating an Elastic Resource Pool and Running a Job", - "githuburl":"" - }, - { - "uri":"dli_01_0516.html", - "node_id":"dli_01_0516.xml", - "product_code":"dli", - "code":"51", - "des":"A company has multiple departments that perform data analysis in different periods during a day.Department A requires a large number of compute resources from 00:00 a.m. ", - "doc_type":"usermanual", - "kw":"Configuring Scaling Policies for Queues,Before You Start,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Configuring Scaling Policies for Queues", - "githuburl":"" - }, - { - "uri":"dli_01_0529.html", - "node_id":"dli_01_0529.xml", - "product_code":"dli", - "code":"52", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Regular Operations", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Regular Operations", + "title":"Overview of DLI Elastic Resource Pools and Queues", "githuburl":"" }, { "uri":"dli_01_0505.html", "node_id":"dli_01_0505.xml", "product_code":"dli", - "code":"53", - "des":"For details about the application scenarios of elastic resource pools, see the Overview. This section describes how to create an elastic resource pool.If you use an enhan", + "code":"23", + "des":"An elastic resource pool offers compute resources (CPU and memory) required for running DLI jobs, which can adapt to the changing demands of services.You can create multi", "doc_type":"usermanual", - "kw":"Creating an Elastic Resource Pool,Regular Operations,User Guide", + "kw":"Creating an Elastic Resource Pool and Creating Queues Within It,Creating an Elastic Resource Pool an", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Creating an Elastic Resource Pool", + "title":"Creating an Elastic Resource Pool and Creating Queues Within It", + "githuburl":"" + }, + { + "uri":"dli_01_0529.html", + "node_id":"dli_01_0529.xml", + "product_code":"dli", + "code":"24", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Managing Elastic Resource Pools", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Managing Elastic Resource Pools", + "githuburl":"" + }, + { + "uri":"dli_01_0622.html", + "node_id":"dli_01_0622.xml", + "product_code":"dli", + "code":"25", + "des":"After creating an elastic resource pool, you can check and manage it on the management console.This section describes how to check basic information about an elastic reso", + "doc_type":"usermanual", + "kw":"Viewing Basic Information,Managing Elastic Resource Pools,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Viewing Basic Information", "githuburl":"" }, { "uri":"dli_01_0526.html", "node_id":"dli_01_0526.xml", "product_code":"dli", - "code":"54", + "code":"26", "des":"Administrators can assign permissions of different operation scopes to users for each elastic resource pool.The administrator and elastic resource pool owner have all per", "doc_type":"usermanual", - "kw":"Managing Permissions,Regular Operations,User Guide", + "kw":"Managing Permissions,Managing Elastic Resource Pools,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Managing Permissions", "githuburl":"" }, - { - "uri":"dli_01_0509.html", - "node_id":"dli_01_0509.xml", - "product_code":"dli", - "code":"55", - "des":"You can add one or more queues to an elastic resource pool to run jobs. This section describes how to add a queue to an elastic resource pool.Automatic scaling of an elas", - "doc_type":"usermanual", - "kw":"Adding a Queue,Regular Operations,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Adding a Queue", - "githuburl":"" - }, { "uri":"dli_01_0530.html", "node_id":"dli_01_0530.xml", "product_code":"dli", - "code":"56", + "code":"27", "des":"If you want a queue to use resources in an elastic resource pool, bind the queue to the pool.You can click Associate Queue on the Resource Pool page to bind a queue to an", "doc_type":"usermanual", - "kw":"Binding a Queue,Regular Operations,User Guide", + "kw":"Binding a Queue,Managing Elastic Resource Pools,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Binding a Queue", "githuburl":"" }, - { - "uri":"dli_01_0506.html", - "node_id":"dli_01_0506.xml", - "product_code":"dli", - "code":"57", - "des":"Multiple queues can be added to an elastic resource pool. For details about how to add a queue, see Adding a Queue. You can configure the number of CUs you want based on ", - "doc_type":"usermanual", - "kw":"Managing Queues,Regular Operations,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Managing Queues", - "githuburl":"" - }, { "uri":"dli_01_0507.html", "node_id":"dli_01_0507.xml", "product_code":"dli", - "code":"58", + "code":"28", "des":"CU settings are used to control the maximum and minimum CU ranges for elastic resource pools to avoid unlimited resource scaling.For example, an elastic resource pool has", "doc_type":"usermanual", - "kw":"Setting CUs,Regular Operations,User Guide", + "kw":"Setting CUs,Managing Elastic Resource Pools,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Setting CUs", @@ -1140,17 +594,18 @@ "uri":"dli_01_0524.html", "node_id":"dli_01_0524.xml", "product_code":"dli", - "code":"59", - "des":"If the current specifications of your elastic resource pool do not meet your service needs, you can modify them using the change specifications function.In the navigation", + "code":"29", + "des":"If the current specifications of your elastic resource pool do not meet your service needs, you can modify them using the change yearly/monthly CUs function.In the naviga", "doc_type":"usermanual", - "kw":"Modifying Specifications,Regular Operations,User Guide", + "kw":"Modifying Specifications,Managing Elastic Resource Pools,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Modifying Specifications", @@ -1160,37 +615,60 @@ "uri":"dli_01_0525.html", "node_id":"dli_01_0525.xml", "product_code":"dli", - "code":"60", + "code":"30", "des":"A tag is a key-value pair that you can customize to identify cloud resources. It helps you to classify and search for cloud resources. A tag consists of a tag key and a t", "doc_type":"usermanual", - "kw":"Managing Tags,Regular Operations,User Guide", + "kw":"Managing Tags,Managing Elastic Resource Pools,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Managing Tags", "githuburl":"" }, { - "uri":"dli_01_0532.html", - "node_id":"dli_01_0532.xml", + "uri":"dli_01_0506.html", + "node_id":"dli_01_0506.xml", "product_code":"dli", - "code":"61", - "des":"If you added a queue to or deleted one from an elastic resource pool, or you scaled an added queue, the CU quantity of the elastic resource pool may be changed. You can v", + "code":"31", + "des":"Multiple queues can be added to an elastic resource pool. For how to add a queue, see Creating an Elastic Resource Pool and Creating Queues Within It. You can configure t", "doc_type":"usermanual", - "kw":"Viewing Scaling History,Regular Operations,User Guide", + "kw":"Adjusting Scaling Policies for Queues in an Elastic Resource Pool,Managing Elastic Resource Pools,Us", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Adjusting Scaling Policies for Queues in an Elastic Resource Pool", + "githuburl":"" + }, + { + "uri":"dli_01_0532.html", + "node_id":"dli_01_0532.xml", + "product_code":"dli", + "code":"32", + "des":"If you added a queue to or deleted one from an elastic resource pool, or you scaled an added queue, the CU quantity of the elastic resource pool may be changed. You can v", + "doc_type":"usermanual", + "kw":"Viewing Scaling History,Managing Elastic Resource Pools,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Viewing Scaling History", @@ -1200,816 +678,900 @@ "uri":"dli_01_0566.html", "node_id":"dli_01_0566.xml", "product_code":"dli", - "code":"62", + "code":"33", "des":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", "doc_type":"usermanual", - "kw":"Allocating to an Enterprise Project,Regular Operations,User Guide", + "kw":"Allocating to an Enterprise Project,Managing Elastic Resource Pools,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Allocating to an Enterprise Project", "githuburl":"" }, { - "uri":"dli_01_0004.html", - "node_id":"dli_01_0004.xml", + "uri":"dli_01_0528.html", + "node_id":"dli_01_0528.xml", "product_code":"dli", - "code":"63", + "code":"34", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Data Management", + "kw":"Managing Queues", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Data Management", + "title":"Managing Queues", + "githuburl":"" + }, + { + "uri":"dli_01_0663.html", + "node_id":"dli_01_0663.xml", + "product_code":"dli", + "code":"35", + "des":"This section walks you through how to view basic information about a queue on the management console, including the engine type and version.Log in to the DLI management c", + "doc_type":"usermanual", + "kw":"Viewing Basic Information About a Queue,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Viewing Basic Information About a Queue", + "githuburl":"" + }, + { + "uri":"dli_01_0015.html", + "node_id":"dli_01_0015.xml", + "product_code":"dli", + "code":"36", + "des":"Administrators and queue owners have full operation permissions on queues. They can grant operation permissions to other users based on service needs. This ensures that u", + "doc_type":"usermanual", + "kw":"Queue Permission Management,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Queue Permission Management", + "githuburl":"" + }, + { + "uri":"dli_01_0565.html", + "node_id":"dli_01_0565.xml", + "product_code":"dli", + "code":"37", + "des":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", + "doc_type":"usermanual", + "kw":"Allocating a Queue to an Enterprise Project,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Allocating a Queue to an Enterprise Project", + "githuburl":"" + }, + { + "uri":"dli_01_0421.html", + "node_id":"dli_01_0421.xml", + "product_code":"dli", + "code":"38", + "des":"Once you have created an SMN topic, you can easily subscribe to it by going to the Topic Management > Topics page of the SMN console. You can choose to receive notificati", + "doc_type":"usermanual", + "kw":"Creating an SMN Topic,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating an SMN Topic", + "githuburl":"" + }, + { + "uri":"dli_01_0022.html", + "node_id":"dli_01_0022.xml", + "product_code":"dli", + "code":"39", + "des":"A tag is a key-value pair that you can customize to identify cloud resources. It helps you to classify and search for cloud resources. A tag consists of a tag key and a t", + "doc_type":"usermanual", + "kw":"Managing Queue Tags,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Managing Queue Tags", + "githuburl":"" + }, + { + "uri":"dli_01_0563.html", + "node_id":"dli_01_0563.xml", + "product_code":"dli", + "code":"40", + "des":"DLI allows you to set properties for queues.You can set Spark driver parameters to improve the scheduling efficiency of queues.This section describes how to set queue pro", + "doc_type":"usermanual", + "kw":"Setting Queue Properties,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Setting Queue Properties", + "githuburl":"" + }, + { + "uri":"dli_01_0489.html", + "node_id":"dli_01_0489.xml", + "product_code":"dli", + "code":"41", + "des":"DLI's address connectivity testing feature can be used to verify network connectivity between DLI queues and destination addresses.This feature is typically utilized for ", + "doc_type":"usermanual", + "kw":"Testing Address Connectivity,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Testing Address Connectivity", + "githuburl":"" + }, + { + "uri":"dli_01_0016.html", + "node_id":"dli_01_0016.xml", + "product_code":"dli", + "code":"42", + "des":"You can delete a queue based on actual conditions.This operation will fail if there are jobs in the Submitting or Running state on this queue.Deleting a queue does not ca", + "doc_type":"usermanual", + "kw":"Deleting a Queue,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Deleting a Queue", + "githuburl":"" + }, + { + "uri":"dli_01_0487.html", + "node_id":"dli_01_0487.xml", + "product_code":"dli", + "code":"43", + "des":"Newly created queues need to run jobs before they can be scaled in or out.The operations described in this section only apply to standard queues.Queues with 16 CUs do not", + "doc_type":"usermanual", + "kw":"Auto Scaling of Standard Queues,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Auto Scaling of Standard Queues", + "githuburl":"" + }, + { + "uri":"dli_01_0488.html", + "node_id":"dli_01_0488.xml", + "product_code":"dli", + "code":"44", + "des":"When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. ", + "doc_type":"usermanual", + "kw":"Setting a Scheduled Auto Scaling Task for a Standard Queue,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Setting a Scheduled Auto Scaling Task for a Standard Queue", + "githuburl":"" + }, + { + "uri":"dli_01_0443.html", + "node_id":"dli_01_0443.xml", + "product_code":"dli", + "code":"45", + "des":"If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.If the queue whose CIDR block is to be modified", + "doc_type":"usermanual", + "kw":"Changing the CIDR Block for a Standard Queue,Managing Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Changing the CIDR Block for a Standard Queue", + "githuburl":"" + }, + { + "uri":"dli_01_0515.html", + "node_id":"dli_01_0515.xml", + "product_code":"dli", + "code":"46", + "des":"This section walks you through the procedure of adding a queue to an elastic resource pool and binding an enhanced datasource connection to the elastic resource pool.Proc", + "doc_type":"usermanual", + "kw":"Example Use Case: Creating an Elastic Resource Pool and Running Jobs,Creating an Elastic Resource Po", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Example Use Case: Creating an Elastic Resource Pool and Running Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0516.html", + "node_id":"dli_01_0516.xml", + "product_code":"dli", + "code":"47", + "des":"A company has multiple departments that perform data analysis in different periods during a day.Department A requires a large number of compute resources from 00:00 a.m. ", + "doc_type":"usermanual", + "kw":"Example Use Case: Configuring Scaling Policies for Queues in an Elastic Resource Pool,Creating an El", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Example Use Case: Configuring Scaling Policies for Queues in an Elastic Resource Pool", + "githuburl":"" + }, + { + "uri":"dli_01_0363.html", + "node_id":"dli_01_0363.xml", + "product_code":"dli", + "code":"48", + "des":"Queues in the non-elastic resource pool mode are the previous-gen of resource management for DLI. It involved purchasing and releasing resources based on usage demands, r", + "doc_type":"usermanual", + "kw":"Creating a Non-Elastic Resource Pool Queue (Discarded and Not Recommended),Creating an Elastic Resou", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating a Non-Elastic Resource Pool Queue (Discarded and Not Recommended)", "githuburl":"" }, { "uri":"dli_01_0390.html", "node_id":"dli_01_0390.xml", "product_code":"dli", - "code":"64", + "code":"49", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Databases and Tables", + "kw":"Creating Databases and Tables", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Databases and Tables", + "title":"Creating Databases and Tables", "githuburl":"" }, { "uri":"dli_01_0228.html", "node_id":"dli_01_0228.xml", "product_code":"dli", - "code":"65", - "des":"DLI database and table management provide the following functions:Database Permission ManagementTable Permission ManagementCreating a Database or a TableDeleting a Databa", + "code":"50", + "des":"Databases and tables are the basis for developing SQL and Spark jobs. Before running a job, you need to define databases and tables based on your service scenarios.Flink ", "doc_type":"usermanual", - "kw":"Overview,Databases and Tables,User Guide", + "kw":"Understanding Data Catalogs, Databases, and Tables,Creating Databases and Tables,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Overview", - "githuburl":"" - }, - { - "uri":"dli_01_0447.html", - "node_id":"dli_01_0447.xml", - "product_code":"dli", - "code":"66", - "des":"By setting permissions, you can assign varying database permissions to different users.The administrator and database owner have all permissions, which cannot be set or m", - "doc_type":"usermanual", - "kw":"Managing Database Permissions,Databases and Tables,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Managing Database Permissions", - "githuburl":"" - }, - { - "uri":"dli_01_0448.html", - "node_id":"dli_01_0448.xml", - "product_code":"dli", - "code":"67", - "des":"By setting permissions, you can assign varying table permissions to different users.The administrator and table owner have all permissions, which cannot be set or modifie", - "doc_type":"usermanual", - "kw":"Managing Table Permissions,Databases and Tables,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Managing Table Permissions", + "title":"Understanding Data Catalogs, Databases, and Tables", "githuburl":"" }, { "uri":"dli_01_0005.html", "node_id":"dli_01_0005.xml", "product_code":"dli", - "code":"68", + "code":"51", "des":"A database, built on the computer storage device, is a data warehouse where data is organized, stored, and managed based on its structure.The table is an important part o", "doc_type":"usermanual", - "kw":"Creating a Database or a Table,Databases and Tables,User Guide", + "kw":"Creating a Database and Table on the DLI Console,Creating Databases and Tables,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Creating a Database or a Table", - "githuburl":"" - }, - { - "uri":"dli_01_0011.html", - "node_id":"dli_01_0011.xml", - "product_code":"dli", - "code":"69", - "des":"You can delete unnecessary databases and tables based on actual conditions.You are not allowed to delete databases or tables that are being used for running jobs.The admi", - "doc_type":"usermanual", - "kw":"Deleting a Database or a Table,Databases and Tables,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Deleting a Database or a Table", - "githuburl":"" - }, - { - "uri":"dli_01_0376.html", - "node_id":"dli_01_0376.xml", - "product_code":"dli", - "code":"70", - "des":"During actual use, developers create databases and tables and submit them to test personnel for testing. After the test is complete, the databases and tables are transfer", - "doc_type":"usermanual", - "kw":"Modifying the Owners of Databases and Tables,Databases and Tables,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Modifying the Owners of Databases and Tables", - "githuburl":"" - }, - { - "uri":"dli_01_0253.html", - "node_id":"dli_01_0253.xml", - "product_code":"dli", - "code":"71", - "des":"You can import data from OBS to a table created in DLI.Only one path can be specified during data import. The path cannot contain commas (,).To import data in CSV format ", - "doc_type":"usermanual", - "kw":"Importing Data to the Table,Databases and Tables,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Importing Data to the Table", - "githuburl":"" - }, - { - "uri":"dli_01_0010.html", - "node_id":"dli_01_0010.xml", - "product_code":"dli", - "code":"72", - "des":"You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.The exported file can be i", - "doc_type":"usermanual", - "kw":"Exporting Data from DLI to OBS,Databases and Tables,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Exporting Data from DLI to OBS", + "title":"Creating a Database and Table on the DLI Console", "githuburl":"" }, { "uri":"dli_01_0008.html", "node_id":"dli_01_0008.xml", "product_code":"dli", - "code":"73", + "code":"52", "des":"Metadata is used to define data types. It describes information about the data, including the source, size, format, and other data features. In database fields, metadata ", "doc_type":"usermanual", - "kw":"Viewing Metadata,Databases and Tables,User Guide", + "kw":"Viewing Table Metadata,Creating Databases and Tables,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Viewing Metadata", + "title":"Viewing Table Metadata", "githuburl":"" }, { - "uri":"dli_01_0007.html", - "node_id":"dli_01_0007.xml", + "uri":"dli_01_0004.html", + "node_id":"dli_01_0004.xml", "product_code":"dli", - "code":"74", - "des":"The Preview page displays the first 10 records in the table.You can preview data on either the Data Management page or the SQL Editor page.To preview data on the Data Man", + "code":"53", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Previewing Data,Databases and Tables,User Guide", + "kw":"Managing Database Resources on the DLI Console", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Previewing Data", + "title":"Managing Database Resources on the DLI Console", + "githuburl":"" + }, + { + "uri":"dli_01_0447.html", + "node_id":"dli_01_0447.xml", + "product_code":"dli", + "code":"54", + "des":"By setting permissions, you can assign varying database permissions to different users.The administrator and database owner have all permissions, which cannot be set or m", + "doc_type":"usermanual", + "kw":"Configuring Database Permissions on the DLI Console,Managing Database Resources on the DLI Console,U", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Configuring Database Permissions on the DLI Console", + "githuburl":"" + }, + { + "uri":"dli_01_0011.html", + "node_id":"dli_01_0011.xml", + "product_code":"dli", + "code":"55", + "des":"You can delete databases as needed.You are not allowed to delete databases or tables that are being used for running jobs.The administrator, database owner, and users wit", + "doc_type":"usermanual", + "kw":"Deleting a Database on the DLI Console,Managing Database Resources on the DLI Console,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Deleting a Database on the DLI Console", + "githuburl":"" + }, + { + "uri":"dli_01_0376.html", + "node_id":"dli_01_0376.xml", + "product_code":"dli", + "code":"56", + "des":"In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed", + "doc_type":"usermanual", + "kw":"Changing the Database Owner on the DLI Console,Managing Database Resources on the DLI Console,User G", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Changing the Database Owner on the DLI Console", "githuburl":"" }, { "uri":"dli_01_0552.html", "node_id":"dli_01_0552.xml", "product_code":"dli", - "code":"75", + "code":"57", "des":"A tag is a key-value pair that you can customize to identify cloud resources. It helps you to classify and search for cloud resources. A tag consists of a tag key and a t", "doc_type":"usermanual", - "kw":"Managing Tags,Databases and Tables,User Guide", + "kw":"Managing Tags,Managing Database Resources on the DLI Console,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Managing Tags", "githuburl":"" }, { - "uri":"dli_01_0366.html", - "node_id":"dli_01_0366.xml", + "uri":"dli_01_0625.html", + "node_id":"dli_01_0625.xml", "product_code":"dli", - "code":"76", + "code":"58", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Package Management", + "kw":"Managing Table Resources on the DLI Console", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Package Management", + "title":"Managing Table Resources on the DLI Console", "githuburl":"" }, { - "uri":"dli_01_0407.html", - "node_id":"dli_01_0407.xml", + "uri":"dli_01_0448.html", + "node_id":"dli_01_0448.xml", "product_code":"dli", - "code":"77", - "des":"Package management provides the following functions:Managing Package PermissionsCreating a PackageDeleting a PackageYou can delete program packages in batches.You can del", + "code":"59", + "des":"By setting permissions, you can assign varying table permissions to different users.The administrator and table owner have all permissions, which cannot be set or modifie", "doc_type":"usermanual", - "kw":"Overview,Package Management,User Guide", + "kw":"Configuring Table Permissions on the DLI Console,Managing Table Resources on the DLI Console,User Gu", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Configuring Table Permissions on the DLI Console", + "githuburl":"" + }, + { + "uri":"dli_01_0626.html", + "node_id":"dli_01_0626.xml", + "product_code":"dli", + "code":"60", + "des":"You can delete tables as needed.Databases or tables that are being used for running jobs cannot be deleted.Only administrators, table owners, and users with table deletio", + "doc_type":"usermanual", + "kw":"Deleting a Table on the DLI Console,Managing Table Resources on the DLI Console,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Deleting a Table on the DLI Console", + "githuburl":"" + }, + { + "uri":"dli_01_0627.html", + "node_id":"dli_01_0627.xml", + "product_code":"dli", + "code":"61", + "des":"In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed", + "doc_type":"usermanual", + "kw":"Changing the Table Owner on the DLI Console,Managing Table Resources on the DLI Console,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Changing the Table Owner on the DLI Console", + "githuburl":"" + }, + { + "uri":"dli_01_0253.html", + "node_id":"dli_01_0253.xml", + "product_code":"dli", + "code":"62", + "des":"This section describes how to import data stored in OBS to a table on the DLI console.Only one path can be specified during data import. The path cannot contain commas (,", + "doc_type":"usermanual", + "kw":"Importing OBS Data to DLI,Managing Table Resources on the DLI Console,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Importing OBS Data to DLI", + "githuburl":"" + }, + { + "uri":"dli_01_0010.html", + "node_id":"dli_01_0010.xml", + "product_code":"dli", + "code":"63", + "des":"You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.The exported file can be i", + "doc_type":"usermanual", + "kw":"Exporting DLI Table Data to OBS,Managing Table Resources on the DLI Console,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Exporting DLI Table Data to OBS", + "githuburl":"" + }, + { + "uri":"dli_01_0007.html", + "node_id":"dli_01_0007.xml", + "product_code":"dli", + "code":"64", + "des":"The Preview page displays the first 10 records in the table.You can preview data on either the Data Management page or the SQL Editor page.To preview data on the Data Man", + "doc_type":"usermanual", + "kw":"Previewing Table Data on the DLI Console,Managing Table Resources on the DLI Console,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Previewing Table Data on the DLI Console", + "githuburl":"" + }, + { + "uri":"dli_01_0661.html", + "node_id":"dli_01_0661.xml", + "product_code":"dli", + "code":"65", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Data Migration and Transmission", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Data Migration and Transmission", + "githuburl":"" + }, + { + "uri":"dli_01_0662.html", + "node_id":"dli_01_0662.xml", + "product_code":"dli", + "code":"66", + "des":"DLI enables direct access to data stored in OBS for query and analysis, eliminating the need for data migration.To begin using DLI for data analysis, just import your loc", + "doc_type":"usermanual", + "kw":"Overview,Data Migration and Transmission,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Overview", "githuburl":"" }, { - "uri":"dli_01_0477.html", - "node_id":"dli_01_0477.xml", + "uri":"dli_01_0634.html", + "node_id":"dli_01_0634.xml", "product_code":"dli", - "code":"78", - "des":"You can isolate package groups or packages allocated to different users by setting permissions to ensure data query performance.The administrator and the owner of a packa", - "doc_type":"usermanual", - "kw":"Managing Permissions on Packages and Package Groups,Package Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Managing Permissions on Packages and Package Groups", - "githuburl":"" - }, - { - "uri":"dli_01_0367.html", - "node_id":"dli_01_0367.xml", - "product_code":"dli", - "code":"79", - "des":"DLI allows you to submit program packages in batches to the general-use queue for running.If you need to update a package, you can use the same package or file to upload ", - "doc_type":"usermanual", - "kw":"Creating a Package,Package Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Creating a Package", - "githuburl":"" - }, - { - "uri":"dli_01_0369.html", - "node_id":"dli_01_0369.xml", - "product_code":"dli", - "code":"80", - "des":"You can delete a package based on actual conditions.On the left of the management console, choose Data Management > Package Management.Click Delete in the Operation colum", - "doc_type":"usermanual", - "kw":"Deleting a Package,Package Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Deleting a Package", - "githuburl":"" - }, - { - "uri":"dli_01_0478.html", - "node_id":"dli_01_0478.xml", - "product_code":"dli", - "code":"81", - "des":"To change the owner of a package, click More > Modify Owner in the Operation column of a package on the Package Management page.If the package has been grouped, you can m", - "doc_type":"usermanual", - "kw":"Modifying the Owner,Package Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Modifying the Owner", - "githuburl":"" - }, - { - "uri":"dli_01_0397.html", - "node_id":"dli_01_0397.xml", - "product_code":"dli", - "code":"82", - "des":"DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar ", - "doc_type":"usermanual", - "kw":"Built-in Dependencies,Package Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Built-in Dependencies", - "githuburl":"" - }, - { - "uri":"dli_01_0379.html", - "node_id":"dli_01_0379.xml", - "product_code":"dli", - "code":"83", + "code":"67", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Job Templates", + "kw":"Configuring DLI to Read and Write Data from and to External Data Sources", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Job Templates", - "githuburl":"" - }, - { - "uri":"dli_01_0021.html", - "node_id":"dli_01_0021.xml", - "product_code":"dli", - "code":"84", - "des":"To facilitate SQL operation execution, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not n", - "doc_type":"usermanual", - "kw":"Managing SQL Templates,Job Templates,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Managing SQL Templates", + "title":"Configuring DLI to Read and Write Data from and to External Data Sources", "githuburl":"" }, { - "uri":"dli_01_0464.html", - "node_id":"dli_01_0464.xml", + "uri":"dli_01_0635.html", + "node_id":"dli_01_0635.xml", "product_code":"dli", - "code":"85", - "des":"Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for edit", + "code":"68", + "des":"To read and write external data sources when running DLI jobs, two conditions must be met:Establish network connectivity between DLI and the external data source to ensur", "doc_type":"usermanual", - "kw":"Managing Flink Templates,Job Templates,User Guide", + "kw":"Configuring DLI to Read and Write External Data Sources,Configuring DLI to Read and Write Data from ", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Managing Flink Templates", - "githuburl":"" - }, - { - "uri":"dli_01_0551.html", - "node_id":"dli_01_0551.xml", - "product_code":"dli", - "code":"86", - "des":"You can modify a sample template to meet the Spark job requirements, saving time for editing SQL statements.Currently, the cloud platform does not provide preset Spark te", - "doc_type":"usermanual", - "kw":"Managing Spark SQL Templates,Job Templates,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Managing Spark SQL Templates", - "githuburl":"" - }, - { - "uri":"dli_01_05110.html", - "node_id":"dli_01_05110.xml", - "product_code":"dli", - "code":"87", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Appendix", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Appendix", - "githuburl":"" - }, - { - "uri":"dli_01_05111.html", - "node_id":"dli_01_05111.xml", - "product_code":"dli", - "code":"88", - "des":"TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and ", - "doc_type":"usermanual", - "kw":"TPC-H Sample Data in the SQL Template,Appendix,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"TPC-H Sample Data in the SQL Template", + "title":"Configuring DLI to Read and Write External Data Sources", "githuburl":"" }, { "uri":"dli_01_0426.html", "node_id":"dli_01_0426.xml", "product_code":"dli", - "code":"89", + "code":"69", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Enhanced Datasource Connections", + "kw":"Configuring the Network Connection Between DLI and Data Sources (Enhanced Datasource Connection)", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Enhanced Datasource Connections", + "title":"Configuring the Network Connection Between DLI and Data Sources (Enhanced Datasource Connection)", "githuburl":"" }, { "uri":"dli_01_0003.html", "node_id":"dli_01_0003.xml", "product_code":"dli", - "code":"90", + "code":"70", "des":"In cross-source data analysis scenarios, DLI needs to connect to external data sources. However, due to the different VPCs between the data source and DLI, the network ca", "doc_type":"usermanual", - "kw":"Overview,Enhanced Datasource Connections,User Guide", + "kw":"Overview of Enhanced Datasource Connections,Configuring the Network Connection Between DLI and Data ", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Overview", - "githuburl":"" - }, - { - "uri":"dli_01_0410.html", - "node_id":"dli_01_0410.xml", - "product_code":"dli", - "code":"91", - "des":"If DLI needs to access external data sources, you need to establish enhanced datasource connections to enable the network between DLI and the data sources, and then devel", - "doc_type":"usermanual", - "kw":"Cross-Source Analysis Development Methods,Enhanced Datasource Connections,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"Yes", - "opensource":"true", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Cross-Source Analysis Development Methods", + "title":"Overview of Enhanced Datasource Connections", "githuburl":"" }, { "uri":"dli_01_0006.html", "node_id":"dli_01_0006.xml", "product_code":"dli", - "code":"92", + "code":"71", "des":"Create an enhanced datasource connection for DLI to access, import, query, and analyze data of other data sources.For example, to connect DLI to the MRS, RDS, CSS, Kafka,", "doc_type":"usermanual", - "kw":"Creating an Enhanced Datasource Connection,Enhanced Datasource Connections,User Guide", + "kw":"Creating an Enhanced Datasource Connection,Configuring the Network Connection Between DLI and Data S", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Creating an Enhanced Datasource Connection", "githuburl":"" }, { - "uri":"dli_01_0624.html", - "node_id":"dli_01_0624.xml", + "uri":"dli_01_0410.html", + "node_id":"dli_01_0410.xml", "product_code":"dli", - "code":"93", - "des":"VPC sharing allows sharing VPC resources created in one account with other accounts using Resource Access Manager (RAM). For example, account A can share its VPC and subn", + "code":"72", + "des":"If DLI needs to access external data sources, you need to establish enhanced datasource connections to enable the network between DLI and the data sources, and then devel", "doc_type":"usermanual", - "kw":"Establishing a Network Connection Between DLI and Resources in a Shared VPC,Enhanced Datasource Conn", + "kw":"Common Development Methods for DLI Cross-Source Analysis,Configuring the Network Connection Between ", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Establishing a Network Connection Between DLI and Resources in a Shared VPC", + "title":"Common Development Methods for DLI Cross-Source Analysis", "githuburl":"" }, { - "uri":"dli_01_0553.html", - "node_id":"dli_01_0553.xml", + "uri":"dli_01_0636.html", + "node_id":"dli_01_0636.xml", "product_code":"dli", - "code":"94", - "des":"Delete an enhanced datasource connection that is no longer used on the console.Log in to the DLI management console.In the left navigation pane, choose Datasource Connect", + "code":"73", + "des":"When using DLI to submit jobs that involve reading and writing data from external sources, it is crucial to securely access these sources by properly storing their access", "doc_type":"usermanual", - "kw":"Deleting an Enhanced Datasource Connection,Enhanced Datasource Connections,User Guide", + "kw":"Using DEW to Manage Access Credentials for Data Sources,Configuring DLI to Read and Write Data from ", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Deleting an Enhanced Datasource Connection", - "githuburl":"" - }, - { - "uri":"dli_01_0013.html", - "node_id":"dli_01_0013.xml", - "product_code":"dli", - "code":"95", - "des":"Host information is the mapping between host IP addresses and domain names. After you configure host information, jobs can only use the configured domain names to access ", - "doc_type":"usermanual", - "kw":"Modifying Host Information in an Elastic Resource Pool,Enhanced Datasource Connections,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"Yes", - "opensource":"true", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Modifying Host Information in an Elastic Resource Pool", - "githuburl":"" - }, - { - "uri":"dli_01_0514.html", - "node_id":"dli_01_0514.xml", - "product_code":"dli", - "code":"96", - "des":"The CIDR block of the DLI queue that is bound with a datasource connection cannot overlap with that of the data source.The default queue cannot be bound with a connection", - "doc_type":"usermanual", - "kw":"Binding and Unbinding a Queue,Enhanced Datasource Connections,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"Yes", - "opensource":"true", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Binding and Unbinding a Queue", - "githuburl":"" - }, - { - "uri":"dli_01_0014.html", - "node_id":"dli_01_0014.xml", - "product_code":"dli", - "code":"97", - "des":"A route is configured with the destination, next hop type, and next hop to determine where the network traffic is directed. Routes are classified into system routes and c", - "doc_type":"usermanual", - "kw":"Adding a Route,Enhanced Datasource Connections,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"Yes", - "opensource":"true", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Adding a Route", - "githuburl":"" - }, - { - "uri":"dli_01_0556.html", - "node_id":"dli_01_0556.xml", - "product_code":"dli", - "code":"98", - "des":"Delete a route that is no longer used.A custom route table cannot be deleted if it is associated with a subnet.Log in to the DLI management console.In the left navigation", - "doc_type":"usermanual", - "kw":"Deleting a Route,Enhanced Datasource Connections,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"Yes", - "opensource":"true", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Deleting a Route", - "githuburl":"" - }, - { - "uri":"dli_01_0018.html", - "node_id":"dli_01_0018.xml", - "product_code":"dli", - "code":"99", - "des":"Enhanced connections support user authorization by project. After authorization, users in the project have the permission to perform operations on the enhanced connection", - "doc_type":"usermanual", - "kw":"Enhanced Connection Permission Management,Enhanced Datasource Connections,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"Yes", - "opensource":"true", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Enhanced Connection Permission Management", - "githuburl":"" - }, - { - "uri":"dli_01_0019.html", - "node_id":"dli_01_0019.xml", - "product_code":"dli", - "code":"100", - "des":"A tag is a key-value pair customized by users and used to identify cloud resources. It helps users to classify and search for cloud resources. A tag consists of a tag key", - "doc_type":"usermanual", - "kw":"Enhanced Datasource Connection Tag Management,Enhanced Datasource Connections,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"Yes", - "opensource":"true", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Enhanced Datasource Connection Tag Management", + "title":"Using DEW to Manage Access Credentials for Data Sources", "githuburl":"" }, { "uri":"dli_01_0422.html", "node_id":"dli_01_0422.xml", "product_code":"dli", - "code":"101", + "code":"74", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Datasource Authentication", + "kw":"Using DLI Datasource Authentication to Manage Access Credentials for Data Sources", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Datasource Authentication", + "title":"Using DLI Datasource Authentication to Manage Access Credentials for Data Sources", "githuburl":"" }, { "uri":"dli_01_0561.html", "node_id":"dli_01_0561.xml", "product_code":"dli", - "code":"102", - "des":"When analyzing across multiple sources, it is not recommended to configure authentication information directly in a job as it can lead to password leakage. Instead, you a", + "code":"75", + "des":"When analyzing across multiple sources, you are advised not to configure authentication information directly in a job as it can lead to password leakage. Instead, you are", "doc_type":"usermanual", - "kw":"Overview,Datasource Authentication,User Guide", + "kw":"Overview,Using DLI Datasource Authentication to Manage Access Credentials for Data Sources,User Guid", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Overview", @@ -2019,18 +1581,18 @@ "uri":"dli_01_0427.html", "node_id":"dli_01_0427.xml", "product_code":"dli", - "code":"103", + "code":"76", "des":"Create a CSS datasource authentication on the DLI console to store the authentication information of the CSS security cluster to DLI. This will allow you to access to the", "doc_type":"usermanual", - "kw":"Creating a CSS Datasource Authentication,Datasource Authentication,User Guide", + "kw":"Creating a CSS Datasource Authentication,Using DLI Datasource Authentication to Manage Access Creden", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Creating a CSS Datasource Authentication", @@ -2040,18 +1602,18 @@ "uri":"dli_01_0558.html", "node_id":"dli_01_0558.xml", "product_code":"dli", - "code":"104", + "code":"77", "des":"Create a Kerberos datasource authentication on the DLI console to store the authentication information of the data source to DLI. This will allow you to access to the dat", "doc_type":"usermanual", - "kw":"Creating a Kerberos Datasource Authentication,Datasource Authentication,User Guide", + "kw":"Creating a Kerberos Datasource Authentication,Using DLI Datasource Authentication to Manage Access C", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Creating a Kerberos Datasource Authentication", @@ -2061,18 +1623,18 @@ "uri":"dli_01_0560.html", "node_id":"dli_01_0560.xml", "product_code":"dli", - "code":"105", + "code":"78", "des":"Create a Kafka_SSL datasource authentication on the DLI console to store the Kafka authentication information to DLI. This will allow you to access to Kafka instances wit", "doc_type":"usermanual", - "kw":"Creating a Kafka_SSL Datasource Authentication,Datasource Authentication,User Guide", + "kw":"Creating a Kafka_SSL Datasource Authentication,Using DLI Datasource Authentication to Manage Access ", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Creating a Kafka_SSL Datasource Authentication", @@ -2082,18 +1644,18 @@ "uri":"dli_01_0559.html", "node_id":"dli_01_0559.xml", "product_code":"dli", - "code":"106", + "code":"79", "des":"Create a password datasource authentication on the DLI console to store passwords of the GaussDB(DWS), RDS, DCS, and DDS data sources to DLI. This will allow you to acces", "doc_type":"usermanual", - "kw":"Creating a Password Datasource Authentication,Datasource Authentication,User Guide", + "kw":"Creating a Password Datasource Authentication,Using DLI Datasource Authentication to Manage Access C", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Creating a Password Datasource Authentication", @@ -2103,86 +1665,1057 @@ "uri":"dli_01_0480.html", "node_id":"dli_01_0480.xml", "product_code":"dli", - "code":"107", + "code":"80", "des":"Grant permissions on a datasource authentication to users so multiple user jobs can use the datasource authentication without affecting each other.The administrator and t", "doc_type":"usermanual", - "kw":"Datasource Authentication Permission Management,Datasource Authentication,User Guide", + "kw":"Datasource Authentication Permission Management,Using DLI Datasource Authentication to Manage Access", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"Yes", "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Datasource Authentication Permission Management", "githuburl":"" }, { - "uri":"dli_01_0485.html", - "node_id":"dli_01_0485.xml", + "uri":"dli_01_0509.html", + "node_id":"dli_01_0509.xml", "product_code":"dli", - "code":"108", + "code":"81", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Global Configuration", + "kw":"Managing Enhanced Datasource Connections", "search_title":"", "metedata":[ { "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Managing Enhanced Datasource Connections", + "githuburl":"" + }, + { + "uri":"dli_01_0623.html", + "node_id":"dli_01_0623.xml", + "product_code":"dli", + "code":"82", + "des":"After creating an enhanced datasource connection, you can view and manage it on the management console.This section describes how to view basic information about an enhan", + "doc_type":"usermanual", + "kw":"Viewing Basic Information About an Enhanced Datasource Connection,Managing Enhanced Datasource Conne", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Viewing Basic Information About an Enhanced Datasource Connection", + "githuburl":"" + }, + { + "uri":"dli_01_0018.html", + "node_id":"dli_01_0018.xml", + "product_code":"dli", + "code":"83", + "des":"Enhanced connections support user authorization by project. After authorization, users in the project have the permission to perform operations on the enhanced connection", + "doc_type":"usermanual", + "kw":"Enhanced Connection Permission Management,Managing Enhanced Datasource Connections,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Enhanced Connection Permission Management", + "githuburl":"" + }, + { + "uri":"dli_01_0514.html", + "node_id":"dli_01_0514.xml", + "product_code":"dli", + "code":"84", + "des":"Bind an elastic resource pool to a queue to connect the queue to a target data source. When an enhanced datasource connection is no longer needed, unbind the queue that w", + "doc_type":"usermanual", + "kw":"Binding or Unbinding an Enhanced Datasource Connection to a Queue,Managing Enhanced Datasource Conne", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Binding or Unbinding an Enhanced Datasource Connection to a Queue", + "githuburl":"" + }, + { + "uri":"dli_01_0014.html", + "node_id":"dli_01_0014.xml", + "product_code":"dli", + "code":"85", + "des":"A route is configured with the destination, next hop type, and next hop to determine where the network traffic is directed. Routes are classified into system routes and c", + "doc_type":"usermanual", + "kw":"Adding a Route for an Enhanced Datasource Connection,Managing Enhanced Datasource Connections,User G", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Adding a Route for an Enhanced Datasource Connection", + "githuburl":"" + }, + { + "uri":"dli_01_0556.html", + "node_id":"dli_01_0556.xml", + "product_code":"dli", + "code":"86", + "des":"Delete a route that is no longer used.A custom route table cannot be deleted if it is associated with a subnet.Log in to the DLI management console.In the left navigation", + "doc_type":"usermanual", + "kw":"Deleting the Route for an Enhanced Datasource Connection,Managing Enhanced Datasource Connections,Us", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Deleting the Route for an Enhanced Datasource Connection", + "githuburl":"" + }, + { + "uri":"dli_01_0013.html", + "node_id":"dli_01_0013.xml", + "product_code":"dli", + "code":"87", + "des":"Host information is the mapping between host IP addresses and domain names. After you configure host information, jobs can only use the configured domain names to access ", + "doc_type":"usermanual", + "kw":"Modifying Host Information in an Elastic Resource Pool,Managing Enhanced Datasource Connections,User", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Modifying Host Information in an Elastic Resource Pool", + "githuburl":"" + }, + { + "uri":"dli_01_0019.html", + "node_id":"dli_01_0019.xml", + "product_code":"dli", + "code":"88", + "des":"A tag is a key-value pair customized by users and used to identify cloud resources. It helps users to classify and search for cloud resources. A tag consists of a tag key", + "doc_type":"usermanual", + "kw":"Enhanced Datasource Connection Tag Management,Managing Enhanced Datasource Connections,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Enhanced Datasource Connection Tag Management", + "githuburl":"" + }, + { + "uri":"dli_01_0553.html", + "node_id":"dli_01_0553.xml", + "product_code":"dli", + "code":"89", + "des":"Delete an enhanced datasource connection that is no longer used on the console.Log in to the DLI management console.In the left navigation pane, choose Datasource Connect", + "doc_type":"usermanual", + "kw":"Deleting an Enhanced Datasource Connection,Managing Enhanced Datasource Connections,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Deleting an Enhanced Datasource Connection", + "githuburl":"" + }, + { + "uri":"dli_01_0486.html", + "node_id":"dli_01_0486.xml", + "product_code":"dli", + "code":"90", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Configuring an Agency to Allow DLI to Access Other Cloud Services", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Configuring an Agency to Allow DLI to Access Other Cloud Services", + "githuburl":"" + }, + { + "uri":"dli_01_0419.html", + "node_id":"dli_01_0419.xml", + "product_code":"dli", + "code":"91", + "des":"Cloud services often interact with each other, with some of which dependent on other services. You can create an agency to delegate DLI to use other cloud services and pe", + "doc_type":"usermanual", + "kw":"DLI Agency Overview,Configuring an Agency to Allow DLI to Access Other Cloud Services,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"DLI Agency Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0616.html", + "node_id":"dli_01_0616.xml", + "product_code":"dli", + "code":"92", + "des":"When Flink 1.15, Spark 3.3, or a later version is used to execute jobs and the required agency is not included in the DLI system agency dli_management_agency, you need to", + "doc_type":"usermanual", + "kw":"Creating a Custom DLI Agency,Configuring an Agency to Allow DLI to Access Other Cloud Services,User ", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating a Custom DLI Agency", + "githuburl":"" + }, + { + "uri":"dli_01_0617.html", + "node_id":"dli_01_0617.xml", + "product_code":"dli", + "code":"93", + "des":"This section provides agency permission policies for common scenarios, which can be used to configure agency permission policies when you customize your permissions. The ", + "doc_type":"usermanual", + "kw":"Agency Permission Policies in Common Scenarios,Configuring an Agency to Allow DLI to Access Other Cl", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Agency Permission Policies in Common Scenarios", + "githuburl":"" + }, + { + "uri":"dli_01_0402.html", + "node_id":"dli_01_0402.xml", + "product_code":"dli", + "code":"94", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Submitting a SQL Job Using DLI", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Submitting a SQL Job Using DLI", + "githuburl":"" + }, + { + "uri":"dli_01_0320.html", + "node_id":"dli_01_0320.xml", + "product_code":"dli", + "code":"95", + "des":"The SQL Editor allows you to execute data query operations using SQL statements.It supports SQL:2003 and is compatible with Spark SQL.To access the SQL editor, choose SQL", + "doc_type":"usermanual", + "kw":"Creating and Submitting a SQL Job,Submitting a SQL Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating and Submitting a SQL Job", + "githuburl":"" + }, + { + "uri":"dli_01_0621.html", + "node_id":"dli_01_0621.xml", + "product_code":"dli", + "code":"96", + "des":"Store the data results of analyzed SQL jobs in a specified location in the desired format.By default, DLI stores SQL job results in its job bucket. You can also download ", + "doc_type":"usermanual", + "kw":"Exporting SQL Job Results,Submitting a SQL Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Exporting SQL Job Results", + "githuburl":"" + }, + { + "uri":"dli_01_0535.html", + "node_id":"dli_01_0535.xml", + "product_code":"dli", + "code":"97", + "des":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", + "doc_type":"usermanual", + "kw":"Setting the Priority for a SQL Job,Submitting a SQL Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Setting the Priority for a SQL Job", + "githuburl":"" + }, + { + "uri":"dli_01_0637.html", + "node_id":"dli_01_0637.xml", + "product_code":"dli", + "code":"98", + "des":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", + "doc_type":"usermanual", + "kw":"Querying Logs for SQL Jobs,Submitting a SQL Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Querying Logs for SQL Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0017.html", + "node_id":"dli_01_0017.xml", + "product_code":"dli", + "code":"99", + "des":"The SQL Jobs page displays all SQL jobs, which may span multiple pages if there are many jobs. You can navigate to a specific page as needed. DLI allows you to view jobs ", + "doc_type":"usermanual", + "kw":"Managing SQL Jobs,Submitting a SQL Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Managing SQL Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0638.html", + "node_id":"dli_01_0638.xml", + "product_code":"dli", + "code":"100", + "des":"A SQL execution plan is a logical flowchart of a database query that shows how a database management system executes a specific SQL query. The execution plan details the ", + "doc_type":"usermanual", + "kw":"Viewing a SQL Execution Plan,Submitting a SQL Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Viewing a SQL Execution Plan", + "githuburl":"" + }, + { + "uri":"dli_01_05110.html", + "node_id":"dli_01_05110.xml", + "product_code":"dli", + "code":"101", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Creating and Managing SQL Job Templates", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating and Managing SQL Job Templates", + "githuburl":"" + }, + { + "uri":"dli_01_0021.html", + "node_id":"dli_01_0021.xml", + "product_code":"dli", + "code":"102", + "des":"To facilitate SQL operation execution, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not n", + "doc_type":"usermanual", + "kw":"Creating a SQL Job Template,Creating and Managing SQL Job Templates,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating a SQL Job Template", + "githuburl":"" + }, + { + "uri":"dli_01_0639.html", + "node_id":"dli_01_0639.xml", + "product_code":"dli", + "code":"103", + "des":"DLI allows you to create custom templates or save currently used SQL statements as templates for quick and convenient SQL operations. Once a template is saved, you can ex", + "doc_type":"usermanual", + "kw":"Developing and Submitting a SQL Job Using a SQL Job Template,Creating and Managing SQL Job Templates", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Developing and Submitting a SQL Job Using a SQL Job Template", + "githuburl":"" + }, + { + "uri":"dli_01_05111.html", + "node_id":"dli_01_05111.xml", + "product_code":"dli", + "code":"104", + "des":"TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and ", + "doc_type":"usermanual", + "kw":"TPC-H Sample Data in the SQL Templates Preset on DLI,Creating and Managing SQL Job Templates,User Gu", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"TPC-H Sample Data in the SQL Templates Preset on DLI", + "githuburl":"" + }, + { + "uri":"dli_01_0389.html", + "node_id":"dli_01_0389.xml", + "product_code":"dli", + "code":"105", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Submitting a Flink Job Using DLI", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Submitting a Flink Job Using DLI", + "githuburl":"" + }, + { + "uri":"dli_01_0403.html", + "node_id":"dli_01_0403.xml", + "product_code":"dli", + "code":"106", + "des":"DLI supports two types of Flink jobs:Flink OpenSource SQL job:It is fully compatible with Flink of the community edition, ensuring that jobs can run smoothly on these Fli", + "doc_type":"usermanual", + "kw":"Flink Job Overview,Submitting a Flink Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Flink Job Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0498.html", + "node_id":"dli_01_0498.xml", + "product_code":"dli", + "code":"107", + "des":"This section describes how to create a Flink OpenSource SQL job.DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink provided by the community. In ", + "doc_type":"usermanual", + "kw":"Creating a Flink OpenSource SQL Job,Submitting a Flink Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating a Flink OpenSource SQL Job", + "githuburl":"" + }, + { + "uri":"dli_01_0457.html", + "node_id":"dli_01_0457.xml", + "product_code":"dli", + "code":"108", + "des":"A Flink Jar job involves developing a custom application Jar package based on Flink's capabilities and submitting it to a DLI queue for execution.To create a Flink Jar jo", + "doc_type":"usermanual", + "kw":"Creating a Flink Jar Job,Submitting a Flink Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating a Flink Jar Job", + "githuburl":"" + }, + { + "uri":"dli_01_0479.html", + "node_id":"dli_01_0479.xml", + "product_code":"dli", + "code":"109", + "des":"You can isolate Flink jobs allocated to different users by setting permissions to ensure data query performance.The administrator and job creator have all permissions, wh", + "doc_type":"usermanual", + "kw":"Configuring Flink Job Permissions,Submitting a Flink Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Configuring Flink Job Permissions", + "githuburl":"" + }, + { + "uri":"dli_01_0377.html", + "node_id":"dli_01_0377.xml", + "product_code":"dli", + "code":"110", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Managing Flink Jobs", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Managing Flink Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0462.html", + "node_id":"dli_01_0462.xml", + "product_code":"dli", + "code":"111", + "des":"After creating a Flink job, you can check the basic information, job details, task list, and execution plan of the job on the DLI console.This section describes how to ch", + "doc_type":"usermanual", + "kw":"Viewing Flink Job Details,Managing Flink Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Viewing Flink Job Details", + "githuburl":"" + }, + { + "uri":"dli_01_0650.html", + "node_id":"dli_01_0650.xml", + "product_code":"dli", + "code":"112", + "des":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", + "doc_type":"usermanual", + "kw":"Setting the Priority for a Flink Job,Managing Flink Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Setting the Priority for a Flink Job", + "githuburl":"" + }, + { + "uri":"dli_01_0534.html", + "node_id":"dli_01_0534.xml", + "product_code":"dli", + "code":"113", + "des":"In actual job operations, the compute resources required by a job vary depending on the data volume. As a result, compute resources are wasted when the volume is small an", + "doc_type":"usermanual", + "kw":"Enabling Dynamic Scaling for Flink Jobs,Managing Flink Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Enabling Dynamic Scaling for Flink Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0651.html", + "node_id":"dli_01_0651.xml", + "product_code":"dli", + "code":"114", + "des":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", + "doc_type":"usermanual", + "kw":"Querying Logs for Flink Jobs,Managing Flink Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Querying Logs for Flink Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0461.html", + "node_id":"dli_01_0461.xml", + "product_code":"dli", + "code":"115", + "des":"After creating a job, you can manage it by performing various operations such as editing its basic information, starting or stopping it, and importing or exporting it.You", + "doc_type":"usermanual", + "kw":"Common Operations of Flink Jobs,Managing Flink Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Common Operations of Flink Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0464.html", + "node_id":"dli_01_0464.xml", + "product_code":"dli", + "code":"116", + "des":"Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for edit", + "doc_type":"usermanual", + "kw":"Managing Flink Job Templates,Submitting a Flink Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Managing Flink Job Templates", + "githuburl":"" + }, + { + "uri":"dli_01_0463.html", + "node_id":"dli_01_0463.xml", + "product_code":"dli", + "code":"117", + "des":"A tag is a key-value pair customized by users and used to identify cloud resources. It helps users to classify and search for cloud resources. A tag consists of a tag key", + "doc_type":"usermanual", + "kw":"Adding Tags to a Flink Job,Submitting a Flink Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Adding Tags to a Flink Job", + "githuburl":"" + }, + { + "uri":"dli_01_0465.html", + "node_id":"dli_01_0465.xml", + "product_code":"dli", + "code":"118", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Submitting a Spark Job Using DLI", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Submitting a Spark Job Using DLI", + "githuburl":"" + }, + { + "uri":"dli_01_0384.html", + "node_id":"dli_01_0384.xml", + "product_code":"dli", + "code":"119", + "des":"DLI provides fully-managed Spark computing services by allowing you to execute Spark jobs.On the Overview page, click Create Job in the upper right corner of the Spark Jo", + "doc_type":"usermanual", + "kw":"Creating a Spark Job,Submitting a Spark Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating a Spark Job", + "githuburl":"" + }, + { + "uri":"dli_01_0652.html", + "node_id":"dli_01_0652.xml", + "product_code":"dli", + "code":"120", + "des":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", + "doc_type":"usermanual", + "kw":"Setting the Priority for a Spark Job,Submitting a Spark Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Setting the Priority for a Spark Job", + "githuburl":"" + }, + { + "uri":"dli_01_0379.html", + "node_id":"dli_01_0379.xml", + "product_code":"dli", + "code":"121", + "des":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", + "doc_type":"usermanual", + "kw":"Querying Logs for Spark Jobs,Submitting a Spark Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Querying Logs for Spark Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0385.html", + "node_id":"dli_01_0385.xml", + "product_code":"dli", + "code":"122", + "des":"On the Overview page, click Spark Jobs to go to the SQL job management page. Alternatively, you can click Job Management > Spark Jobs. The page displays all Spark jobs. I", + "doc_type":"usermanual", + "kw":"Managing Spark Jobs,Submitting a Spark Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Managing Spark Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0551.html", + "node_id":"dli_01_0551.xml", + "product_code":"dli", + "code":"123", + "des":"You can modify a sample template to meet the Spark job requirements, saving time for editing SQL statements.Currently, the cloud platform does not provide preset Spark te", + "doc_type":"usermanual", + "kw":"Managing Spark Job Templates,Submitting a Spark Job Using DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Managing Spark Job Templates", + "githuburl":"" + }, + { + "uri":"dli_01_0445.html", + "node_id":"dli_01_0445.xml", + "product_code":"dli", + "code":"124", + "des":"This section describes metrics reported by DLI to Cloud Eye as well as their namespaces and dimensions. You can use the management console or APIs provided by Cloud Eye t", + "doc_type":"usermanual", + "kw":"Using Cloud Eye to Monitor DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Using Cloud Eye to Monitor DLI", + "githuburl":"" + }, + { + "uri":"dli_01_0664.html", + "node_id":"dli_01_0664.xml", + "product_code":"dli", + "code":"125", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Using AOM to Monitor DLI", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Using AOM to Monitor DLI", + "githuburl":"" + }, + { + "uri":"dli_01_0665.html", + "node_id":"dli_01_0665.xml", + "product_code":"dli", + "code":"126", + "des":"AOM's Prometheus Monitoring is a comprehensive solution that integrates with the open source Prometheus ecosystem. It supports monitoring of various types of components, ", + "doc_type":"usermanual", + "kw":"Connecting DLI to AOM's Prometheus Monitoring,Using AOM to Monitor DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No;Yes;NO", + "IsBot":"No;Yes", + "opensource":"true", "documenttype":"usermanual" } ], - "title":"Global Configuration", + "title":"Connecting DLI to AOM's Prometheus Monitoring", "githuburl":"" }, { - "uri":"dli_01_0476.html", - "node_id":"dli_01_0476.xml", + "uri":"dli_01_0666.html", + "node_id":"dli_01_0666.xml", "product_code":"dli", - "code":"109", - "des":"DLI allows you to set variables that are frequently used during job development as global variables on the DLI management console. This avoids repeated definitions during", + "code":"127", + "des":"When connecting DLI to AOM's Prometheus Monitoring, the system automatically configures the parameters listed in table 1 in Configuration Items for Connecting DLI to AOM'", "doc_type":"usermanual", - "kw":"Global Variables,Global Configuration,User Guide", + "kw":"Configuration Items for Connecting DLI to AOM's Prometheus Monitoring,Using AOM to Monitor DLI,User ", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;Yes;NO", + "IsBot":"No;Yes", + "opensource":"true", + "documenttype":"usermanual" } ], - "title":"Global Variables", + "title":"Configuration Items for Connecting DLI to AOM's Prometheus Monitoring", "githuburl":"" }, { - "uri":"dli_01_0533.html", - "node_id":"dli_01_0533.xml", + "uri":"dli_01_0667.html", + "node_id":"dli_01_0667.xml", "product_code":"dli", - "code":"110", - "des":"You can grant permissions on a global variable to users.The administrator and the global variable owner have all permissions. You do not need to set permissions for them,", + "code":"128", + "des":"Table 1 lists the basic monitoring metrics that are reported by DLI to Prometheus. AOM Prometheus can store these basic metrics for free.", "doc_type":"usermanual", - "kw":"Permission Management for Global Variables,Global Configuration,User Guide", + "kw":"Basic Monitoring Metrics Reported by DLI to Prometheus,Using AOM to Monitor DLI,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Permission Management for Global Variables", + "title":"Basic Monitoring Metrics Reported by DLI to Prometheus", + "githuburl":"" + }, + { + "uri":"dli_01_0318.html", + "node_id":"dli_01_0318.xml", + "product_code":"dli", + "code":"129", + "des":"With CTS, you can log operations related to DLI, making it easier to search, audit, and trace in the future.", + "doc_type":"usermanual", + "kw":"Using CTS to Audit DLI,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Using CTS to Audit DLI", "githuburl":"" }, { "uri":"dli_01_0408.html", "node_id":"dli_01_0408.xml", "product_code":"dli", - "code":"111", + "code":"130", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Permissions Management", @@ -2190,7 +2723,10 @@ "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Permissions Management", @@ -2200,7 +2736,7 @@ "uri":"dli_01_0440.html", "node_id":"dli_01_0440.xml", "product_code":"dli", - "code":"112", + "code":"131", "des":"DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM", "doc_type":"usermanual", "kw":"Overview,Permissions Management,User Guide", @@ -2208,39 +2744,20 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Overview", "githuburl":"" }, - { - "uri":"dli_01_0418.html", - "node_id":"dli_01_0418.xml", - "product_code":"dli", - "code":"113", - "des":"You can use Identity and Access Management (IAM) to implement fine-grained permissions control on DLI resources. For details, see Overview.If your cloud account does not ", - "doc_type":"usermanual", - "kw":"Creating an IAM User and Granting Permissions,Permissions Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "IsMulti":"No", - "documenttype":"usermanual", - "IsBot":"Yes" - } - ], - "title":"Creating an IAM User and Granting Permissions", - "githuburl":"" - }, { "uri":"dli_01_0451.html", "node_id":"dli_01_0451.xml", "product_code":"dli", - "code":"114", + "code":"132", "des":"Custom policies can be created as a supplement to the system policies of DLI. You can add actions to custom policies. For the actions supported for custom policies, see \"", "doc_type":"usermanual", "kw":"Creating a Custom Policy,Permissions Management,User Guide", @@ -2248,9 +2765,10 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Creating a Custom Policy", @@ -2260,7 +2778,7 @@ "uri":"dli_01_0417.html", "node_id":"dli_01_0417.xml", "product_code":"dli", - "code":"115", + "code":"133", "des":"A resource is an object that exists within a service. You can select DLI resources by specifying their paths.", "doc_type":"usermanual", "kw":"DLI Resources,Permissions Management,User Guide", @@ -2268,9 +2786,10 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"DLI Resources", @@ -2280,7 +2799,7 @@ "uri":"dli_01_0475.html", "node_id":"dli_01_0475.xml", "product_code":"dli", - "code":"116", + "code":"134", "des":"Request conditions are useful in determining when a custom policy takes effect. A request condition consists of a condition key and operator. Condition keys are either gl", "doc_type":"usermanual", "kw":"DLI Request Conditions,Permissions Management,User Guide", @@ -2288,9 +2807,10 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"DLI Request Conditions", @@ -2300,7 +2820,7 @@ "uri":"dli_01_0441.html", "node_id":"dli_01_0441.xml", "product_code":"dli", - "code":"117", + "code":"135", "des":"Table 1 lists the common operations supported by each system policy of DLI. Choose proper system policies according to this table. For details about the SQL statement per", "doc_type":"usermanual", "kw":"Common Operations Supported by DLI System Policy,Permissions Management,User Guide", @@ -2308,117 +2828,251 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Common Operations Supported by DLI System Policy", "githuburl":"" }, { - "uri":"dli_01_0513.html", - "node_id":"dli_01_0513.xml", + "uri":"dli_01_0567.html", + "node_id":"dli_01_0567.xml", "product_code":"dli", - "code":"118", + "code":"136", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Other Common Operations", + "kw":"Common DLI Management Operations", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Other Common Operations", + "title":"Common DLI Management Operations", "githuburl":"" }, { - "uri":"dli_01_0420.html", - "node_id":"dli_01_0420.xml", + "uri":"dli_01_0494.html", + "node_id":"dli_01_0494.xml", "product_code":"dli", - "code":"119", - "des":"On the DLI management console, you can import data stored in OBS into DLI tables.To import OBS data to a DLI table, either choose Data Management > Databases and Tables i", + "code":"137", + "des":"To enhance the functions and performance of Spark and Flink jobs, you can create custom images by downloading the base images provided by DLI and adding dependencies (fil", "doc_type":"usermanual", - "kw":"Importing Data to a DLI Table,Other Common Operations,User Guide", + "kw":"Using a Custom Image to Enhance the Job Running Environment,Common DLI Management Operations,User Gu", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Importing Data to a DLI Table", + "title":"Using a Custom Image to Enhance the Job Running Environment", "githuburl":"" }, { - "uri":"dli_01_0445.html", - "node_id":"dli_01_0445.xml", + "uri":"dli_01_0476.html", + "node_id":"dli_01_0476.xml", "product_code":"dli", - "code":"120", - "des":"This section describes metrics reported by DLI to Cloud Eye as well as their namespaces and dimensions. You can use the management console or APIs provided by Cloud Eye t", + "code":"138", + "des":"DLI allows you to set variables that are frequently used during job development as global variables on the DLI management console. This avoids repeated definitions during", "doc_type":"usermanual", - "kw":"Viewing Monitoring Metrics,Other Common Operations,User Guide", + "kw":"Managing DLI Global Variables,Common DLI Management Operations,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Viewing Monitoring Metrics", + "title":"Managing DLI Global Variables", "githuburl":"" }, { - "uri":"dli_01_0318.html", - "node_id":"dli_01_0318.xml", + "uri":"dli_01_0366.html", + "node_id":"dli_01_0366.xml", "product_code":"dli", - "code":"121", - "des":"With CTS, you can record operations associated with DLI for later query, audit, and backtrack operations.", + "code":"139", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"DLI Operations That Can Be Recorded by CTS,Other Common Operations,User Guide", + "kw":"Managing Program Packages of Jar Jobs", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"DLI Operations That Can Be Recorded by CTS", + "title":"Managing Program Packages of Jar Jobs", + "githuburl":"" + }, + { + "uri":"dli_01_0407.html", + "node_id":"dli_01_0407.xml", + "product_code":"dli", + "code":"140", + "des":"Before running DLI jobs, UDF JAR files or Jar job packages need to be uploaded to the cloud platform for unified management and maintenance.There are two ways to manage p", + "doc_type":"usermanual", + "kw":"Package Management Overview,Managing Program Packages of Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Package Management Overview", + "githuburl":"" + }, + { + "uri":"dli_01_0367.html", + "node_id":"dli_01_0367.xml", + "product_code":"dli", + "code":"141", + "des":"DLI allows you to submit program packages in batches to the general-use queue for running.If you need to update a package, you can use the same package or file to upload ", + "doc_type":"usermanual", + "kw":"Creating a DLI Package,Managing Program Packages of Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Creating a DLI Package", + "githuburl":"" + }, + { + "uri":"dli_01_0477.html", + "node_id":"dli_01_0477.xml", + "product_code":"dli", + "code":"142", + "des":"By configuring permissions, you can grant different package groups or packages to various users, ensuring that job efficiency remains unaffected and job performance is ma", + "doc_type":"usermanual", + "kw":"Configuring DLI Package Permissions,Managing Program Packages of Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Configuring DLI Package Permissions", + "githuburl":"" + }, + { + "uri":"dli_01_0478.html", + "node_id":"dli_01_0478.xml", + "product_code":"dli", + "code":"143", + "des":"DLI allows you to change the owner of a package group or package.Log in to the DLI management console and choose Data Management > Package Management.On the Package Manag", + "doc_type":"usermanual", + "kw":"Changing the DLI Package Owner,Managing Program Packages of Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Changing the DLI Package Owner", + "githuburl":"" + }, + { + "uri":"dli_01_0369.html", + "node_id":"dli_01_0369.xml", + "product_code":"dli", + "code":"144", + "des":"Tags are key-value pairs that you can define to identify cloud resources. They assist you in categorizing and searching for cloud resources. A tag consists of a key and a", + "doc_type":"usermanual", + "kw":"Managing DLI Package Tags,Managing Program Packages of Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"Managing DLI Package Tags", + "githuburl":"" + }, + { + "uri":"dli_01_0397.html", + "node_id":"dli_01_0397.xml", + "product_code":"dli", + "code":"145", + "des":"DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar ", + "doc_type":"usermanual", + "kw":"DLI Built-in Dependencies,Managing Program Packages of Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" + } + ], + "title":"DLI Built-in Dependencies", "githuburl":"" }, { "uri":"dli_01_0550.html", "node_id":"dli_01_0550.xml", "product_code":"dli", - "code":"122", + "code":"146", "des":"A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.You can also request for an increased quota if your e", "doc_type":"usermanual", - "kw":"Quota Management,Other Common Operations,User Guide", + "kw":"Managing DLI Resource Quotas,Common DLI Management Operations,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "IsMulti":"No", + "opensource":"true", "documenttype":"usermanual", - "IsBot":"Yes" + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], - "title":"Quota Management", + "title":"Managing DLI Resource Quotas", "githuburl":"" }, { "uri":"dli_01_0539.html", "node_id":"dli_01_0539.xml", "product_code":"dli", - "code":"123", + "code":"147", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"FAQ", @@ -2426,2023 +3080,50 @@ "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"FAQ", "githuburl":"" }, - { - "uri":"dli_03_0037.html", - "node_id":"dli_03_0037.xml", - "product_code":"dli", - "code":"124", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Flink Jobs", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Flink Jobs", - "githuburl":"" - }, - { - "uri":"dli_03_0137.html", - "node_id":"dli_03_0137.xml", - "product_code":"dli", - "code":"125", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Usage", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Usage", - "githuburl":"" - }, - { - "uri":"dli_03_0083.html", - "node_id":"dli_03_0083.xml", - "product_code":"dli", - "code":"126", - "des":"DLI Flink jobs support the following data formats:Avro, Avro_merge, BLOB, CSV, EMAIL, JSON, ORC, Parquet, and XML.DLI Flink jobs support data from the following data sour", - "doc_type":"usermanual", - "kw":"What Data Formats and Data Sources Are Supported by DLI Flink Jobs?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"What Data Formats and Data Sources Are Supported by DLI Flink Jobs?", - "githuburl":"" - }, - { - "uri":"dli_03_0139.html", - "node_id":"dli_03_0139.xml", - "product_code":"dli", - "code":"127", - "des":"A sub-user can view queues but cannot view Flink jobs. You can authorize the sub-user using DLI or IAM.Authorization on DLILog in to the DLI console using a tenant accoun", - "doc_type":"usermanual", - "kw":"How Do I Authorize a Subuser to View Flink Jobs?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Authorize a Subuser to View Flink Jobs?", - "githuburl":"" - }, - { - "uri":"dli_03_0090.html", - "node_id":"dli_03_0090.xml", - "product_code":"dli", - "code":"128", - "des":"DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are r", - "doc_type":"usermanual", - "kw":"How Do I Set Auto Restart upon Exception for a Flink Job?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Set Auto Restart upon Exception for a Flink Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0099.html", - "node_id":"dli_03_0099.xml", - "product_code":"dli", - "code":"129", - "des":"When you create a Flink SQL job or Flink Jar job, you can select Save Job Log on the job editing page to save job running logs to OBS.To set the OBS bucket for storing th", - "doc_type":"usermanual", - "kw":"How Do I Save Flink Job Logs?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Save Flink Job Logs?", - "githuburl":"" - }, - { - "uri":"dli_03_0043.html", - "node_id":"dli_03_0043.xml", - "product_code":"dli", - "code":"130", - "des":"DLI can output Flink job results to DIS. You can view the results in DIS. For details, see \"Obtaining Data from DIS\" in Data Ingestion Service User Guide.DLI can output F", - "doc_type":"usermanual", - "kw":"How Can I Check Flink Job Results?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Can I Check Flink Job Results?", - "githuburl":"" - }, - { - "uri":"dli_03_0160.html", - "node_id":"dli_03_0160.xml", - "product_code":"dli", - "code":"131", - "des":"Choose Job Management > Flink Jobs. In the Operation column of the target job, choose More > Permissions. When a new user is authorized, No such user. userName:xxxx. is d", - "doc_type":"usermanual", - "kw":"Why Is Error \"No such user. userName:xxxx.\" Reported on the Flink Job Management Page When I Grant P", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"No such user. userName:xxxx.\" Reported on the Flink Job Management Page When I Grant Permission to a User?", - "githuburl":"" - }, - { - "uri":"dli_03_0180.html", - "node_id":"dli_03_0180.xml", - "product_code":"dli", - "code":"132", - "des":"Checkpoint was enabled when a Flink job is created, and the OBS bucket for storing checkpoints was specified. After a Flink job is manually stopped, no message is display", - "doc_type":"usermanual", - "kw":"How Do I Know Which Checkpoint the Flink Job I Stopped Will Be Restored to When I Start the Job Agai", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Know Which Checkpoint the Flink Job I Stopped Will Be Restored to When I Start the Job Again?", - "githuburl":"" - }, - { - "uri":"dli_03_0036.html", - "node_id":"dli_03_0036.xml", - "product_code":"dli", - "code":"133", - "des":"When you set running parameters of a DLI Flink job, you can enable Alarm Generation upon Job Exception to receive alarms when the job runs abnormally or is in arrears.If ", - "doc_type":"usermanual", - "kw":"Why Is a Message Displayed Indicating That the SMN Topic Does Not Exist When I Use the SMN Topic in ", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is a Message Displayed Indicating That the SMN Topic Does Not Exist When I Use the SMN Topic in DLI?", - "githuburl":"" - }, - { - "uri":"dli_03_0131.html", - "node_id":"dli_03_0131.xml", - "product_code":"dli", - "code":"134", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Flink SQL", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Flink SQL", - "githuburl":"" - }, - { - "uri":"dli_03_0130.html", - "node_id":"dli_03_0130.xml", - "product_code":"dli", - "code":"135", - "des":"The consumption capability of a Flink SQL job depends on the data source transmission, queue size, and job parameter settings. The peak consumption is 10 Mbit/s.", - "doc_type":"usermanual", - "kw":"How Much Data Can Be Processed in a Day by a Flink SQL Job?,Flink SQL,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Much Data Can Be Processed in a Day by a Flink SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0061.html", - "node_id":"dli_03_0061.xml", - "product_code":"dli", - "code":"136", - "des":"The temp stream in Flink SQL is similar to a subquery. It is a logical stream used to simplify the SQL logic and does not generate data storage. Therefore, there is no ne", - "doc_type":"usermanual", - "kw":"Does Data in the Temporary Stream of Flink SQL Need to Be Cleared Periodically? How Do I Clear the D", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Does Data in the Temporary Stream of Flink SQL Need to Be Cleared Periodically? How Do I Clear the Data?", - "githuburl":"" - }, - { - "uri":"dli_03_0138.html", - "node_id":"dli_03_0138.xml", - "product_code":"dli", - "code":"137", - "des":"When you create a Flink SQL job and configure the parameters, you select an OBS bucket you have created. The system displays a message indicating that the OBS bucket is n", - "doc_type":"usermanual", - "kw":"Why Is a Message Displayed Indicating That the OBS Bucket Is Not Authorized When I Select an OBS Buc", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is a Message Displayed Indicating That the OBS Bucket Is Not Authorized When I Select an OBS Bucket for a Flink SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0089.html", - "node_id":"dli_03_0089.xml", - "product_code":"dli", - "code":"138", - "des":"When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.In the following example, the day field is used as the partition fie", - "doc_type":"usermanual", - "kw":"How Do I Create an OBS Partitioned Table for a Flink SQL Job?,Flink SQL,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Create an OBS Partitioned Table for a Flink SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0075.html", - "node_id":"dli_03_0075.xml", - "product_code":"dli", - "code":"139", - "des":"In this example, the day field is used as the partition field with the parquet encoding format (only the parquet format is supported currently) to dump car_info data to O", - "doc_type":"usermanual", - "kw":"How Do I Dump Data to OBS and Create an OBS Partitioned Table?,Flink SQL,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Dump Data to OBS and Create an OBS Partitioned Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0167.html", - "node_id":"dli_03_0167.xml", - "product_code":"dli", - "code":"140", - "des":"When I run the creation statement with an EL expression in the table name in a Flink SQL job, the following error message is displayed:DLI.0005: AnalysisException: t_user", - "doc_type":"usermanual", - "kw":"Why Is Error Message \"DLI.0005\" Displayed When I Use an EL Expression to Create a Table in a Flink S", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error Message \"DLI.0005\" Displayed When I Use an EL Expression to Create a Table in a Flink SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0168.html", - "node_id":"dli_03_0168.xml", - "product_code":"dli", - "code":"141", - "des":"After data is written to OBS through the Flink job output stream, data cannot be queried from the DLI table created in the OBS file path.For example, use the following Fl", - "doc_type":"usermanual", - "kw":"Why Is No Data Queried in the DLI Table Created Using the OBS File Path When Data Is Written to OBS ", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is No Data Queried in the DLI Table Created Using the OBS File Path When Data Is Written to OBS by a Flink Job Output Stream?", - "githuburl":"" - }, - { - "uri":"dli_03_0174.html", - "node_id":"dli_03_0174.xml", - "product_code":"dli", - "code":"142", - "des":"After a Flink SQL job is submitted on DLI, the job fails to be executed. The following error information is displayed in the job log:connect to DIS failed java.lang.Illeg", - "doc_type":"usermanual", - "kw":"Why Does a Flink SQL Job Fails to Be Executed, and Is \"connect to DIS failed java.lang.IllegalArgume", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does a Flink SQL Job Fails to Be Executed, and Is \"connect to DIS failed java.lang.IllegalArgumentException: Access key cannot be null\" Displayed in the Log?", - "githuburl":"" - }, - { - "uri":"dli_03_0176.html", - "node_id":"dli_03_0176.xml", - "product_code":"dli", - "code":"143", - "des":"Semantic verification for a Flink SQL job (reading DIS data) fails. The following information is displayed when the job fails:Get dis channel xxxinfo failed. error info: ", - "doc_type":"usermanual", - "kw":"Why Is Error \"Not authorized\" Reported When a Flink SQL Job Reads DIS Data?,Flink SQL,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"Not authorized\" Reported When a Flink SQL Job Reads DIS Data?", - "githuburl":"" - }, - { - "uri":"dli_03_0232.html", - "node_id":"dli_03_0232.xml", - "product_code":"dli", - "code":"144", - "des":"After a Flink SQL job consumed Kafka and sent data to the Elasticsearch cluster, the job was successfully executed, but no data is available.Possible causes are as follow", - "doc_type":"usermanual", - "kw":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster,F", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster", - "githuburl":"" - }, - { - "uri":"dli_03_0132.html", - "node_id":"dli_03_0132.xml", - "product_code":"dli", - "code":"145", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Flink Jar Jobs", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Flink Jar Jobs", - "githuburl":"" - }, - { - "uri":"dli_03_0044.html", - "node_id":"dli_03_0044.xml", - "product_code":"dli", - "code":"146", - "des":"You can upload configuration files for custom jobs (Jar).Upload the configuration file to DLI through Package Management.In the Other Dependencies area of the Flink Jar j", - "doc_type":"usermanual", - "kw":"Can I Upload Configuration Files for Flink Jar Jobs?,Flink Jar Jobs,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Can I Upload Configuration Files for Flink Jar Jobs?", - "githuburl":"" - }, - { - "uri":"dli_03_0119.html", - "node_id":"dli_03_0119.xml", - "product_code":"dli", - "code":"147", - "des":"The dependency of your Flink job conflicts with a built-in dependency of the DLI Flink platform. As a result, the job submission fails.Delete your JAR file that is the sa", - "doc_type":"usermanual", - "kw":"Why Does a Flink Jar Package Conflict Result in Submission Failure?,Flink Jar Jobs,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does a Flink Jar Package Conflict Result in Submission Failure?", - "githuburl":"" - }, - { - "uri":"dli_03_0161.html", - "node_id":"dli_03_0161.xml", - "product_code":"dli", - "code":"148", - "des":"When a Flink Jar job is submitted to access GaussDB(DWS), an error message is displayed indicating that the job fails to be started. The job log contains the following er", - "doc_type":"usermanual", - "kw":"Why Does a Flink Jar Job Fail to Access GaussDB(DWS) and a Message Is Displayed Indicating Too Many ", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does a Flink Jar Job Fail to Access GaussDB(DWS) and a Message Is Displayed Indicating Too Many Client Connections?", - "githuburl":"" - }, - { - "uri":"dli_03_0165.html", - "node_id":"dli_03_0165.xml", - "product_code":"dli", - "code":"149", - "des":"An exception occurred when a Flink Jar job is running. The following error information is displayed in the job log:org.apache.flink.shaded.curator.org.apache.curator.Conn", - "doc_type":"usermanual", - "kw":"Why Is Error Message \"Authentication failed\" Displayed During Flink Jar Job Running?,Flink Jar Jobs,", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error Message \"Authentication failed\" Displayed During Flink Jar Job Running?", - "githuburl":"" - }, - { - "uri":"dli_03_0233.html", - "node_id":"dli_03_0233.xml", - "product_code":"dli", - "code":"150", - "des":"The storage path of the Flink Jar job checkpoints was set to an OBS bucket. The job failed to be submitted, and an error message indicating an invalid OBS bucket name was", - "doc_type":"usermanual", - "kw":"Why Is Error Invalid OBS Bucket Name Reported After a Flink Job Submission Failed?,Flink Jar Jobs,Us", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error Invalid OBS Bucket Name Reported After a Flink Job Submission Failed?", - "githuburl":"" - }, - { - "uri":"dli_03_0234.html", - "node_id":"dli_03_0234.xml", - "product_code":"dli", - "code":"151", - "des":"Flink Job submission failed. The exception information is as follows:Flink JAR files conflicted. The submitted Flink JAR file conflicted with the HDFS JAR file of the DLI", - "doc_type":"usermanual", - "kw":"Why Does the Flink Submission Fail Due to Hadoop JAR File Conflict?,Flink Jar Jobs,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does the Flink Submission Fail Due to Hadoop JAR File Conflict?", - "githuburl":"" - }, - { - "uri":"dli_03_0266.html", - "node_id":"dli_03_0266.xml", - "product_code":"dli", - "code":"152", - "des":"You can use Flink Jar to connect to Kafka with SASL SSL authentication enabled.", - "doc_type":"usermanual", - "kw":"How Do I Connect a Flink jar Job to SASL_SSL?,Flink Jar Jobs,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Connect a Flink jar Job to SASL_SSL?", - "githuburl":"" - }, - { - "uri":"dli_03_0133.html", - "node_id":"dli_03_0133.xml", - "product_code":"dli", - "code":"153", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Performance Tuning", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Performance Tuning", - "githuburl":"" - }, - { - "uri":"dli_03_0106.html", - "node_id":"dli_03_0106.xml", - "product_code":"dli", - "code":"154", - "des":"Data Stacking in a Consumer GroupThe accumulated data of a consumer group can be calculated by the following formula: Total amount of data to be consumed by the consumer ", - "doc_type":"usermanual", - "kw":"How Do I Optimize Performance of a Flink Job?,Performance Tuning,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Optimize Performance of a Flink Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0048.html", - "node_id":"dli_03_0048.xml", - "product_code":"dli", - "code":"155", - "des":"Add the following SQL statements to the Flink job:", - "doc_type":"usermanual", - "kw":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?,Performance Tuning,User Guid", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0096.html", - "node_id":"dli_03_0096.xml", - "product_code":"dli", - "code":"156", - "des":"The DLI Flink checkpoint/savepoint mechanism is complete and reliable. You can use this mechanism to prevent data loss when a job is manually restarted or restarted due t", - "doc_type":"usermanual", - "kw":"How Do I Prevent Data Loss After Flink Job Restart?,Performance Tuning,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Prevent Data Loss After Flink Job Restart?", - "githuburl":"" - }, - { - "uri":"dli_03_0135.html", - "node_id":"dli_03_0135.xml", - "product_code":"dli", - "code":"157", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"O&M Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"O&M Guide", - "githuburl":"" - }, - { - "uri":"dli_03_0103.html", - "node_id":"dli_03_0103.xml", - "product_code":"dli", - "code":"158", - "des":"On the Flink job management page, hover the cursor on the status of the job that fails to be submitted to view the brief information about the failure.The possible causes", - "doc_type":"usermanual", - "kw":"How Do I Locate a Flink Job Submission Error?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Locate a Flink Job Submission Error?", - "githuburl":"" - }, - { - "uri":"dli_03_0105.html", - "node_id":"dli_03_0105.xml", - "product_code":"dli", - "code":"159", - "des":"On the Flink job management, click Edit in the Operation column of the target job. On the displayed page, check whether Save Job Log in the Running Parameters tab is enab", - "doc_type":"usermanual", - "kw":"How Do I Locate a Flink Job Running Error?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Locate a Flink Job Running Error?", - "githuburl":"" - }, - { - "uri":"dli_03_0136.html", - "node_id":"dli_03_0136.xml", - "product_code":"dli", - "code":"160", - "des":"Flink's checkpointing is a fault tolerance and recovery mechanism. This mechanism ensures that real-time programs can self-recover in case of exceptions or machine issues", - "doc_type":"usermanual", - "kw":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?,O&M Guide,User", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?", - "githuburl":"" - }, - { - "uri":"dli_03_0040.html", - "node_id":"dli_03_0040.xml", - "product_code":"dli", - "code":"161", - "des":"To rectify this fault, perform the following steps:Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statemen", - "doc_type":"usermanual", - "kw":"Why Does DIS Stream Not Exist During Job Semantic Check?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does DIS Stream Not Exist During Job Semantic Check?", - "githuburl":"" - }, - { - "uri":"dli_03_0045.html", - "node_id":"dli_03_0045.xml", - "product_code":"dli", - "code":"162", - "des":"If the OBS bucket selected for a job is not authorized, perform the following steps:Select Enable Checkpointing or Save Job Log.Specify OBS Bucket.Select Authorize OBS.", - "doc_type":"usermanual", - "kw":"Why Is the OBS Bucket Selected for Job Not Authorized?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is the OBS Bucket Selected for Job Not Authorized?", - "githuburl":"" - }, - { - "uri":"dli_03_0064.html", - "node_id":"dli_03_0064.xml", - "product_code":"dli", - "code":"163", - "des":"Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:If the submission fails, a submission log is genera", - "doc_type":"usermanual", - "kw":"Why Are Logs Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?,", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Are Logs Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?", - "githuburl":"" - }, - { - "uri":"dli_03_0235.html", - "node_id":"dli_03_0235.xml", - "product_code":"dli", - "code":"164", - "des":"The Flink/Spark UI was displayed with incomplete information.When the queue is used to run a job, the system releases the cluster and takes about 10 minutes to create a n", - "doc_type":"usermanual", - "kw":"Why Is Information Displayed on the FlinkUI/Spark UI Page Incomplete?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Information Displayed on the FlinkUI/Spark UI Page Incomplete?", - "githuburl":"" - }, - { - "uri":"dli_03_0236.html", - "node_id":"dli_03_0236.xml", - "product_code":"dli", - "code":"165", - "des":"JobManager and TaskManager heartbeats timed out. As a result, the Flink job is abnormal.Check whether the network is intermittently disconnected and whether the cluster l", - "doc_type":"usermanual", - "kw":"Why Is the Flink Job Abnormal Due to Heartbeat Timeout Between JobManager and TaskManager?,O&M Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is the Flink Job Abnormal Due to Heartbeat Timeout Between JobManager and TaskManager?", - "githuburl":"" - }, - { - "uri":"dli_03_0265.html", - "node_id":"dli_03_0265.xml", - "product_code":"dli", - "code":"166", - "des":"Test address connectivity.If the network is unreachable, rectify the network connection first. Ensure that the network connection between the DLI queue and the external d", - "doc_type":"usermanual", - "kw":"Why Is Error \"Timeout expired while fetching topic metadata\" Repeatedly Reported in Flink JobManager", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"Timeout expired while fetching topic metadata\" Repeatedly Reported in Flink JobManager Logs?", - "githuburl":"" - }, - { - "uri":"dli_03_0020.html", - "node_id":"dli_03_0020.xml", - "product_code":"dli", - "code":"167", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Problems Related to SQL Jobs", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Problems Related to SQL Jobs", - "githuburl":"" - }, - { - "uri":"dli_03_0216.html", - "node_id":"dli_03_0216.xml", - "product_code":"dli", - "code":"168", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Usage", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Usage", - "githuburl":"" - }, - { - "uri":"dli_03_0200.html", - "node_id":"dli_03_0200.xml", - "product_code":"dli", - "code":"169", - "des":"A temporary table is used to store intermediate results. When a transaction or session ends, the data in the temporary table can be automatically deleted. For example, in", - "doc_type":"usermanual", - "kw":"SQL Jobs,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"SQL Jobs", - "githuburl":"" - }, - { - "uri":"dli_03_0204.html", - "node_id":"dli_03_0204.xml", - "product_code":"dli", - "code":"170", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Job Development", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Job Development", - "githuburl":"" - }, - { - "uri":"dli_03_0086.html", - "node_id":"dli_03_0086.xml", - "product_code":"dli", - "code":"171", - "des":"If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.Set t", - "doc_type":"usermanual", - "kw":"How Do I Merge Small Files?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Merge Small Files?", - "githuburl":"" - }, - { - "uri":"dli_03_0092.html", - "node_id":"dli_03_0092.xml", - "product_code":"dli", - "code":"172", - "des":"When creating an OBS table, you must specify a table path in the database. The path format is as follows: obs://xxx/database name/table name.If the specified path is akdc", - "doc_type":"usermanual", - "kw":"How Do I Specify an OBS Path When Creating an OBS Table?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Specify an OBS Path When Creating an OBS Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0108.html", - "node_id":"dli_03_0108.xml", - "product_code":"dli", - "code":"173", - "des":"DLI allows you to associate JSON data in an OBS bucket to create tables in asynchronous mode.The statement for creating the table is as follows:", - "doc_type":"usermanual", - "kw":"How Do I Create a Table Using JSON Data in an OBS Bucket?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Create a Table Using JSON Data in an OBS Bucket?", - "githuburl":"" - }, - { - "uri":"dli_03_0087.html", - "node_id":"dli_03_0087.xml", - "product_code":"dli", - "code":"174", - "des":"You can use the where condition statement in the select statement to filter data. For example:", - "doc_type":"usermanual", - "kw":"How Do I Set Local Variables in SQL Statements?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Set Local Variables in SQL Statements?", - "githuburl":"" - }, - { - "uri":"dli_03_0069.html", - "node_id":"dli_03_0069.xml", - "product_code":"dli", - "code":"175", - "des":"The correct method for using the count function to perform aggregation is as follows:OrIf an incorrect method is used, an error will be reported.", - "doc_type":"usermanual", - "kw":"How Can I Use the count Function to Perform Aggregation?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Can I Use the count Function to Perform Aggregation?", - "githuburl":"" - }, - { - "uri":"dli_03_0072.html", - "node_id":"dli_03_0072.xml", - "product_code":"dli", - "code":"176", - "des":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket.Use the OBS cro", - "doc_type":"usermanual", - "kw":"How Do I Synchronize DLI Table Data from One Region to Another?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Synchronize DLI Table Data from One Region to Another?", - "githuburl":"" - }, - { - "uri":"dli_03_0191.html", - "node_id":"dli_03_0191.xml", - "product_code":"dli", - "code":"177", - "des":"Currently, DLI does not allow you to insert table data into specific fields. To insert table data, you must insert data of all table fields at a time.", - "doc_type":"usermanual", - "kw":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?,Job Development,User Gui", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0206.html", - "node_id":"dli_03_0206.xml", - "product_code":"dli", - "code":"178", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Job O&M Errors", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Job O&M Errors", - "githuburl":"" - }, - { - "uri":"dli_03_0014.html", - "node_id":"dli_03_0014.xml", - "product_code":"dli", - "code":"179", - "des":"Create an OBS directory with a unique name. Alternatively, you can manually delete the existing OBS directory and submit the job again. However, exercise caution when del", - "doc_type":"usermanual", - "kw":"Why Is Error \"path obs://xxx already exists\" Reported When Data Is Exported to OBS?,Job O&M Errors,U", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"path obs://xxx already exists\" Reported When Data Is Exported to OBS?", - "githuburl":"" - }, - { - "uri":"dli_03_0066.html", - "node_id":"dli_03_0066.xml", - "product_code":"dli", - "code":"180", - "des":"This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.For example, ta", - "doc_type":"usermanual", - "kw":"Why Is Error \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Displayed W", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Displayed When Two Tables Are Joined?", - "githuburl":"" - }, - { - "uri":"dli_03_0071.html", - "node_id":"dli_03_0071.xml", - "product_code":"dli", - "code":"181", - "des":"Check if your account is in arrears and top it up if necessary.If the same error message persists after the top-up, log out of your account and log back in.", - "doc_type":"usermanual", - "kw":"Why Is Error \"The current account does not have permission to perform this operation,the current acc", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget.\" Reported when a SQL Statement Is Executed?", - "githuburl":"" - }, - { - "uri":"dli_03_0145.html", - "node_id":"dli_03_0145.xml", - "product_code":"dli", - "code":"182", - "des":"Cause AnalysisWhen you query the partitioned table XX.YYY, the partition column is not specified in the search criteria.A partitioned table can be queried only when the q", - "doc_type":"usermanual", - "kw":"Why Is Error \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" ", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" Reported When a Query Statement Is Executed?", - "githuburl":"" - }, - { - "uri":"dli_03_0169.html", - "node_id":"dli_03_0169.xml", - "product_code":"dli", - "code":"183", - "des":"The following error message is displayed when the LOAD DATA command is executed by a Spark SQL job to import data to a DLI table:In some cases ,the following error messag", - "doc_type":"usermanual", - "kw":"Why Is Error \"IllegalArgumentException: Buffer size too small. size\" Reported When Data Is Loaded to", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"IllegalArgumentException: Buffer size too small. size\" Reported When Data Is Loaded to an OBS Foreign Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0189.html", - "node_id":"dli_03_0189.xml", - "product_code":"dli", - "code":"184", - "des":"An error is reported during SQL job execution:Please contact DLI service. DLI.0002: FileNotFoundException: getFileStatus on obs://xxx: status [404]Check whether there is ", - "doc_type":"usermanual", - "kw":"Why Is Error \"DLI.0002 FileNotFoundException\" Reported During SQL Job Running?,Job O&M Errors,User G", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"DLI.0002 FileNotFoundException\" Reported During SQL Job Running?", - "githuburl":"" - }, - { - "uri":"dli_03_0046.html", - "node_id":"dli_03_0046.xml", - "product_code":"dli", - "code":"185", - "des":"Currently, DLI supports the Hive syntax for creating tables of the TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET file types. If the file format specified for cre", - "doc_type":"usermanual", - "kw":"Why Is a Schema Parsing Error Reported When I Create a Hive Table Using CTAS?,Job O&M Errors,User Gu", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is a Schema Parsing Error Reported When I Create a Hive Table Using CTAS?", - "githuburl":"" - }, - { - "uri":"dli_03_0173.html", - "node_id":"dli_03_0173.xml", - "product_code":"dli", - "code":"186", - "des":"When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:DLI.0999: RuntimeException: o", - "doc_type":"usermanual", - "kw":"Why Is Error \"org.apache.hadoop.fs.obs.OBSIOException\" Reported When I Run DLI SQL Scripts on DataAr", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"org.apache.hadoop.fs.obs.OBSIOException\" Reported When I Run DLI SQL Scripts on DataArts Studio?", - "githuburl":"" - }, - { - "uri":"dli_03_0172.html", - "node_id":"dli_03_0172.xml", - "product_code":"dli", - "code":"187", - "des":"After the migration job is submitted, the following error information is displayed in the log:org.apache.sqoop.common.SqoopException:UQUERY_CONNECTOR_0001:Invoke DLI serv", - "doc_type":"usermanual", - "kw":"Why Is Error \"UQUERY_CONNECTOR_0001:Invoke DLI service api failed\" Reported in the Job Log When I Us", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"UQUERY_CONNECTOR_0001:Invoke DLI service api failed\" Reported in the Job Log When I Use CDM to Migrate Data to DLI?", - "githuburl":"" - }, - { - "uri":"dli_03_0207.html", - "node_id":"dli_03_0207.xml", - "product_code":"dli", - "code":"188", - "des":"Error message \"File not Found\" is displayed when a SQL job is accessed.Generally, the file cannot be found due to a read/write conflict. Check whether a job is overwritin", - "doc_type":"usermanual", - "kw":"Why Is Error \"File not Found\" Reported When I Access a SQL Job?,Job O&M Errors,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"File not Found\" Reported When I Access a SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0208.html", - "node_id":"dli_03_0208.xml", - "product_code":"dli", - "code":"189", - "des":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.Check the OBS bucket written in the AccessControlException to confirm if your ", - "doc_type":"usermanual", - "kw":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?,Job O&M Errors", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0209.html", - "node_id":"dli_03_0209.xml", - "product_code":"dli", - "code":"190", - "des":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Yo", - "doc_type":"usermanual", - "kw":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{b", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" Reported When I Access a SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0210.html", - "node_id":"dli_03_0210.xml", - "product_code":"dli", - "code":"191", - "des":"Error message \"The current account does not have permission to perform this operation,the current account was restricted.\" is reported during SQL statement execution.Chec", - "doc_type":"usermanual", - "kw":"Why Is Error \"The current account does not have permission to perform this operation,the current acc", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget\" Reported During SQL Statement Execution? Restricted for no budget.", - "githuburl":"" - }, - { - "uri":"dli_03_0211.html", - "node_id":"dli_03_0211.xml", - "product_code":"dli", - "code":"192", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"O&M Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"O&M Guide", - "githuburl":"" - }, - { - "uri":"dli_03_0196.html", - "node_id":"dli_03_0196.xml", - "product_code":"dli", - "code":"193", - "des":"If the job runs slowly, perform the following steps to find the causes and rectify the fault:Check whether the problem is caused by FullGC.Log in to the DLI console. In t", - "doc_type":"usermanual", - "kw":"How Do I Troubleshoot Slow SQL Jobs?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Troubleshoot Slow SQL Jobs?", - "githuburl":"" - }, - { - "uri":"dli_03_0091.html", - "node_id":"dli_03_0091.xml", - "product_code":"dli", - "code":"194", - "des":"You can view SQL job logs for routine O&M.Obtain the ID of the DLI job executed on the DataArts Studio console.Job IDOn the DLI console, choose Job Management > SQL Jobs.", - "doc_type":"usermanual", - "kw":"How Do I View DLI SQL Logs?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I View DLI SQL Logs?", - "githuburl":"" - }, - { - "uri":"dli_03_0116.html", - "node_id":"dli_03_0116.xml", - "product_code":"dli", - "code":"195", - "des":"You can view the job execution records when a job is running.Log in to the DLI management console.In the navigation pane on the left, choose Job Management > SQL Jobs.Ent", - "doc_type":"usermanual", - "kw":"How Do I View SQL Execution Records?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I View SQL Execution Records?", - "githuburl":"" - }, - { - "uri":"dli_03_0093.html", - "node_id":"dli_03_0093.xml", - "product_code":"dli", - "code":"196", - "des":"Data skew is a common issue during the execution of SQL jobs. When data is unevenly distributed, some compute nodes process significantly more data than others, which can", - "doc_type":"usermanual", - "kw":"How Do I Do When Data Skew Occurs During the Execution of a SQL Job?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Do When Data Skew Occurs During the Execution of a SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0184.html", - "node_id":"dli_03_0184.xml", - "product_code":"dli", - "code":"197", - "des":"A DLI table exists but cannot be queried on the DLI console.If a table exists but cannot be queried, there is a high probability that the current user does not have the p", - "doc_type":"usermanual", - "kw":"What Can I Do If a Table Cannot Be Queried on the DLI Console?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"What Can I Do If a Table Cannot Be Queried on the DLI Console?", - "githuburl":"" - }, - { - "uri":"dli_03_0013.html", - "node_id":"dli_03_0013.xml", - "product_code":"dli", - "code":"198", - "des":"A high compression ratio of OBS tables in the Parquet or ORC format (for example, a compression ratio of 5 or higher compared with text compression) will lead to large da", - "doc_type":"usermanual", - "kw":"The Compression Ratio of OBS Tables Is Too High,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"The Compression Ratio of OBS Tables Is Too High", - "githuburl":"" - }, - { - "uri":"dli_03_0009.html", - "node_id":"dli_03_0009.xml", - "product_code":"dli", - "code":"199", - "des":"DLI supports only UTF-8-encoded texts. Ensure that data is encoded using UTF-8 during table creation and import.", - "doc_type":"usermanual", - "kw":"How Can I Avoid Garbled Characters Caused by Inconsistent Character Codes?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Can I Avoid Garbled Characters Caused by Inconsistent Character Codes?", - "githuburl":"" - }, - { - "uri":"dli_03_0175.html", - "node_id":"dli_03_0175.xml", - "product_code":"dli", - "code":"200", - "des":"User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table a", - "doc_type":"usermanual", - "kw":"Do I Need to Grant Table Permissions to a User and Project After I Delete a Table and Create One wit", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Do I Need to Grant Table Permissions to a User and Project After I Delete a Table and Create One with the Same Name?", - "githuburl":"" - }, - { - "uri":"dli_03_0177.html", - "node_id":"dli_03_0177.xml", - "product_code":"dli", - "code":"201", - "des":"A CSV file is imported to a DLI partitioned table, but the imported file data does not contain the data in the partitioning column. The partitioning column needs to be sp", - "doc_type":"usermanual", - "kw":"Why Can't I Query Table Data After Data Is Imported to a DLI Partitioned Table Because the File to B", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Can't I Query Table Data After Data Is Imported to a DLI Partitioned Table Because the File to Be Imported Does Not Contain Data in the Partitioning Column?", - "githuburl":"" - }, - { - "uri":"dli_03_0181.html", - "node_id":"dli_03_0181.xml", - "product_code":"dli", - "code":"202", - "des":"When an OBS foreign table is created, a field in the specified OBS file contains a carriage return line feed (CRLF) character. As a result, the data is incorrect.The stat", - "doc_type":"usermanual", - "kw":"How Do I Fix the Data Error Caused by CRLF Characters in a Field of the OBS File Used to Create an E", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Fix the Data Error Caused by CRLF Characters in a Field of the OBS File Used to Create an External OBS Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0182.html", - "node_id":"dli_03_0182.xml", - "product_code":"dli", - "code":"203", - "des":"A SQL job contains join operations. After the job is submitted, it is stuck in the Running state and no result is returned.When a Spark SQL job has join operations on sma", - "doc_type":"usermanual", - "kw":"Why Does a SQL Job That Has Join Operations Stay in the Running State?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does a SQL Job That Has Join Operations Stay in the Running State?", - "githuburl":"" - }, - { - "uri":"dli_03_0187.html", - "node_id":"dli_03_0187.xml", - "product_code":"dli", - "code":"204", - "des":"The on clause was not added to the SQL statement for joining tables. As a result, the Cartesian product query occurs due to multi-table association, and the queue resourc", - "doc_type":"usermanual", - "kw":"The on Clause Is Not Added When Tables Are Joined. Cartesian Product Query Causes High Resource Usag", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"The on Clause Is Not Added When Tables Are Joined. Cartesian Product Query Causes High Resource Usage of the Queue, and the Job Fails to Be Executed", - "githuburl":"" - }, - { - "uri":"dli_03_0190.html", - "node_id":"dli_03_0190.xml", - "product_code":"dli", - "code":"205", - "des":"Partition data is manually uploaded to a partition of an OBS table. However, the data cannot be queried using DLI SQL editor.After manually adding partition data, you nee", - "doc_type":"usermanual", - "kw":"Why Can't I Query Data After I Manually Add Data to the Partition Directory of an OBS Table?,O&M Gui", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Can't I Query Data After I Manually Add Data to the Partition Directory of an OBS Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0212.html", - "node_id":"dli_03_0212.xml", - "product_code":"dli", - "code":"206", - "des":"To dynamically overwrite the specified partitioned data in the DataSource table, set dli.sql.dynamicPartitionOverwrite.enabled to true and then run the insert overwrite s", - "doc_type":"usermanual", - "kw":"Why Is All Data Overwritten When insert overwrite Is Used to Overwrite Partitioned Table?,O&M Guide,", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is All Data Overwritten When insert overwrite Is Used to Overwrite Partitioned Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0213.html", - "node_id":"dli_03_0213.xml", - "product_code":"dli", - "code":"207", - "des":"The possible causes and solutions are as follows:After you purchase a DLI queue and submit a SQL job for the first time, wait for 5 to 10 minutes. After the cluster is st", - "doc_type":"usermanual", - "kw":"Why Is a SQL Job Stuck in the Submitting State?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is a SQL Job Stuck in the Submitting State?", - "githuburl":"" - }, - { - "uri":"dli_03_0214.html", - "node_id":"dli_03_0214.xml", - "product_code":"dli", - "code":"208", - "des":"Spark does not have the datetime type and uses the TIMESTAMP type instead.You can use a function to convert data types.The following is an example.select cast(create_date", - "doc_type":"usermanual", - "kw":"Why Is the create_date Field in the RDS Table Is a Timestamp in the DLI query result?,O&M Guide,User", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is the create_date Field in the RDS Table Is a Timestamp in the DLI query result?", - "githuburl":"" - }, - { - "uri":"dli_03_0215.html", - "node_id":"dli_03_0215.xml", - "product_code":"dli", - "code":"209", - "des":"If the table name is changed immediately after SQL statements are executed, the data size of the table may be incorrect.If you need to change the table name, change it 5 ", - "doc_type":"usermanual", - "kw":"What Can I Do If datasize Cannot Be Changed After the Table Name Is Changed in a Finished SQL Job?,O", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"What Can I Do If datasize Cannot Be Changed After the Table Name Is Changed in a Finished SQL Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0231.html", - "node_id":"dli_03_0231.xml", - "product_code":"dli", - "code":"210", - "des":"When DLI is used to insert data into an OBS temporary table, only part of data is imported.Possible causes are as follows:The amount of data read during job execution is ", - "doc_type":"usermanual", - "kw":"Why Is the Data Volume Changes When Data Is Imported from DLI to OBS?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is the Data Volume Changes When Data Is Imported from DLI to OBS?", - "githuburl":"" - }, - { - "uri":"dli_03_0021.html", - "node_id":"dli_03_0021.xml", - "product_code":"dli", - "code":"211", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Problems Related to Spark Jobs", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Problems Related to Spark Jobs", - "githuburl":"" - }, - { - "uri":"dli_03_0163.html", - "node_id":"dli_03_0163.xml", - "product_code":"dli", - "code":"212", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Usage", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Usage", - "githuburl":"" - }, - { - "uri":"dli_03_0201.html", - "node_id":"dli_03_0201.xml", - "product_code":"dli", - "code":"213", - "des":"DLI Spark does not support job scheduling. You can use other services, such as DataArts Studio, or use APIs or SDKs to customize job schedule.The Spark SQL syntax does no", - "doc_type":"usermanual", - "kw":"Spark Jobs,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Spark Jobs", - "githuburl":"" - }, - { - "uri":"dli_03_0217.html", - "node_id":"dli_03_0217.xml", - "product_code":"dli", - "code":"214", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Job Development", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Job Development", - "githuburl":"" - }, - { - "uri":"dli_03_0107.html", - "node_id":"dli_03_0107.xml", - "product_code":"dli", - "code":"215", - "des":"To use Spark to write data into a DLI table, configure the following parameters:fs.obs.access.keyfs.obs.secret.keyfs.obs.implfs.obs.endpointThe following is an example:", - "doc_type":"usermanual", - "kw":"How Do I Use Spark to Write Data into a DLI Table?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Use Spark to Write Data into a DLI Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0017.html", - "node_id":"dli_03_0017.xml", - "product_code":"dli", - "code":"216", - "des":"To obtain the AK/SK, set the parameters as follows:Create a SparkContext using code.val sc: SparkContext = new SparkContext()\nsc.hadoopConfiguration.set(\"fs.obs.access.ke", - "doc_type":"usermanual", - "kw":"How Do I Set the AK/SK for a Queue to Operate an OBS Table?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Set the AK/SK for a Queue to Operate an OBS Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0102.html", - "node_id":"dli_03_0102.xml", - "product_code":"dli", - "code":"217", - "des":"Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs. In the job list, locate the target job and click next to Job ID to view the parame", - "doc_type":"usermanual", - "kw":"How Do I View the Resource Usage of DLI Spark Jobs?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I View the Resource Usage of DLI Spark Jobs?", - "githuburl":"" - }, - { - "uri":"dli_03_0076.html", - "node_id":"dli_03_0076.xml", - "product_code":"dli", - "code":"218", - "des":"If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management p", - "doc_type":"usermanual", - "kw":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the S", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the Spark Job Results Stored in MySQL?", - "githuburl":"" - }, - { - "uri":"dli_03_0082.html", - "node_id":"dli_03_0082.xml", - "product_code":"dli", - "code":"219", - "des":"DLI natively supports PySpark.For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are pac", - "doc_type":"usermanual", - "kw":"How Do I Run a Complex PySpark Program in DLI?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Run a Complex PySpark Program in DLI?", - "githuburl":"" - }, - { - "uri":"dli_03_0127.html", - "node_id":"dli_03_0127.xml", - "product_code":"dli", - "code":"220", - "des":"You can use DLI Spark jobs to access data in the MySQL database using either of the following methods:Solution 1: Buy a queue, create an enhanced datasource connection, a", - "doc_type":"usermanual", - "kw":"How Does a Spark Job Access a MySQL Database?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Does a Spark Job Access a MySQL Database?", - "githuburl":"" - }, - { - "uri":"dli_03_0068.html", - "node_id":"dli_03_0068.xml", - "product_code":"dli", - "code":"221", - "des":"When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.To solve this problem, you can config", - "doc_type":"usermanual", - "kw":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", - "githuburl":"" - }, - { - "uri":"dli_03_0118.html", - "node_id":"dli_03_0118.xml", - "product_code":"dli", - "code":"222", - "des":"You can use SparkFiles to read the file submitted using –-file form a local path: SparkFiles.get(\"Name of the uploaded file\").The file path in the Driver is different fro", - "doc_type":"usermanual", - "kw":"How Do I Read Uploaded Files for a Spark Jar Job?,Job Development,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Read Uploaded Files for a Spark Jar Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0218.html", - "node_id":"dli_03_0218.xml", - "product_code":"dli", - "code":"223", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Job O&M Errors", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Job O&M Errors", - "githuburl":"" - }, - { - "uri":"dli_03_0156.html", - "node_id":"dli_03_0156.xml", - "product_code":"dli", - "code":"224", - "des":"The following error is reported when a Spark job accesses OBS data:Set the AK/SK to enable Spark jobs to access OBS data. For details, see How Do I Set the AK/SK for a Qu", - "doc_type":"usermanual", - "kw":"Why Are Errors \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Reported When a Spark Job Accesse", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Are Errors \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Reported When a Spark Job Accesses OBS Data?", - "githuburl":"" - }, - { - "uri":"dli_03_0164.html", - "node_id":"dli_03_0164.xml", - "product_code":"dli", - "code":"225", - "des":"Check whether the OBS bucket is used to store DLI logs on the Global Configuration > Job Configurations page. The job log bucket cannot be used for other purpose.", - "doc_type":"usermanual", - "kw":"Why Is Error \"verifyBucketExists on XXXX: status [403]\" Reported When I Use a Spark Job to Access an", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"verifyBucketExists on XXXX: status [403]\" Reported When I Use a Spark Job to Access an OBS Bucket That I Have Access Permission?", - "githuburl":"" - }, - { - "uri":"dli_03_0157.html", - "node_id":"dli_03_0157.xml", - "product_code":"dli", - "code":"226", - "des":"When a Spark job accesses a large amount of data, for example, accessing data in a GaussDB(DWS) database, you are advised to set the number of concurrent tasks and enable", - "doc_type":"usermanual", - "kw":"Why Is a Job Running Timeout Reported When a Spark Job Runs a Large Amount of Data?,Job O&M Errors,U", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is a Job Running Timeout Reported When a Spark Job Runs a Large Amount of Data?", - "githuburl":"" - }, - { - "uri":"dli_03_0188.html", - "node_id":"dli_03_0188.xml", - "product_code":"dli", - "code":"227", - "des":"Spark jobs cannot access SFTP. Upload the files you want to access to OBS and then you can analyze the data using Spark jobs.", - "doc_type":"usermanual", - "kw":"Why Does the Job Fail to Be Executed and the Log Shows that the File Directory Is Abnormal When I Us", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does the Job Fail to Be Executed and the Log Shows that the File Directory Is Abnormal When I Use a Spark Job to Access Files in SFTP?", - "githuburl":"" - }, - { - "uri":"dli_03_0192.html", - "node_id":"dli_03_0192.xml", - "product_code":"dli", - "code":"228", - "des":"When a Spark job is running, an error message is displayed, indicating that the user does not have the database permission. The error information is as follows:org.apache", - "doc_type":"usermanual", - "kw":"Why Does the Job Fail to Be Executed Due to Insufficient Database and Table Permissions?,Job O&M Err", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does the Job Fail to Be Executed Due to Insufficient Database and Table Permissions?", - "githuburl":"" - }, - { - "uri":"dli_03_0219.html", - "node_id":"dli_03_0219.xml", - "product_code":"dli", - "code":"229", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"O&M Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"O&M Guide", - "githuburl":"" - }, - { - "uri":"dli_03_0077.html", - "node_id":"dli_03_0077.xml", - "product_code":"dli", - "code":"230", - "des":"I cannot find the specified Python environment after adding the Python 3 package.Set spark.yarn.appMasterEnv.PYSPARK_PYTHON to python3 in the conf file to specify the Pyt", - "doc_type":"usermanual", - "kw":"Why Can't I Find the Specified Python Environment After Adding the Python Package?,O&M Guide,User Gu", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Can't I Find the Specified Python Environment After Adding the Python Package?", - "githuburl":"" - }, - { - "uri":"dli_03_0220.html", - "node_id":"dli_03_0220.xml", - "product_code":"dli", - "code":"231", - "des":"The remaining CUs in the queue may be insufficient. As a result, the job cannot be submitted.To view the remaining CUs of a queue, perform the following steps:Check the C", - "doc_type":"usermanual", - "kw":"Why Is a Spark Jar Job Stuck in the Submitting State?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is a Spark Jar Job Stuck in the Submitting State?", - "githuburl":"" - }, { "uri":"dli_03_0001.html", "node_id":"dli_03_0001.xml", "product_code":"dli", - "code":"232", + "code":"148", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"Product Consultation", + "kw":"DLI Basics", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" } ], - "title":"Product Consultation", - "githuburl":"" - }, - { - "uri":"dli_03_0221.html", - "node_id":"dli_03_0221.xml", - "product_code":"dli", - "code":"233", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Usage", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Usage", - "githuburl":"" - }, - { - "uri":"dli_03_0025.html", - "node_id":"dli_03_0025.xml", - "product_code":"dli", - "code":"234", - "des":"DLI supports the following data formats:ParquetCSVORCJsonAvro", - "doc_type":"usermanual", - "kw":"Which Data Formats Does DLI Support?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Which Data Formats Does DLI Support?", + "title":"DLI Basics", "githuburl":"" }, { "uri":"dli_03_0115.html", "node_id":"dli_03_0115.xml", "product_code":"dli", - "code":"235", - "des":"The Spark component of DLI is a fully managed service. You can only use the DLI Spark through its APIs. .The Spark component of MRS is built on the VM in an MRS cluster. ", + "code":"149", + "des":"Both DLI and MRS support Spark, but there are some differences in service mode, interface, application scenarios, and performance characteristics.The Spark component of D", "doc_type":"usermanual", - "kw":"What Are the Differences Between MRS Spark and DLI Spark?,Usage,User Guide", + "kw":"What Are the Differences Between MRS Spark and DLI Spark?,DLI Basics,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" } ], "title":"What Are the Differences Between MRS Spark and DLI Spark?", @@ -4452,424 +3133,10 @@ "uri":"dli_03_0029.html", "node_id":"dli_03_0029.xml", "product_code":"dli", - "code":"236", - "des":"DLI data can be stored in either of the following:OBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.DLI: The column-base", + "code":"150", + "des":"Supported data formats:ParquetCSVORCJSONAvroOBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.DLI: The column-based Parq", "doc_type":"usermanual", - "kw":"Where Can DLI Data Be Stored?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Where Can DLI Data Be Stored?", - "githuburl":"" - }, - { - "uri":"dli_03_0117.html", - "node_id":"dli_03_0117.xml", - "product_code":"dli", - "code":"237", - "des":"DLI tables store data within the DLI service, and you do not need to know the data storage path.OBS tables store data in your OBS buckets, and you need to manage the sour", - "doc_type":"usermanual", - "kw":"What Are the Differences Between DLI Tables and OBS Tables?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"What Are the Differences Between DLI Tables and OBS Tables?", - "githuburl":"" - }, - { - "uri":"dli_03_0010.html", - "node_id":"dli_03_0010.xml", - "product_code":"dli", - "code":"238", - "des":"Currently, DLI supports analysis only on the data uploaded to the cloud. In scenarios where regular (for example, on a per day basis) one-off analysis on incremental data", - "doc_type":"usermanual", - "kw":"How Can I Use DLI If Data Is Not Uploaded to OBS?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Can I Use DLI If Data Is Not Uploaded to OBS?", - "githuburl":"" - }, - { - "uri":"dli_03_0129.html", - "node_id":"dli_03_0129.xml", - "product_code":"dli", - "code":"239", - "des":"Data in the OBS bucket shared by IAM users under the same account can be imported. You cannot import data in the OBS bucket shared with other IAM account.", - "doc_type":"usermanual", - "kw":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?", - "githuburl":"" - }, - { - "uri":"dli_03_0264.html", - "node_id":"dli_03_0264.xml", - "product_code":"dli", - "code":"240", - "des":"Log in to the management console.Click in the upper left corner and select a region and a project.Click the My Quota icon in the upper right corner of the page.The Serv", - "doc_type":"usermanual", - "kw":"Why Is Error \"Failed to create the database. {\"error_code\":\"DLI.1028\";\"error_msg\":\"Already reached t", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"Failed to create the database. {\"error_code\":\"DLI.1028\";\"error_msg\":\"Already reached the maximum quota of databases:XXX\".\" Reported?", - "githuburl":"" - }, - { - "uri":"dli_03_0263.html", - "node_id":"dli_03_0263.xml", - "product_code":"dli", - "code":"241", - "des":"No, a global variable can only be used by the user who created it. Global variables can be used to simplify complex parameters. For example, long and difficult variables ", - "doc_type":"usermanual", - "kw":"Can a Member Account Use Global Variables Created by Other Member Accounts?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Can a Member Account Use Global Variables Created by Other Member Accounts?", - "githuburl":"" - }, - { - "uri":"dli_03_0222.html", - "node_id":"dli_03_0222.xml", - "product_code":"dli", - "code":"242", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Job Management", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Job Management", - "githuburl":"" - }, - { - "uri":"dli_03_0126.html", - "node_id":"dli_03_0126.xml", - "product_code":"dli", - "code":"243", - "des":"If you are suggested to perform following operations to run a large number of DLI jobs:Group the DLI jobs by type, and run each group on a queue.Alternatively, create IAM", - "doc_type":"usermanual", - "kw":"How Do I Manage Tens of Thousands of Jobs Running on DLI?,Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Manage Tens of Thousands of Jobs Running on DLI?", - "githuburl":"" - }, - { - "uri":"dli_03_0162.html", - "node_id":"dli_03_0162.xml", - "product_code":"dli", - "code":"244", - "des":"The field names of tables that have been created cannot be changed.You can create a table, define new table fields, and migrate data from the old table to the new one.", - "doc_type":"usermanual", - "kw":"How Do I Change the Name of a Field in a Created Table?,Job Management,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Change the Name of a Field in a Created Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0261.html", - "node_id":"dli_03_0261.xml", - "product_code":"dli", - "code":"245", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Privacy and Security", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Privacy and Security", - "githuburl":"" - }, - { - "uri":"dli_03_0260.html", - "node_id":"dli_03_0260.xml", - "product_code":"dli", - "code":"246", - "des":"No. The spark.acls.enable configuration item is not used in DLI. The Apache Spark command injection vulnerability (CVE-2022-33891) does not exist in DLI.", - "doc_type":"usermanual", - "kw":"Does DLI Have the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?,Privacy and Securit", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Does DLI Have the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?", - "githuburl":"" - }, - { - "uri":"dli_03_0053.html", - "node_id":"dli_03_0053.xml", - "product_code":"dli", - "code":"247", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Quota", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Quota", - "githuburl":"" - }, - { - "uri":"dli_03_0031.html", - "node_id":"dli_03_0031.xml", - "product_code":"dli", - "code":"248", - "des":"Log in to the management console.Click in the upper left corner and select Region and Project.Click (the My Quotas icon) in the upper right corner.The Service Quota pag", - "doc_type":"usermanual", - "kw":"How Do I View My Quotas?,Quota,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I View My Quotas?", - "githuburl":"" - }, - { - "uri":"dli_03_0032.html", - "node_id":"dli_03_0032.xml", - "product_code":"dli", - "code":"249", - "des":"The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your applica", - "doc_type":"usermanual", - "kw":"How Do I Increase a Quota?,Quota,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Increase a Quota?", - "githuburl":"" - }, - { - "uri":"dli_03_0054.html", - "node_id":"dli_03_0054.xml", - "product_code":"dli", - "code":"250", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Permission", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Permission", - "githuburl":"" - }, - { - "uri":"dli_03_0223.html", - "node_id":"dli_03_0223.xml", - "product_code":"dli", - "code":"251", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Usage", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Usage", - "githuburl":"" - }, - { - "uri":"dli_03_0100.html", - "node_id":"dli_03_0100.xml", - "product_code":"dli", - "code":"252", - "des":"DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM", - "doc_type":"usermanual", - "kw":"How Do I Manage Fine-Grained DLI Permissions?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Manage Fine-Grained DLI Permissions?", - "githuburl":"" - }, - { - "uri":"dli_03_0008.html", - "node_id":"dli_03_0008.xml", - "product_code":"dli", - "code":"253", - "des":"You cannot perform permission-related operations on the partition column of a partitioned table.However, when you grant the permission of any non-partition column in a pa", - "doc_type":"usermanual", - "kw":"What Is Column Permission Granting of a DLI Partition Table?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"What Is Column Permission Granting of a DLI Partition Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0226.html", - "node_id":"dli_03_0226.xml", - "product_code":"dli", - "code":"254", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"O&M Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"O&M Guide", - "githuburl":"" - }, - { - "uri":"dli_03_0140.html", - "node_id":"dli_03_0140.xml", - "product_code":"dli", - "code":"255", - "des":"When you submit a job, a message is displayed indicating that the job fails to be submitted due to insufficient permission caused by arrears. In this case, you need to ch", - "doc_type":"usermanual", - "kw":"Why Does My Account Have Insufficient Permissions Due to Arrears?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does My Account Have Insufficient Permissions Due to Arrears?", - "githuburl":"" - }, - { - "uri":"dli_03_0195.html", - "node_id":"dli_03_0195.xml", - "product_code":"dli", - "code":"256", - "des":"When the user update an existing program package, the following error information is displayed:\"error_code\"*DLI.0003\",\"error_msg\":\"Permission denied for resource 'resourc", - "doc_type":"usermanual", - "kw":"Why Does the System Display a Message Indicating Insufficient Permissions When I Update a Program Pa", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Does the System Display a Message Indicating Insufficient Permissions When I Update a Program Package?", - "githuburl":"" - }, - { - "uri":"dli_03_0227.html", - "node_id":"dli_03_0227.xml", - "product_code":"dli", - "code":"257", - "des":"When the SQL query statement is executed, the system displays a message indicating that the user does not have the permission to query resources.Error information: DLI.00", - "doc_type":"usermanual", - "kw":"Why Is Error \"DLI.0003: Permission denied for resource...\" Reported When I Run a SQL Statement?,O&M ", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"DLI.0003: Permission denied for resource...\" Reported When I Run a SQL Statement?", - "githuburl":"" - }, - { - "uri":"dli_03_0228.html", - "node_id":"dli_03_0228.xml", - "product_code":"dli", - "code":"258", - "des":"The table permission has been granted and verified. However, after a period of time, an error is reported indicating that the table query fails.There are two possible rea", - "doc_type":"usermanual", - "kw":"Why Can't I Query Table Data After I've Been Granted Table Permissions?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Can't I Query Table Data After I've Been Granted Table Permissions?", - "githuburl":"" - }, - { - "uri":"dli_03_0057.html", - "node_id":"dli_03_0057.xml", - "product_code":"dli", - "code":"259", - "des":"If a table inherits database permissions, you do not need to regrant the inherited permissions to the table.When you grant permissions on a table on the console:If you se", - "doc_type":"usermanual", - "kw":"Will an Error Be Reported if the Inherited Permissions Are Regranted to a Table That Inherits Databa", + "kw":"Where Can Data Be Stored in DLI?,DLI Basics,User Guide", "search_title":"", "metedata":[ { @@ -4879,17 +3146,2617 @@ "IsBot":"Yes" } ], - "title":"Will an Error Be Reported if the Inherited Permissions Are Regranted to a Table That Inherits Database Permissions?", + "title":"Where Can Data Be Stored in DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0129.html", + "node_id":"dli_03_0129.xml", + "product_code":"dli", + "code":"151", + "des":"DLI supports importing data from OBS buckets shared by IAM users under the same tenant, but not from OBS buckets shared by other tenants.This ensures data security and is", + "doc_type":"usermanual", + "kw":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?,DLI Basics,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0263.html", + "node_id":"dli_03_0263.xml", + "product_code":"dli", + "code":"152", + "des":"No, a global variable can only be used by the user who created it. Global variables can be used to simplify complex parameters. For example, long and difficult variables ", + "doc_type":"usermanual", + "kw":"Can a Member Account Use Global Variables Created by Other Member Accounts?,DLI Basics,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Can a Member Account Use Global Variables Created by Other Member Accounts?", + "githuburl":"" + }, + { + "uri":"dli_03_0260.html", + "node_id":"dli_03_0260.xml", + "product_code":"dli", + "code":"153", + "des":"No.The spark.acls.enable configuration item is not used in DLI. The Apache Spark command injection vulnerability (CVE-2022-33891) does not exist in DLI.This vulnerability", + "doc_type":"usermanual", + "kw":"Is DLI Affected by the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?,DLI Basics,Use", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Is DLI Affected by the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?", + "githuburl":"" + }, + { + "uri":"dli_03_0126.html", + "node_id":"dli_03_0126.xml", + "product_code":"dli", + "code":"154", + "des":"To manage a large number of DLI jobs, you can use the following methods:Manage jobs by group.Group tens of thousands of jobs by type and run each group on a queue.Group t", + "doc_type":"usermanual", + "kw":"How Do I Manage Jobs Running on DLI?,DLI Basics,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Manage Jobs Running on DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0162.html", + "node_id":"dli_03_0162.xml", + "product_code":"dli", + "code":"155", + "des":"DLI does not support directly changing the field names of a table. However, you can solve this issue by migrating the table data using the following steps:Create a table:", + "doc_type":"usermanual", + "kw":"How Do I Change the Field Names of an Existing Table on DLI?,DLI Basics,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Change the Field Names of an Existing Table on DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0049.html", + "node_id":"dli_03_0049.xml", + "product_code":"dli", + "code":"156", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"DLI Elastic Resource Pools and Queues", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"DLI Elastic Resource Pools and Queues", + "githuburl":"" + }, + { + "uri":"dli_03_0276.html", + "node_id":"dli_03_0276.xml", + "product_code":"dli", + "code":"157", + "des":"In daily big data analysis work, it is important to allocate and manage compute resources properly to provide a good job execution environment.You can allocate resources ", + "doc_type":"usermanual", + "kw":"How Can I Check the Actual and Used CUs for an Elastic Resource Pool as Well as the Required CUs for", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Can I Check the Actual and Used CUs for an Elastic Resource Pool as Well as the Required CUs for a Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0183.html", + "node_id":"dli_03_0183.xml", + "product_code":"dli", + "code":"158", + "des":"You need to check the large number of jobs in the Submitting and Running states on the queue.Use Cloud Eye to view jobs in different states on the queue. The procedure is", + "doc_type":"usermanual", + "kw":"How Do I Check for a Backlog of Jobs in the Current DLI Queue?,DLI Elastic Resource Pools and Queues", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Check for a Backlog of Jobs in the Current DLI Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0095.html", + "node_id":"dli_03_0095.xml", + "product_code":"dli", + "code":"159", + "des":"To check the running status of the DLI queue and determine whether to run more jobs on that queue, you need to check the queue load.Search for Cloud Eye on the console.In", + "doc_type":"usermanual", + "kw":"How Do I View the Load of a DLI Queue?,DLI Elastic Resource Pools and Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I View the Load of a DLI Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0098.html", + "node_id":"dli_03_0098.xml", + "product_code":"dli", + "code":"160", + "des":"DLI allows you to subscribe to an SMN topic for failed jobs.Log in to the DLI console.In the navigation pane on the left, choose Queue Management.On the Queue Management ", + "doc_type":"usermanual", + "kw":"How Do I Monitor Job Exceptions on a DLI Queue?,DLI Elastic Resource Pools and Queues,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Monitor Job Exceptions on a DLI Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0065.html", + "node_id":"dli_03_0065.xml", + "product_code":"dli", + "code":"161", + "des":"Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earl", + "doc_type":"usermanual", + "kw":"How Do I Migrate an Old Version Spark Queue to a General-Purpose Queue?,DLI Elastic Resource Pools a", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Migrate an Old Version Spark Queue to a General-Purpose Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0171.html", + "node_id":"dli_03_0171.xml", + "product_code":"dli", + "code":"162", + "des":"After a SQL job was submitted to the default queue, the job runs abnormally. The job log reported that the execution timed out. The exception logs are as follows:[ERROR] ", + "doc_type":"usermanual", + "kw":"How Do I Do If I Encounter a Timeout Exception When Executing DLI SQL Statements on the default Queu", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If I Encounter a Timeout Exception When Executing DLI SQL Statements on the default Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0211.html", + "node_id":"dli_03_0211.xml", + "product_code":"dli", + "code":"163", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"DLI Databases and Tables", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"DLI Databases and Tables", + "githuburl":"" + }, + { + "uri":"dli_03_0184.html", + "node_id":"dli_03_0184.xml", + "product_code":"dli", + "code":"164", + "des":"A DLI table exists but cannot be queried on the DLI console.If a table exists but cannot be queried, there is a high probability that the current user does not have the p", + "doc_type":"usermanual", + "kw":"Why Am I Unable to Query a Table on the DLI Console?,DLI Databases and Tables,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Am I Unable to Query a Table on the DLI Console?", + "githuburl":"" + }, + { + "uri":"dli_03_0013.html", + "node_id":"dli_03_0013.xml", + "product_code":"dli", + "code":"165", + "des":"When submitting a job to import data into a DLI table, if the compression rate of the Parquet/ORC file corresponding to the OBS table is high, exceeding 5 times the compr", + "doc_type":"usermanual", + "kw":"How Do I Do If the Compression Rate of an OBS Table Is High?,DLI Databases and Tables,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If the Compression Rate of an OBS Table Is High?", + "githuburl":"" + }, + { + "uri":"dli_03_0009.html", + "node_id":"dli_03_0009.xml", + "product_code":"dli", + "code":"166", + "des":"To avoid garbled characters caused by inconsistent character encoding, you are advised to unify the encoding format of your data source when executing jobs in DLI.DLI onl", + "doc_type":"usermanual", + "kw":"How Do I Do If Inconsistent Character Encoding Leads to Garbled Characters?,DLI Databases and Tables", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If Inconsistent Character Encoding Leads to Garbled Characters?", + "githuburl":"" + }, + { + "uri":"dli_03_0175.html", + "node_id":"dli_03_0175.xml", + "product_code":"dli", + "code":"167", + "des":"User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table a", + "doc_type":"usermanual", + "kw":"Do I Need to to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Do I Need to to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?", + "githuburl":"" + }, + { + "uri":"dli_03_0177.html", + "node_id":"dli_03_0177.xml", + "product_code":"dli", + "code":"168", + "des":"A CSV file is imported to a DLI partitioned table, but the imported file data does not contain the data in the partitioning column. The partitioning column needs to be sp", + "doc_type":"usermanual", + "kw":"How Do I Do If Files Imported Into a DLI Partitioned Table Lack Data for the Partition Columns, Caus", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If Files Imported Into a DLI Partitioned Table Lack Data for the Partition Columns, Causing Query Failures After the Import Is Completed?", + "githuburl":"" + }, + { + "uri":"dli_03_0181.html", + "node_id":"dli_03_0181.xml", + "product_code":"dli", + "code":"169", + "des":"When an OBS foreign table is created, a field in the specified OBS file contains a carriage return line feed (CRLF) character. As a result, the data is incorrect.The stat", + "doc_type":"usermanual", + "kw":"How Do I Fix Incorrect Data in an OBS Foreign Table Caused by Newline Characters in OBS File Fields?", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Fix Incorrect Data in an OBS Foreign Table Caused by Newline Characters in OBS File Fields?", + "githuburl":"" + }, + { + "uri":"dli_03_0187.html", + "node_id":"dli_03_0187.xml", + "product_code":"dli", + "code":"170", + "des":"The on clause was not added to the SQL statement for joining tables. As a result, the Cartesian product query occurs due to multi-table association, and the queue resourc", + "doc_type":"usermanual", + "kw":"How Do I Prevent a Cartesian Product Query and Resource Overload Due to Missing \"ON\" Conditions in T", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Prevent a Cartesian Product Query and Resource Overload Due to Missing \"ON\" Conditions in Table Joins?", + "githuburl":"" + }, + { + "uri":"dli_03_0190.html", + "node_id":"dli_03_0190.xml", + "product_code":"dli", + "code":"171", + "des":"Partition data is manually uploaded to a partition of an OBS table. However, the data cannot be queried using DLI SQL editor.After manually adding partition data, you nee", + "doc_type":"usermanual", + "kw":"How Do I Do If I Can't Query Data After Manually Adding It to the Partition Directory of an OBS Tabl", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If I Can't Query Data After Manually Adding It to the Partition Directory of an OBS Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0212.html", + "node_id":"dli_03_0212.xml", + "product_code":"dli", + "code":"172", + "des":"When using the INSERT OVERWRITE statement to overwrite data in a partitioned table, if you find that it overwrites all data instead of the expected partitioned data, it m", + "doc_type":"usermanual", + "kw":"Why Does the \"insert overwrite\" Operation Affect All Data in a Partitioned Table Instead of Just the", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does the \"insert overwrite\" Operation Affect All Data in a Partitioned Table Instead of Just the Targeted Partition?", + "githuburl":"" + }, + { + "uri":"dli_03_0214.html", + "node_id":"dli_03_0214.xml", + "product_code":"dli", + "code":"173", + "des":"Spark does not have the datetime type and uses the TIMESTAMP type instead.You can use a function to convert data types.The following is an example.select cast(create_date", + "doc_type":"usermanual", + "kw":"Why Does the \"create_date\" Field in an RDS Table (Datetime Data Type) Appear as a Timestamp in DLI Q", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does the \"create_date\" Field in an RDS Table (Datetime Data Type) Appear as a Timestamp in DLI Queries?", + "githuburl":"" + }, + { + "uri":"dli_03_0215.html", + "node_id":"dli_03_0215.xml", + "product_code":"dli", + "code":"174", + "des":"Changing a table name immediately after executing a SQL job may result in an incorrect data size for the table.This is because DLI updates the metadata of the table when ", + "doc_type":"usermanual", + "kw":"How Do I Do If Renaming a Table After a SQL Job Causes Incorrect Data Size?,DLI Databases and Tables", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If Renaming a Table After a SQL Job Causes Incorrect Data Size?", + "githuburl":"" + }, + { + "uri":"dli_03_0231.html", + "node_id":"dli_03_0231.xml", + "product_code":"dli", + "code":"175", + "des":"When DLI is used to insert data into an OBS temporary table, only part of data is imported.Possible causes are as follows:The amount of data read during job execution is ", + "doc_type":"usermanual", + "kw":"How Can I Resolve Data Inconsistencies When Importing Data from DLI to OBS?,DLI Databases and Tables", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Can I Resolve Data Inconsistencies When Importing Data from DLI to OBS?", + "githuburl":"" + }, + { + "uri":"dli_03_0022.html", + "node_id":"dli_03_0022.xml", + "product_code":"dli", + "code":"176", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Enhanced Datasource Connections", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Enhanced Datasource Connections", + "githuburl":"" + }, + { + "uri":"dli_03_0238.html", + "node_id":"dli_03_0238.xml", + "product_code":"dli", + "code":"177", + "des":"The outbound rule had been configured for the security group of the queue associated with the enhanced datasource connection. The datasource authentication used a passwor", + "doc_type":"usermanual", + "kw":"How Do I Resolve a Failure in Connecting DLI to GaussDB(DWS) Through an Enhanced Datasource Connecti", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Resolve a Failure in Connecting DLI to GaussDB(DWS) Through an Enhanced Datasource Connection?", + "githuburl":"" + }, + { + "uri":"dli_03_0179.html", + "node_id":"dli_03_0179.xml", + "product_code":"dli", + "code":"178", + "des":"A datasource connection is created and bound to a queue. The connectivity test fails and the following error information is displayed:failed to connect to specified addre", + "doc_type":"usermanual", + "kw":"How Do I Do If the Datasource Connection Is Successfully Created but the Network Connectivity Test F", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If the Datasource Connection Is Successfully Created but the Network Connectivity Test Fails?", + "githuburl":"" + }, + { + "uri":"dli_03_0186.html", + "node_id":"dli_03_0186.xml", + "product_code":"dli", + "code":"179", + "des":"Configuring the Connection Between a DLI Queue and a Data Source in a Private NetworkIf your DLI job needs to connect to a data source, for example, MRS, RDS, CSS, Kafka,", + "doc_type":"usermanual", + "kw":"How Do I Configure Network Connectivity Between a DLI Queue and a Data Source?,Enhanced Datasource C", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Configure Network Connectivity Between a DLI Queue and a Data Source?", + "githuburl":"" + }, + { + "uri":"dli_03_0128.html", + "node_id":"dli_03_0128.xml", + "product_code":"dli", + "code":"180", + "des":"The main reason for creating a VPC peering connection for DLI's enhanced datasource connection is to establish network connectivity between DLI and data sources in differ", + "doc_type":"usermanual", + "kw":"Why Is Creating a VPC Peering Connection Necessary for Enhanced Datasource Connections in DLI?,Enhan", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Creating a VPC Peering Connection Necessary for Enhanced Datasource Connections in DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0257.html", + "node_id":"dli_03_0257.xml", + "product_code":"dli", + "code":"181", + "des":"The possible causes and solutions are as follows:If you have created a queue, do not bind it to a datasource connection immediately. Wait for 5 to 10 minutes. After the c", + "doc_type":"usermanual", + "kw":"How Do I Do If Creating a Datasource Connection in DLI Gets Stuck in the \"Creating\" State When Bindi", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If Creating a Datasource Connection in DLI Gets Stuck in the \"Creating\" State When Binding It to a Queue?", + "githuburl":"" + }, + { + "uri":"dli_03_0047.html", + "node_id":"dli_03_0047.xml", + "product_code":"dli", + "code":"182", + "des":"The network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.Example: When you create an RDS d", + "doc_type":"usermanual", + "kw":"How Do I Resolve the \"communication link failure\" Error When Using a Newly Created Datasource Connec", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Resolve the \"communication link failure\" Error When Using a Newly Created Datasource Connection That Appears to Be Activated?", + "githuburl":"" + }, + { + "uri":"dli_03_0080.html", + "node_id":"dli_03_0080.xml", + "product_code":"dli", + "code":"183", + "des":"If you have not added the cluster host information to the datasource connection, it can lead to KRB authentication failure, resulting in a connection timeout. In this cas", + "doc_type":"usermanual", + "kw":"How Do I Troubleshoot a Connection Timeout Issue That Isn't Recorded in Logs When Accessing MRS HBas", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Troubleshoot a Connection Timeout Issue That Isn't Recorded in Logs When Accessing MRS HBase Through a Datasource Connection?", + "githuburl":"" + }, + { + "uri":"dli_03_0239.html", + "node_id":"dli_03_0239.xml", + "product_code":"dli", + "code":"184", + "des":"A datasource RDS table was created in the DataArts Studio, and the insert overwrite statement was executed to write data into RDS. DLI.0999: BatchUpdateException: Incorre", + "doc_type":"usermanual", + "kw":"How Do I Do If I Encounter the \"Incorrect string value\" Error When Executing insert overwrite on a D", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If I Encounter the \"Incorrect string value\" Error When Executing insert overwrite on a Datasource RDS Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0250.html", + "node_id":"dli_03_0250.xml", + "product_code":"dli", + "code":"185", + "des":"The system failed to create a datasource RDS table, and null pointer error was reported.The following table creation statement was used:The RDS database is in a PostGre c", + "doc_type":"usermanual", + "kw":"How Do I Resolve the Null Pointer Error When Creating an RDS Datasource Table?,Enhanced Datasource C", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Resolve the Null Pointer Error When Creating an RDS Datasource Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0251.html", + "node_id":"dli_03_0251.xml", + "product_code":"dli", + "code":"186", + "des":"The system failed to execute insert overwrite on the datasource GaussDB(DWS) table, and org.postgresql.util.PSQLException: ERROR: tuple concurrently updated was displayed", + "doc_type":"usermanual", + "kw":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed Wh", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed When the System Executes insert overwrite on a Datasource GaussDB(DWS) Table", + "githuburl":"" + }, + { + "uri":"dli_03_0252.html", + "node_id":"dli_03_0252.xml", + "product_code":"dli", + "code":"187", + "des":"A datasource table was used to import data to a CloudTable HBase table. This HBase table contains a column family and a rowkey for 100 million simulating data records. Th", + "doc_type":"usermanual", + "kw":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Datas", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Datasource Table", + "githuburl":"" + }, + { + "uri":"dli_03_0253.html", + "node_id":"dli_03_0253.xml", + "product_code":"dli", + "code":"188", + "des":"A table was created on GaussDB(DWS) and then a datasource connection was created on DLI to read and write data. An error message was displayed during data writing, indica", + "doc_type":"usermanual", + "kw":"How Do I Do If A Null Value Is Written Into a Non-Null Field When Using a DLI Datasource Connection ", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If A Null Value Is Written Into a Non-Null Field When Using a DLI Datasource Connection to Connect to a GaussDB(DWS) Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0254.html", + "node_id":"dli_03_0254.xml", + "product_code":"dli", + "code":"189", + "des":"A datasource GaussDB(DWS) table and the datasource connection were created in DLI, and the schema of the source table in GaussDB(DWS) were updated. During the job executi", + "doc_type":"usermanual", + "kw":"How Do I Do If an Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updat", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If an Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated?", + "githuburl":"" + }, + { + "uri":"dli_03_0028.html", + "node_id":"dli_03_0028.xml", + "product_code":"dli", + "code":"190", + "des":"When creating an associated RDS table in DLI, if the RDS table contains an auto-increment primary key or other auto-populated fields, you can take the following measures ", + "doc_type":"usermanual", + "kw":"How Do I Insert Data into an RDS Table with an Auto-Increment Primary Key Using DLI?,Enhanced Dataso", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Insert Data into an RDS Table with an Auto-Increment Primary Key Using DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0020.html", + "node_id":"dli_03_0020.xml", + "product_code":"dli", + "code":"191", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"SQL Jobs", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"SQL Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0204.html", + "node_id":"dli_03_0204.xml", + "product_code":"dli", + "code":"192", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"SQL Job Development", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"SQL Job Development", + "githuburl":"" + }, + { + "uri":"dli_03_0200.html", + "node_id":"dli_03_0200.xml", + "product_code":"dli", + "code":"193", + "des":"A temporary table is used to store intermediate results. When a transaction or session ends, the data in the temporary table can be automatically deleted. For example, in", + "doc_type":"usermanual", + "kw":"SQL Jobs,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"SQL Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0086.html", + "node_id":"dli_03_0086.xml", + "product_code":"dli", + "code":"194", + "des":"If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.You a", + "doc_type":"usermanual", + "kw":"How Do I Merge Small Files?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Merge Small Files?", + "githuburl":"" + }, + { + "uri":"dli_03_0092.html", + "node_id":"dli_03_0092.xml", + "product_code":"dli", + "code":"195", + "des":"When creating an OBS table, you must specify a table path in the database. The path format is as follows: obs://xxx/database name/table name.If the specified path is akdc", + "doc_type":"usermanual", + "kw":"How Do I Specify an OBS Path When Creating an OBS Table?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Specify an OBS Path When Creating an OBS Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0108.html", + "node_id":"dli_03_0108.xml", + "product_code":"dli", + "code":"196", + "des":"To associate JSON data nested in an OBS bucket, you can create a table in asynchronous mode.The following is an example of a table creation statement that shows how to us", + "doc_type":"usermanual", + "kw":"How Do I Create a Table Using JSON Data in an OBS Bucket?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Create a Table Using JSON Data in an OBS Bucket?", + "githuburl":"" + }, + { + "uri":"dli_03_0069.html", + "node_id":"dli_03_0069.xml", + "product_code":"dli", + "code":"197", + "des":"The correct method for using the count function to perform aggregation is as follows:OrIf an incorrect method is used, an error will be reported.", + "doc_type":"usermanual", + "kw":"How Can I Use the count Function to Perform Aggregation?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Can I Use the count Function to Perform Aggregation?", + "githuburl":"" + }, + { + "uri":"dli_03_0072.html", + "node_id":"dli_03_0072.xml", + "product_code":"dli", + "code":"198", + "des":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket.Use the OBS cro", + "doc_type":"usermanual", + "kw":"How Do I Synchronize DLI Table Data Across Regions?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Synchronize DLI Table Data Across Regions?", + "githuburl":"" + }, + { + "uri":"dli_03_0191.html", + "node_id":"dli_03_0191.xml", + "product_code":"dli", + "code":"199", + "des":"If you need to insert data into a table but only want to specify certain fields, you can use the INSERT INTO statement combined with the SELECT clause.However, DLI curren", + "doc_type":"usermanual", + "kw":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?,SQL Job Development,User", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0196.html", + "node_id":"dli_03_0196.xml", + "product_code":"dli", + "code":"200", + "des":"If the job runs slowly, perform the following steps to find the causes and rectify the fault:Check whether the problem is caused by FullGC.Log in to the DLI console. In t", + "doc_type":"usermanual", + "kw":"How Do I Troubleshoot Slow SQL Jobs?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Troubleshoot Slow SQL Jobs?", + "githuburl":"" + }, + { + "uri":"dli_03_0091.html", + "node_id":"dli_03_0091.xml", + "product_code":"dli", + "code":"201", + "des":"You can view SQL job logs for routine O&M.Obtain the ID of the DLI job executed on the DataArts Studio console.Job IDOn the DLI console, choose Job Management > SQL Jobs.", + "doc_type":"usermanual", + "kw":"How Do I View DLI SQL Logs?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I View DLI SQL Logs?", + "githuburl":"" + }, + { + "uri":"dli_03_0116.html", + "node_id":"dli_03_0116.xml", + "product_code":"dli", + "code":"202", + "des":"You can view the job execution records when a job is running.Log in to the DLI management console.In the navigation pane on the left, choose Job Management > SQL Jobs.Ent", + "doc_type":"usermanual", + "kw":"How Do I View SQL Execution Records in DLI?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I View SQL Execution Records in DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0093.html", + "node_id":"dli_03_0093.xml", + "product_code":"dli", + "code":"203", + "des":"Data skew is a common issue during the execution of SQL jobs. When data is unevenly distributed, some compute nodes process significantly more data than others, which can", + "doc_type":"usermanual", + "kw":"How Do I Do When Data Skew Occurs During the Execution of a SQL Job?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do When Data Skew Occurs During the Execution of a SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0182.html", + "node_id":"dli_03_0182.xml", + "product_code":"dli", + "code":"204", + "des":"A SQL job contains join operations. After the job is submitted, it is stuck in the Running state and no result is returned.When a Spark SQL job has join operations on sma", + "doc_type":"usermanual", + "kw":"Why Does a SQL Job That Has Join Operations Stay in the Running State?,SQL Job Development,User Guid", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does a SQL Job That Has Join Operations Stay in the Running State?", + "githuburl":"" + }, + { + "uri":"dli_03_0213.html", + "node_id":"dli_03_0213.xml", + "product_code":"dli", + "code":"205", + "des":"The possible causes and solutions are as follows:After you purchase a DLI queue and submit a SQL job for the first time, wait for 5 to 10 minutes. After the cluster is st", + "doc_type":"usermanual", + "kw":"Why Is a SQL Job Stuck in the Submitting State?,SQL Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is a SQL Job Stuck in the Submitting State?", + "githuburl":"" + }, + { + "uri":"dli_03_0206.html", + "node_id":"dli_03_0206.xml", + "product_code":"dli", + "code":"206", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"SQL Job O&M", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"SQL Job O&M", + "githuburl":"" + }, + { + "uri":"dli_03_0014.html", + "node_id":"dli_03_0014.xml", + "product_code":"dli", + "code":"207", + "des":"This message indicates that you are exporting data to an existing OBS path.Solution:Create an OBS directory.You can create an OBS directory that does not exist to store t", + "doc_type":"usermanual", + "kw":"Why Is Error \"path obs://xxx already exists\" Reported When Data Is Exported to OBS?,SQL Job O&M,User", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"path obs://xxx already exists\" Reported When Data Is Exported to OBS?", + "githuburl":"" + }, + { + "uri":"dli_03_0066.html", + "node_id":"dli_03_0066.xml", + "product_code":"dli", + "code":"208", + "des":"This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.For example, ta", + "doc_type":"usermanual", + "kw":"Why Is Error \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Displayed W", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Displayed When Two Tables Are Joined?", + "githuburl":"" + }, + { + "uri":"dli_03_0071.html", + "node_id":"dli_03_0071.xml", + "product_code":"dli", + "code":"209", + "des":"This message indicates that your operation is restricted because your account is in arrears or there is insufficient balance in your account.Solution:Check the account st", + "doc_type":"usermanual", + "kw":"Why Is Error \"The current account does not have permission to perform this operation,the current acc", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget.\" Reported when a SQL Statement Is Executed?", + "githuburl":"" + }, + { + "uri":"dli_03_0145.html", + "node_id":"dli_03_0145.xml", + "product_code":"dli", + "code":"210", + "des":"When you query the partitioned table XX.YYY, the partition column is not specified in the search criteria.A partitioned table can be queried only when the query condition", + "doc_type":"usermanual", + "kw":"Why Is Error \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" ", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" Reported When a Query Statement Is Executed?", + "githuburl":"" + }, + { + "uri":"dli_03_0169.html", + "node_id":"dli_03_0169.xml", + "product_code":"dli", + "code":"211", + "des":"The following error message is displayed when the LOAD DATA command is executed by a Spark SQL job to import data to a DLI table:In some cases ,the following error messag", + "doc_type":"usermanual", + "kw":"Why Is Error \"IllegalArgumentException: Buffer size too small. size\" Reported When Data Is Loaded to", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"IllegalArgumentException: Buffer size too small. size\" Reported When Data Is Loaded to an OBS Foreign Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0189.html", + "node_id":"dli_03_0189.xml", + "product_code":"dli", + "code":"212", + "des":"An error is reported during SQL job execution:Please contact DLI service. DLI.0002: FileNotFoundException: getFileStatus on obs://xxx: status [404]Check whether there is ", + "doc_type":"usermanual", + "kw":"Why Is Error \"DLI.0002 FileNotFoundException\" Reported During SQL Job Running?,SQL Job O&M,User Guid", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"DLI.0002 FileNotFoundException\" Reported During SQL Job Running?", + "githuburl":"" + }, + { + "uri":"dli_03_0046.html", + "node_id":"dli_03_0046.xml", + "product_code":"dli", + "code":"213", + "des":"Currently, DLI supports the creation of TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET tables using the Hive syntax.If you create a table using CTAS statements, s", + "doc_type":"usermanual", + "kw":"Why Is a Schema Parsing Error Reported When I Create a Hive Table Using CTAS?,SQL Job O&M,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is a Schema Parsing Error Reported When I Create a Hive Table Using CTAS?", + "githuburl":"" + }, + { + "uri":"dli_03_0173.html", + "node_id":"dli_03_0173.xml", + "product_code":"dli", + "code":"214", + "des":"When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:DLI.0999: RuntimeException: o", + "doc_type":"usermanual", + "kw":"Why Is Error \"org.apache.hadoop.fs.obs.OBSIOException\" Reported When I Run DLI SQL Scripts on DataAr", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"org.apache.hadoop.fs.obs.OBSIOException\" Reported When I Run DLI SQL Scripts on DataArts Studio?", + "githuburl":"" + }, + { + "uri":"dli_03_0207.html", + "node_id":"dli_03_0207.xml", + "product_code":"dli", + "code":"215", + "des":"Error message \"File not Found\" is displayed when a SQL job is accessed.The system may not be able to locate the specified file path or file due to an incorrect file path ", + "doc_type":"usermanual", + "kw":"Why Is Error \"File not Found\" Reported When I Access a SQL Job?,SQL Job O&M,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"File not Found\" Reported When I Access a SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0208.html", + "node_id":"dli_03_0208.xml", + "product_code":"dli", + "code":"216", + "des":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.Check the permissions of the OBS bucket to ensure that the account has access ", + "doc_type":"usermanual", + "kw":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?,SQL Job O&M,Us", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0209.html", + "node_id":"dli_03_0209.xml", + "product_code":"dli", + "code":"217", + "des":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Ch", + "doc_type":"usermanual", + "kw":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{b", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" Reported When I Access a SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0210.html", + "node_id":"dli_03_0210.xml", + "product_code":"dli", + "code":"218", + "des":"Error message \"The current account does not have permission to perform this operation,the current account was restricted.\" is reported during SQL statement execution.Chec", + "doc_type":"usermanual", + "kw":"Why Am I Seeing the Error Message \"The current account does not have permission to perform this oper", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Am I Seeing the Error Message \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget\" When Executing a SQL Statement?", + "githuburl":"" + }, + { + "uri":"dli_03_0037.html", + "node_id":"dli_03_0037.xml", + "product_code":"dli", + "code":"219", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Flink Jobs", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Flink Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0137.html", + "node_id":"dli_03_0137.xml", + "product_code":"dli", + "code":"220", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Flink Job Consulting", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Flink Job Consulting", + "githuburl":"" + }, + { + "uri":"dli_03_0139.html", + "node_id":"dli_03_0139.xml", + "product_code":"dli", + "code":"221", + "des":"A sub-user can view queues but cannot view Flink jobs. You can authorize the sub-user using DLI or IAM.Authorization on DLILog in to the DLI console using a tenant accoun", + "doc_type":"usermanual", + "kw":"How Do I Authorize a Subuser to View Flink Jobs?,Flink Job Consulting,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Authorize a Subuser to View Flink Jobs?", + "githuburl":"" + }, + { + "uri":"dli_03_0090.html", + "node_id":"dli_03_0090.xml", + "product_code":"dli", + "code":"222", + "des":"DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are r", + "doc_type":"usermanual", + "kw":"How Do I Configure Auto Restart upon Exception for a Flink Job?,Flink Job Consulting,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Configure Auto Restart upon Exception for a Flink Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0099.html", + "node_id":"dli_03_0099.xml", + "product_code":"dli", + "code":"223", + "des":"When you create a Flink SQL job or Flink Jar job, you can select Save Job Log on the job editing page to save job running logs to OBS.To set the OBS bucket for storing th", + "doc_type":"usermanual", + "kw":"How Do I Save Logs for Flink Jobs?,Flink Job Consulting,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Save Logs for Flink Jobs?", + "githuburl":"" + }, + { + "uri":"dli_03_0160.html", + "node_id":"dli_03_0160.xml", + "product_code":"dli", + "code":"224", + "des":"Choose Job Management > Flink Jobs. In the Operation column of the target job, choose More > Permissions. When a new user is authorized, No such user. userName:xxxx. is d", + "doc_type":"usermanual", + "kw":"Why Is Error \"No such user. userName:xxxx.\" Reported on the Flink Job Management Page When I Grant P", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"No such user. userName:xxxx.\" Reported on the Flink Job Management Page When I Grant Permission to a User?", + "githuburl":"" + }, + { + "uri":"dli_03_0180.html", + "node_id":"dli_03_0180.xml", + "product_code":"dli", + "code":"225", + "des":"Checkpoint was enabled when a Flink job is created, and the OBS bucket for storing checkpoints was specified. I am not sure how to restore a Flink job from a specific che", + "doc_type":"usermanual", + "kw":"How Do I Restore a Flink Job from a Specific Checkpoint After Manually Stopping the Job?,Flink Job C", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Restore a Flink Job from a Specific Checkpoint After Manually Stopping the Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0036.html", + "node_id":"dli_03_0036.xml", + "product_code":"dli", + "code":"226", + "des":"When you set running parameters of a DLI Flink job, you can enable Alarm Generation upon Job Exception to receive alarms when the job runs abnormally or is in arrears.If ", + "doc_type":"usermanual", + "kw":"Why Is a Message Displayed Indicating That the SMN Topic Does Not Exist When I Use the SMN Topic in ", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is a Message Displayed Indicating That the SMN Topic Does Not Exist When I Use the SMN Topic in DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0131.html", + "node_id":"dli_03_0131.xml", + "product_code":"dli", + "code":"227", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Flink SQL Jobs", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Flink SQL Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0089.html", + "node_id":"dli_03_0089.xml", + "product_code":"dli", + "code":"228", + "des":"When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.In the following example, the day field is used as the partition fie", + "doc_type":"usermanual", + "kw":"How Do I Map an OBS Table to a DLI Partitioned Table?,Flink SQL Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Map an OBS Table to a DLI Partitioned Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0120.html", + "node_id":"dli_03_0120.xml", + "product_code":"dli", + "code":"229", + "des":"You used Flink 1.10 to run a Flink Opensource SQL job. You set the number of Kafka partitions for the job a small value at the beginning and need to increase the number n", + "doc_type":"usermanual", + "kw":"How Do I Change the Number of Kafka Partitions in a Flink SQL Job Without Stopping It?,Flink SQL Job", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Change the Number of Kafka Partitions in a Flink SQL Job Without Stopping It?", + "githuburl":"" + }, + { + "uri":"dli_03_0167.html", + "node_id":"dli_03_0167.xml", + "product_code":"dli", + "code":"230", + "des":"When I run the creation statement with an EL expression in the table name in a Flink SQL job, the following error message is displayed:DLI.0005: AnalysisException: t_user", + "doc_type":"usermanual", + "kw":"How Do I Fix the DLI.0005 Error When Using EL Expressions to Create a Table in a Flink SQL Job?,Flin", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Fix the DLI.0005 Error When Using EL Expressions to Create a Table in a Flink SQL Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0168.html", + "node_id":"dli_03_0168.xml", + "product_code":"dli", + "code":"231", + "des":"After data is written to OBS through the Flink job output stream, data cannot be queried from the DLI table created in the OBS file path.For example, use the following Fl", + "doc_type":"usermanual", + "kw":"Why Is No Data Queried in the DLI Table Created Using the OBS File Path When Data Is Written to OBS ", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is No Data Queried in the DLI Table Created Using the OBS File Path When Data Is Written to OBS by a Flink Job Output Stream?", + "githuburl":"" + }, + { + "uri":"dli_03_0174.html", + "node_id":"dli_03_0174.xml", + "product_code":"dli", + "code":"232", + "des":"After a Flink SQL job is submitted on DLI, the job fails to be executed. The following error information is displayed in the job log:connect to DIS failed java.lang.Illeg", + "doc_type":"usermanual", + "kw":"Why Does a Flink SQL Job Fails to Be Executed, and Is \"connect to DIS failed java.lang.IllegalArgume", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does a Flink SQL Job Fails to Be Executed, and Is \"connect to DIS failed java.lang.IllegalArgumentException: Access key cannot be null\" Displayed in the Log?", + "githuburl":"" + }, + { + "uri":"dli_03_0232.html", + "node_id":"dli_03_0232.xml", + "product_code":"dli", + "code":"233", + "des":"After a Flink SQL job consumed Kafka and sent data to the Elasticsearch cluster, the job was successfully executed, but no data is available.Possible causes are as follow", + "doc_type":"usermanual", + "kw":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster,F", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster", + "githuburl":"" + }, + { + "uri":"dli_03_0121.html", + "node_id":"dli_03_0121.xml", + "product_code":"dli", + "code":"234", + "des":"You used Flink 1.10 to run a Flink Opensource SQL job. The job failed after the following error was reported when Flink Sink wrote data to Kafka.The CPU usage is too high", + "doc_type":"usermanual", + "kw":"How Do I Configure Connection Retries for Kafka Sink If it is Disconnected?,Flink SQL Jobs,User Guid", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Configure Connection Retries for Kafka Sink If it is Disconnected?", + "githuburl":"" + }, + { + "uri":"dli_03_0048.html", + "node_id":"dli_03_0048.xml", + "product_code":"dli", + "code":"235", + "des":"In a Flink job, you can use CREATE statements to define source and sink tables, and specify their connector types and related attributes.If you need to write data to diff", + "doc_type":"usermanual", + "kw":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?,Flink SQL Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0040.html", + "node_id":"dli_03_0040.xml", + "product_code":"dli", + "code":"236", + "des":"To rectify this fault, perform the following steps:Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statemen", + "doc_type":"usermanual", + "kw":"Why Does DIS Stream Not Exist During Job Semantic Check?,Flink SQL Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does DIS Stream Not Exist During Job Semantic Check?", + "githuburl":"" + }, + { + "uri":"dli_03_0265.html", + "node_id":"dli_03_0265.xml", + "product_code":"dli", + "code":"237", + "des":"If the Flink JobManager prompts \"Timeout expired while fetching topic metadata\", it means that the Flink job timed out while trying to fetch metadata for the Kafka topic.", + "doc_type":"usermanual", + "kw":"Why Is Error \"Timeout expired while fetching topic metadata\" Repeatedly Reported in Flink JobManager", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"Timeout expired while fetching topic metadata\" Repeatedly Reported in Flink JobManager Logs?", + "githuburl":"" + }, + { + "uri":"dli_03_0132.html", + "node_id":"dli_03_0132.xml", + "product_code":"dli", + "code":"238", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Flink Jar Jobs", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Flink Jar Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0044.html", + "node_id":"dli_03_0044.xml", + "product_code":"dli", + "code":"239", + "des":"You can upload configuration files for custom jobs (Jar).Upload the configuration file to DLI through Package Management.In the Other Dependencies area of the Flink Jar j", + "doc_type":"usermanual", + "kw":"Can I Upload Configuration Files for Flink Jar Jobs?,Flink Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Can I Upload Configuration Files for Flink Jar Jobs?", + "githuburl":"" + }, + { + "uri":"dli_03_0119.html", + "node_id":"dli_03_0119.xml", + "product_code":"dli", + "code":"240", + "des":"The dependency of your Flink job conflicts with a built-in dependency of the DLI Flink platform. As a result, the job submission fails.Check whether there are conflicting", + "doc_type":"usermanual", + "kw":"Why Does a Flink Jar Package Conflict Result in Job Submission Failure?,Flink Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does a Flink Jar Package Conflict Result in Job Submission Failure?", + "githuburl":"" + }, + { + "uri":"dli_03_0161.html", + "node_id":"dli_03_0161.xml", + "product_code":"dli", + "code":"241", + "des":"When a Flink Jar job is submitted to access GaussDB(DWS), an error message is displayed indicating that the job fails to be started. The job log contains the following er", + "doc_type":"usermanual", + "kw":"Why Does a Flink Jar Job Fail to Access GaussDB(DWS) and a Message Is Displayed Indicating Too Many ", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does a Flink Jar Job Fail to Access GaussDB(DWS) and a Message Is Displayed Indicating Too Many Client Connections?", + "githuburl":"" + }, + { + "uri":"dli_03_0233.html", + "node_id":"dli_03_0233.xml", + "product_code":"dli", + "code":"242", + "des":"The storage path of the Flink Jar job checkpoints was set to an OBS bucket. The job failed to be submitted, and an error message indicating an invalid OBS bucket name was", + "doc_type":"usermanual", + "kw":"Why Is Error Invalid OBS Bucket Name Reported After a Flink Job Submission Failed?,Flink Jar Jobs,Us", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error Invalid OBS Bucket Name Reported After a Flink Job Submission Failed?", + "githuburl":"" + }, + { + "uri":"dli_03_0234.html", + "node_id":"dli_03_0234.xml", + "product_code":"dli", + "code":"243", + "des":"Flink Job submission failed. The exception information is as follows:Flink JAR files conflicted. The submitted Flink JAR file conflicted with the HDFS JAR file of the DLI", + "doc_type":"usermanual", + "kw":"Why Does the Flink Submission Fail Due to Hadoop JAR File Conflict?,Flink Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does the Flink Submission Fail Due to Hadoop JAR File Conflict?", + "githuburl":"" + }, + { + "uri":"dli_03_0103.html", + "node_id":"dli_03_0103.xml", + "product_code":"dli", + "code":"244", + "des":"On the Flink job management page, hover the cursor on the status of the job that fails to be submitted to view the brief information about the failure.The possible causes", + "doc_type":"usermanual", + "kw":"How Do I Locate a Flink Job Submission Error?,Flink Jar Jobs,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Locate a Flink Job Submission Error?", + "githuburl":"" + }, + { + "uri":"dli_03_0133.html", + "node_id":"dli_03_0133.xml", + "product_code":"dli", + "code":"245", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Flink Job Performance Tuning", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Flink Job Performance Tuning", + "githuburl":"" + }, + { + "uri":"dli_03_0106.html", + "node_id":"dli_03_0106.xml", + "product_code":"dli", + "code":"246", + "des":"Data Stacking in a Consumer GroupThe accumulated data of a consumer group can be calculated by the following formula: Total amount of data to be consumed by the consumer ", + "doc_type":"usermanual", + "kw":"Flink Job Performance Tuning,Flink Job Performance Tuning,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Flink Job Performance Tuning", + "githuburl":"" + }, + { + "uri":"dli_03_0096.html", + "node_id":"dli_03_0096.xml", + "product_code":"dli", + "code":"247", + "des":"The DLI Flink checkpoint/savepoint mechanism is complete and reliable. You can use this mechanism to prevent data loss when a job is manually restarted or restarted due t", + "doc_type":"usermanual", + "kw":"How Do I Prevent Data Loss After Flink Job Restart?,Flink Job Performance Tuning,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Prevent Data Loss After Flink Job Restart?", + "githuburl":"" + }, + { + "uri":"dli_03_0105.html", + "node_id":"dli_03_0105.xml", + "product_code":"dli", + "code":"248", + "des":"On the Flink job management, click Edit in the Operation column of the target job. On the displayed page, check whether Save Job Log in the Running Parameters tab is enab", + "doc_type":"usermanual", + "kw":"How Do I Locate a Flink Job Running Error?,Flink Job Performance Tuning,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Locate a Flink Job Running Error?", + "githuburl":"" + }, + { + "uri":"dli_03_0136.html", + "node_id":"dli_03_0136.xml", + "product_code":"dli", + "code":"249", + "des":"Flink's checkpointing is a fault tolerance and recovery mechanism. This mechanism ensures that real-time programs can self-recover in case of exceptions or machine issues", + "doc_type":"usermanual", + "kw":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?,Flink Job Perf", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?", + "githuburl":"" + }, + { + "uri":"dli_03_0064.html", + "node_id":"dli_03_0064.xml", + "product_code":"dli", + "code":"250", + "des":"Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:If the submission fails, a submission log is genera", + "doc_type":"usermanual", + "kw":"Why Are Logs Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?,", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Are Logs Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?", + "githuburl":"" + }, + { + "uri":"dli_03_0236.html", + "node_id":"dli_03_0236.xml", + "product_code":"dli", + "code":"251", + "des":"JobManager and TaskManager heartbeats timed out. As a result, the Flink job is abnormal.Check whether the network is intermittently disconnected and whether the cluster l", + "doc_type":"usermanual", + "kw":"Why Is the Flink Job Abnormal Due to Heartbeat Timeout Between JobManager and TaskManager?,Flink Job", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is the Flink Job Abnormal Due to Heartbeat Timeout Between JobManager and TaskManager?", + "githuburl":"" + }, + { + "uri":"dli_03_0021.html", + "node_id":"dli_03_0021.xml", + "product_code":"dli", + "code":"252", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Spark Jobs", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Spark Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0217.html", + "node_id":"dli_03_0217.xml", + "product_code":"dli", + "code":"253", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Spark Job Development", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Spark Job Development", + "githuburl":"" + }, + { + "uri":"dli_03_0201.html", + "node_id":"dli_03_0201.xml", + "product_code":"dli", + "code":"254", + "des":"DLI Spark does not support job scheduling. You can use other services, such as DataArts Studio, or use APIs or SDKs to customize job schedule.The Spark SQL syntax does no", + "doc_type":"usermanual", + "kw":"Spark Jobs,Spark Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Spark Jobs", + "githuburl":"" + }, + { + "uri":"dli_03_0107.html", + "node_id":"dli_03_0107.xml", + "product_code":"dli", + "code":"255", + "des":"To use Spark to write data into a DLI table, configure the following parameters:fs.obs.access.keyfs.obs.secret.keyfs.obs.implfs.obs.endpointThe following is an example:", + "doc_type":"usermanual", + "kw":"How Do I Use Spark to Write Data into a DLI Table?,Spark Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Use Spark to Write Data into a DLI Table?", + "githuburl":"" + }, + { + "uri":"dli_03_0017.html", + "node_id":"dli_03_0017.xml", + "product_code":"dli", + "code":"256", + "des":"To obtain the AK/SK, set the parameters as follows:Create a SparkContext using code.val sc: SparkContext = new SparkContext()\nsc.hadoopConfiguration.set(\"fs.obs.access.ke", + "doc_type":"usermanual", + "kw":"How Do I Set Up AK/SK So That a General Queue Can Access Tables Stored in OBS?,Spark Job Development", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Set Up AK/SK So That a General Queue Can Access Tables Stored in OBS?", + "githuburl":"" + }, + { + "uri":"dli_03_0102.html", + "node_id":"dli_03_0102.xml", + "product_code":"dli", + "code":"257", + "des":"Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs. In the job list, locate the target job and click next to Job ID to view the parame", + "doc_type":"usermanual", + "kw":"How Do I View the Resource Usage of DLI Spark Jobs?,Spark Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I View the Resource Usage of DLI Spark Jobs?", + "githuburl":"" + }, + { + "uri":"dli_03_0076.html", + "node_id":"dli_03_0076.xml", + "product_code":"dli", + "code":"258", + "des":"If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management p", + "doc_type":"usermanual", + "kw":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the S", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the Spark Job Results Stored in MySQL?", + "githuburl":"" + }, + { + "uri":"dli_03_0082.html", + "node_id":"dli_03_0082.xml", + "product_code":"dli", + "code":"259", + "des":"DLI natively supports PySpark.For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are pac", + "doc_type":"usermanual", + "kw":"How Do I Run a Complex PySpark Program in DLI?,Spark Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Run a Complex PySpark Program in DLI?", + "githuburl":"" + }, + { + "uri":"dli_03_0068.html", + "node_id":"dli_03_0068.xml", + "product_code":"dli", + "code":"260", + "des":"When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.To solve this problem, you can config", + "doc_type":"usermanual", + "kw":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", + "githuburl":"" + }, + { + "uri":"dli_03_0118.html", + "node_id":"dli_03_0118.xml", + "product_code":"dli", + "code":"261", + "des":"You can use SparkFiles to read the file submitted using –-file form a local path: SparkFiles.get(\"Name of the uploaded file\").The file path in the Driver is different fro", + "doc_type":"usermanual", + "kw":"How Do I Read Uploaded Files for a Spark Jar Job?,Spark Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Read Uploaded Files for a Spark Jar Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0077.html", + "node_id":"dli_03_0077.xml", + "product_code":"dli", + "code":"262", + "des":"I cannot find the specified Python environment after adding the Python 3 package.Set spark.yarn.appMasterEnv.PYSPARK_PYTHON to python3 in the conf file to specify the Pyt", + "doc_type":"usermanual", + "kw":"Why Can't I Find the Specified Python Environment After Adding the Python Package?,Spark Job Develop", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Can't I Find the Specified Python Environment After Adding the Python Package?", + "githuburl":"" + }, + { + "uri":"dli_03_0220.html", + "node_id":"dli_03_0220.xml", + "product_code":"dli", + "code":"263", + "des":"The remaining CUs in the queue may be insufficient. As a result, the job cannot be submitted.To view the remaining CUs of a queue, perform the following steps:Check the C", + "doc_type":"usermanual", + "kw":"Why Is a Spark Jar Job Stuck in the Submitting State?,Spark Job Development,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is a Spark Jar Job Stuck in the Submitting State?", + "githuburl":"" + }, + { + "uri":"dli_03_0218.html", + "node_id":"dli_03_0218.xml", + "product_code":"dli", + "code":"264", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"Spark Job O&M", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Spark Job O&M", + "githuburl":"" + }, + { + "uri":"dli_03_0156.html", + "node_id":"dli_03_0156.xml", + "product_code":"dli", + "code":"265", + "des":"The following error is reported when a Spark job accesses OBS data:Set the AK/SK to enable Spark jobs to access OBS data.For details, see How Do I Set Up AK/SK So That a ", + "doc_type":"usermanual", + "kw":"Why Do I Get \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Errors When a Spark Job Accesses OB", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Do I Get \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Errors When a Spark Job Accesses OBS Data?", + "githuburl":"" + }, + { + "uri":"dli_03_0164.html", + "node_id":"dli_03_0164.xml", + "product_code":"dli", + "code":"266", + "des":"This error message may be due to the OBS bucket being set as the DLI log bucket, which cannot be used for other purposes.You can follow these steps to check:Check if the ", + "doc_type":"usermanual", + "kw":"Why Do I Encounter the Error \"verifyBucketExists on XXXX: status [403]\" When Using a Spark Job to Ac", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Do I Encounter the Error \"verifyBucketExists on XXXX: status [403]\" When Using a Spark Job to Access an OBS Bucket That I Have Permission to Access?", + "githuburl":"" + }, + { + "uri":"dli_03_0157.html", + "node_id":"dli_03_0157.xml", + "product_code":"dli", + "code":"267", + "des":"When running large amounts of data in a Spark job, if a timeout exception error occurs, it is usually due to insufficient resource configuration, data skew, network issue", + "doc_type":"usermanual", + "kw":"Why Does a Job Running Timeout Occur When Processing a Large Amount of Data with a Spark Job?,Spark ", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does a Job Running Timeout Occur When Processing a Large Amount of Data with a Spark Job?", + "githuburl":"" + }, + { + "uri":"dli_03_0188.html", + "node_id":"dli_03_0188.xml", + "product_code":"dli", + "code":"268", + "des":"Spark jobs cannot access SFTP. Upload the files you want to access to OBS and then you can analyze the data using Spark jobs.Upload data to an OBS bucket: Upload data sto", + "doc_type":"usermanual", + "kw":"Why Does a Spark Job Fail to Execute with an Abnormal Access Directory Error When Accessing Files in", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does a Spark Job Fail to Execute with an Abnormal Access Directory Error When Accessing Files in SFTP?", + "githuburl":"" + }, + { + "uri":"dli_03_0192.html", + "node_id":"dli_03_0192.xml", + "product_code":"dli", + "code":"269", + "des":"When a Spark job is running, an error message is displayed, indicating that the user does not have the database permission. The error information is as follows:org.apache", + "doc_type":"usermanual", + "kw":"Why Does the Job Fail to Be Executed Due to Insufficient Database and Table Permissions?,Spark Job O", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does the Job Fail to Be Executed Due to Insufficient Database and Table Permissions?", + "githuburl":"" + }, + { + "uri":"dli_03_0272.html", + "node_id":"dli_03_0272.xml", + "product_code":"dli", + "code":"270", + "des":"I cannot find the global_temp database in the Spark 3.x job log.The global_temp database is the default built-in database of Spark 3.x and is Spark's global temporary vie", + "doc_type":"usermanual", + "kw":"Why Is the global_temp Database Missing in the Job Log of Spark 3.x?,Spark Job O&M,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is the global_temp Database Missing in the Job Log of Spark 3.x?", + "githuburl":"" + }, + { + "uri":"dli_03_0275.html", + "node_id":"dli_03_0275.xml", + "product_code":"dli", + "code":"271", + "des":"I failed to use the DataSource syntax to create an OBS table in Avro format when selecting Spark to access metadata.Spark 2.3.x does not support creating OBS tables in Av", + "doc_type":"usermanual", + "kw":"Why Does Using DataSource Syntax to Create an OBS Table of Avro Type Fail When Accessing Metadata Wi", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Does Using DataSource Syntax to Create an OBS Table of Avro Type Fail When Accessing Metadata With Spark 2.3.x?", + "githuburl":"" + }, + { + "uri":"dli_03_0053.html", + "node_id":"dli_03_0053.xml", + "product_code":"dli", + "code":"272", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"DLI Resource Quotas", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"DLI Resource Quotas", + "githuburl":"" + }, + { + "uri":"dli_03_0031.html", + "node_id":"dli_03_0031.xml", + "product_code":"dli", + "code":"273", + "des":"Log in to the management console.Click in the upper left corner and select a region and a project.Click the My Quota icon in the upper right corner of the page.The Serv", + "doc_type":"usermanual", + "kw":"How Do I View My Quotas?,DLI Resource Quotas,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I View My Quotas?", + "githuburl":"" + }, + { + "uri":"dli_03_0032.html", + "node_id":"dli_03_0032.xml", + "product_code":"dli", + "code":"274", + "des":"The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your applica", + "doc_type":"usermanual", + "kw":"How Do I Apply for a Higher Quota?,DLI Resource Quotas,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Apply for a Higher Quota?", + "githuburl":"" + }, + { + "uri":"dli_03_0054.html", + "node_id":"dli_03_0054.xml", + "product_code":"dli", + "code":"275", + "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "doc_type":"usermanual", + "kw":"DLI Permissions Management", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"DLI Permissions Management", + "githuburl":"" + }, + { + "uri":"dli_03_0008.html", + "node_id":"dli_03_0008.xml", + "product_code":"dli", + "code":"276", + "des":"You are unable to perform permission operations on the partition columns of partitioned tables.However, when you grant the permission of any non-partition column in a par", + "doc_type":"usermanual", + "kw":"What Is Column-Level Authorization for DLI Partitioned Tables?,DLI Permissions Management,User Guide", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"What Is Column-Level Authorization for DLI Partitioned Tables?", + "githuburl":"" + }, + { + "uri":"dli_03_0195.html", + "node_id":"dli_03_0195.xml", + "product_code":"dli", + "code":"277", + "des":"When the user update an existing program package, the following error information is displayed:\"error_code\"*DLI.0003\",\"error_msg\":\"Permission denied for resource 'resourc", + "doc_type":"usermanual", + "kw":"How Do I Do If I Encounter Insufficient Permissions While Updating Packages?,DLI Permissions Managem", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If I Encounter Insufficient Permissions While Updating Packages?", + "githuburl":"" + }, + { + "uri":"dli_03_0227.html", + "node_id":"dli_03_0227.xml", + "product_code":"dli", + "code":"278", + "des":"When the SQL query statement is executed, the system displays a message indicating that the user does not have the permission to query resources.Error information: DLI.00", + "doc_type":"usermanual", + "kw":"Why Is Error \"DLI.0003: Permission denied for resource...\" Reported When I Run a SQL Statement?,DLI ", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Why Is Error \"DLI.0003: Permission denied for resource...\" Reported When I Run a SQL Statement?", + "githuburl":"" + }, + { + "uri":"dli_03_0228.html", + "node_id":"dli_03_0228.xml", + "product_code":"dli", + "code":"279", + "des":"If you have already granted authorization to a table and a test query was successful, but you encounter an error when trying to query it again after some time, you should", + "doc_type":"usermanual", + "kw":"How Do I Do If I Can't Query Table Data After Being Granted Table Permissions?,DLI Permissions Manag", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"How Do I Do If I Can't Query Table Data After Being Granted Table Permissions?", + "githuburl":"" + }, + { + "uri":"dli_03_0057.html", + "node_id":"dli_03_0057.xml", + "product_code":"dli", + "code":"280", + "des":"If a table inherits database permissions, you do not need to regrant the inherited permissions to the table.Re-authorizing may cause confusion in table permission managem", + "doc_type":"usermanual", + "kw":"Will Granting Duplicate Permissions to a Table After Inheriting Database Permissions Cause an Error?", + "search_title":"", + "metedata":[ + { + "prodname":"dli", + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" + } + ], + "title":"Will Granting Duplicate Permissions to a Table After Inheriting Database Permissions Cause an Error?", "githuburl":"" }, { "uri":"dli_03_0067.html", "node_id":"dli_03_0067.xml", "product_code":"dli", - "code":"260", + "code":"281", "des":"User A created Table1.User B created View1 based on Table1.After the Select Table permission on Table1 is granted to user C, user C fails to query View1.User B does not h", "doc_type":"usermanual", - "kw":"Why Can't I Query a View After I'm Granted the Select Table Permission on the View?,O&M Guide,User G", + "kw":"Why Can't I Query a View After I'm Granted the Select Table Permission on the View?,DLI Permissions ", "search_title":"", "metedata":[ { @@ -4903,748 +5770,70 @@ "githuburl":"" }, { - "uri":"dli_03_0049.html", - "node_id":"dli_03_0049.xml", - "product_code":"dli", - "code":"261", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Queue", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Queue", - "githuburl":"" - }, - { - "uri":"dli_03_0229.html", - "node_id":"dli_03_0229.xml", - "product_code":"dli", - "code":"262", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Usage", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Usage", - "githuburl":"" - }, - { - "uri":"dli_03_0109.html", - "node_id":"dli_03_0109.xml", - "product_code":"dli", - "code":"263", - "des":"Currently, you are not allowed to modify the description of a created queue. You can add the description when purchasing the queue.", - "doc_type":"usermanual", - "kw":"Does the Description of a DLI Queue Can Be Modified?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Does the Description of a DLI Queue Can Be Modified?", - "githuburl":"" - }, - { - "uri":"dli_03_0166.html", - "node_id":"dli_03_0166.xml", - "product_code":"dli", - "code":"264", - "des":"Deleting a queue does not cause table data loss in your database.", - "doc_type":"usermanual", - "kw":"Will Table Data in My Database Be Lost If I Delete a Queue?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Will Table Data in My Database Be Lost If I Delete a Queue?", - "githuburl":"" - }, - { - "uri":"dli_03_0170.html", - "node_id":"dli_03_0170.xml", - "product_code":"dli", - "code":"265", - "des":"You need to develop a mechanism to retry failed jobs. When a faulty queue is recovered, your application tries to submit the failed jobs to the queue again.", - "doc_type":"usermanual", - "kw":"How Does DLI Ensure the Reliability of Spark Jobs When a Queue Is Abnormal?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Does DLI Ensure the Reliability of Spark Jobs When a Queue Is Abnormal?", - "githuburl":"" - }, - { - "uri":"dli_03_0098.html", - "node_id":"dli_03_0098.xml", - "product_code":"dli", - "code":"266", - "des":"DLI allows you to subscribe to an SMN topic for failed jobs.Log in to the DLI console.In the navigation pane on the left, choose Queue Management.On the Queue Management ", - "doc_type":"usermanual", - "kw":"How Do I Monitor Queue Exceptions?,Usage,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Monitor Queue Exceptions?", - "githuburl":"" - }, - { - "uri":"dli_03_0230.html", - "node_id":"dli_03_0230.xml", - "product_code":"dli", - "code":"267", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"O&M Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"O&M Guide", - "githuburl":"" - }, - { - "uri":"dli_03_0095.html", - "node_id":"dli_03_0095.xml", - "product_code":"dli", - "code":"268", - "des":"To check the running status of the DLI queue and determine whether to run more jobs on that queue, you need to check the queue load.Search for Cloud Eye on the console.In", - "doc_type":"usermanual", - "kw":"How Do I View DLI Queue Load?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I View DLI Queue Load?", - "githuburl":"" - }, - { - "uri":"dli_03_0183.html", - "node_id":"dli_03_0183.xml", - "product_code":"dli", - "code":"269", - "des":"You need to check the large number of jobs in the Submitting and Running states on the queue.Use Cloud Eye to view jobs in different states on the queue. The procedure is", - "doc_type":"usermanual", - "kw":"How Do I Determine Whether There Are Too Many Jobs in the Current Queue?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Determine Whether There Are Too Many Jobs in the Current Queue?", - "githuburl":"" - }, - { - "uri":"dli_03_0065.html", - "node_id":"dli_03_0065.xml", - "product_code":"dli", - "code":"270", - "des":"Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earl", - "doc_type":"usermanual", - "kw":"How Do I Switch an Earlier-Version Spark Queue to a General-Purpose Queue?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Switch an Earlier-Version Spark Queue to a General-Purpose Queue?", - "githuburl":"" - }, - { - "uri":"dli_03_0193.html", - "node_id":"dli_03_0193.xml", - "product_code":"dli", - "code":"271", - "des":"DLI queues do not use resources or bandwidth when no job is running. In this case, the running status of DLI queues is not displayed on CES.", - "doc_type":"usermanual", - "kw":"Why Cannot I View the Resource Running Status of DLI Queues on Cloud Eye?,O&M Guide,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Cannot I View the Resource Running Status of DLI Queues on Cloud Eye?", - "githuburl":"" - }, - { - "uri":"dli_03_0088.html", - "node_id":"dli_03_0088.xml", - "product_code":"dli", - "code":"272", - "des":"In DLI, 64 CU = 64 cores and 256 GB memory.In a Spark job, if the driver occupies 4 cores and 16 GB memory, the executor can occupy 60 cores and 240 GB memory.", - "doc_type":"usermanual", - "kw":"How Do I Allocate Queue Resources for Running Spark Jobs If I Have Purchased 64 CUs?,O&M Guide,User ", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Allocate Queue Resources for Running Spark Jobs If I Have Purchased 64 CUs?", - "githuburl":"" - }, - { - "uri":"dli_03_0159.html", - "node_id":"dli_03_0159.xml", - "product_code":"dli", - "code":"273", - "des":"Queue plans create failed. The plan xxx target cu is out of quota is displayed when you create a scheduled scaling task.The CU quota of the current account is insufficien", - "doc_type":"usermanual", - "kw":"Why Is Error \"Queue plans create failed. The plan xxx target cu is out of quota\" Reported When I Sch", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is Error \"Queue plans create failed. The plan xxx target cu is out of quota\" Reported When I Schedule CU Changes?", - "githuburl":"" - }, - { - "uri":"dli_03_0171.html", - "node_id":"dli_03_0171.xml", - "product_code":"dli", - "code":"274", - "des":"After a SQL job was submitted to the default queue, the job runs abnormally. The job log reported that the execution timed out. The exception logs are as follows:[ERROR] ", - "doc_type":"usermanual", - "kw":"Why Is a Timeout Exception Reported When a DLI SQL Statement Fails to Be Executed on the Default Que", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is a Timeout Exception Reported When a DLI SQL Statement Fails to Be Executed on the Default Queue?", - "githuburl":"" - }, - { - "uri":"dli_03_0276.html", - "node_id":"dli_03_0276.xml", - "product_code":"dli", - "code":"275", - "des":"In daily big data analysis work, it is important to allocate and manage compute resources properly to provide a good job execution environment.You can allocate resources ", - "doc_type":"usermanual", - "kw":"How Can I Check the Actual and Used CUs for an Elastic Resource Pool as Well as the Required CUs for", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Can I Check the Actual and Used CUs for an Elastic Resource Pool as Well as the Required CUs for a Job?", - "githuburl":"" - }, - { - "uri":"dli_03_0022.html", - "node_id":"dli_03_0022.xml", - "product_code":"dli", - "code":"276", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Datasource Connections", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Datasource Connections", - "githuburl":"" - }, - { - "uri":"dli_03_0110.html", - "node_id":"dli_03_0110.xml", - "product_code":"dli", - "code":"277", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Datasource Connections", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Datasource Connections", - "githuburl":"" - }, - { - "uri":"dli_03_0128.html", - "node_id":"dli_03_0128.xml", - "product_code":"dli", - "code":"278", - "des":"You need to create a VPC peering connection to enable network connectivity. Take MRS as an example. If DLI and MRS clusters are in the same VPC, and the security group is", - "doc_type":"usermanual", - "kw":"Why Do I Need to Create a VPC Peering Connection for an Enhanced Datasource Connection?,Datasource C", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Do I Need to Create a VPC Peering Connection for an Enhanced Datasource Connection?", - "githuburl":"" - }, - { - "uri":"dli_03_0237.html", - "node_id":"dli_03_0237.xml", - "product_code":"dli", - "code":"279", - "des":"An enhanced datasource connection failed to pass the network connectivity test. Datasource connection cannot be bound to a queue. The following error information is displ", - "doc_type":"usermanual", - "kw":"Failed to Bind a Queue to an Enhanced Datasource Connection,Datasource Connections,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Failed to Bind a Queue to an Enhanced Datasource Connection", - "githuburl":"" - }, - { - "uri":"dli_03_0238.html", - "node_id":"dli_03_0238.xml", - "product_code":"dli", - "code":"280", - "des":"The outbound rule had been configured for the security group of the queue associated with the enhanced datasource connection. The datasource authentication used a passwor", - "doc_type":"usermanual", - "kw":"DLI Failed to Connect to GaussDB(DWS) Through an Enhanced Datasource Connection,Datasource Connectio", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"DLI Failed to Connect to GaussDB(DWS) Through an Enhanced Datasource Connection", - "githuburl":"" - }, - { - "uri":"dli_03_0179.html", - "node_id":"dli_03_0179.xml", - "product_code":"dli", - "code":"281", - "des":"A datasource connection is created and bound to a queue. The connectivity test fails and the following error information is displayed:failed to connect to specified addre", - "doc_type":"usermanual", - "kw":"How Do I Do if the Datasource Connection Is Created But the Network Connectivity Test Fails?,Datasou", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Do if the Datasource Connection Is Created But the Network Connectivity Test Fails?", - "githuburl":"" - }, - { - "uri":"dli_03_0186.html", - "node_id":"dli_03_0186.xml", + "uri":"dli_03_0225.html", + "node_id":"dli_03_0225.xml", "product_code":"dli", "code":"282", - "des":"Configuring the Connection Between a DLI Queue and a Data Source in a Private NetworkIf your DLI job needs to connect to a data source, for example, MRS, RDS, CSS, Kafka,", + "des":"Despite configuring a job bucket and authorizing DLI to access it, I still receive an error message stating that DLI is not authorized to access the bucket when attemptin", "doc_type":"usermanual", - "kw":"How Do I Configure the Network Between a DLI Queue and a Data Source?,Datasource Connections,User Gu", + "kw":"How Do I Do If I Receive a Message Saying I Don't Have Sufficient Permissions to Submit My Jobs to t", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" } ], - "title":"How Do I Configure the Network Between a DLI Queue and a Data Source?", - "githuburl":"" - }, - { - "uri":"dli_03_0257.html", - "node_id":"dli_03_0257.xml", - "product_code":"dli", - "code":"283", - "des":"The possible causes and solutions are as follows:If you have created a queue, do not bind it to a datasource connection immediately. Wait for 5 to 10 minutes. After the c", - "doc_type":"usermanual", - "kw":"What Can I Do If a Datasource Connection Is Stuck in Creating State When I Try to Bind a Queue to It", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"What Can I Do If a Datasource Connection Is Stuck in Creating State When I Try to Bind a Queue to It?", - "githuburl":"" - }, - { - "uri":"dli_03_0259.html", - "node_id":"dli_03_0259.xml", - "product_code":"dli", - "code":"284", - "des":"DLI enhanced datasource connection uses VPC peering to directly connect the VPC networks of the desired data sources for point-to-point data exchanges.", - "doc_type":"usermanual", - "kw":"How Do I Connect DLI to Data Sources?,Datasource Connections,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Connect DLI to Data Sources?", - "githuburl":"" - }, - { - "uri":"dli_03_0112.html", - "node_id":"dli_03_0112.xml", - "product_code":"dli", - "code":"285", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Cross-Source Analysis", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Cross-Source Analysis", - "githuburl":"" - }, - { - "uri":"dli_03_0011.html", - "node_id":"dli_03_0011.xml", - "product_code":"dli", - "code":"286", - "des":"To perform query on data stored on services rather than DLI, perform the following steps:Assume that the data to be queried is stored on multiple services (for example, O", - "doc_type":"usermanual", - "kw":"How Can I Perform Query on Data Stored on Services Rather Than DLI?,Cross-Source Analysis,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Can I Perform Query on Data Stored on Services Rather Than DLI?", - "githuburl":"" - }, - { - "uri":"dli_03_0085.html", - "node_id":"dli_03_0085.xml", - "product_code":"dli", - "code":"287", - "des":"Connect VPCs in different regions.Create an enhanced datasource connection on DLI and bind it to a queue.Add a DLI route.", - "doc_type":"usermanual", - "kw":"How Can I Access Data Across Regions?,Cross-Source Analysis,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Can I Access Data Across Regions?", - "githuburl":"" - }, - { - "uri":"dli_03_0028.html", - "node_id":"dli_03_0028.xml", - "product_code":"dli", - "code":"288", - "des":"When data is inserted into DLI, set the ID field to NULL.", - "doc_type":"usermanual", - "kw":"How Do I Set the Auto-increment Primary Key or Other Fields That Are Automatically Filled in the RDS", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Set the Auto-increment Primary Key or Other Fields That Are Automatically Filled in the RDS Table When Creating a DLI and Associating It with the RDS Table?", - "githuburl":"" - }, - { - "uri":"dli_03_0256.html", - "node_id":"dli_03_0256.xml", - "product_code":"dli", - "code":"289", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"Datasource Connection O&M", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Datasource Connection O&M", - "githuburl":"" - }, - { - "uri":"dli_03_0047.html", - "node_id":"dli_03_0047.xml", - "product_code":"dli", - "code":"290", - "des":"Possible CausesThe network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.The network connec", - "doc_type":"usermanual", - "kw":"Why Is the Error Message \"communication link failure\" Displayed When I Use a Newly Activated Datasou", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Is the Error Message \"communication link failure\" Displayed When I Use a Newly Activated Datasource Connection?", - "githuburl":"" - }, - { - "uri":"dli_03_0080.html", - "node_id":"dli_03_0080.xml", - "product_code":"dli", - "code":"291", - "des":"The cluster host information is not added to the datasource connection. As a result, the KRB authentication fails, the connection times out, and no error is recorded in l", - "doc_type":"usermanual", - "kw":"Connection Times Out During MRS HBase Datasource Connection, and No Error Is Recorded in Logs,Dataso", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Connection Times Out During MRS HBase Datasource Connection, and No Error Is Recorded in Logs", - "githuburl":"" - }, - { - "uri":"dli_03_0111.html", - "node_id":"dli_03_0111.xml", - "product_code":"dli", - "code":"292", - "des":"When you create a VPC peering connection for the datasource connection, the following error information is displayed:Before you create a datasource connection, check whet", - "doc_type":"usermanual", - "kw":"Why Can't I Find the Subnet When Creating a DLI Datasource Connection?,Datasource Connection O&M,Use", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Why Can't I Find the Subnet When Creating a DLI Datasource Connection?", - "githuburl":"" - }, - { - "uri":"dli_03_0239.html", - "node_id":"dli_03_0239.xml", - "product_code":"dli", - "code":"293", - "des":"A datasource RDS table was created in the DataArts Studio, and the insert overwrite statement was executed to write data into RDS. DLI.0999: BatchUpdateException: Incorre", - "doc_type":"usermanual", - "kw":"Error Message \"Incorrect string value\" Is Displayed When insert overwrite Is Executed on a Datasourc", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Error Message \"Incorrect string value\" Is Displayed When insert overwrite Is Executed on a Datasource RDS Table", - "githuburl":"" - }, - { - "uri":"dli_03_0250.html", - "node_id":"dli_03_0250.xml", - "product_code":"dli", - "code":"294", - "des":"The system failed to create a datasource RDS table, and null pointer error was reported.The following table creation statement was used:The RDS database is in a PostGre c", - "doc_type":"usermanual", - "kw":"Null Pointer Error Is Displayed When the System Creates a Datasource RDS Table,Datasource Connection", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Null Pointer Error Is Displayed When the System Creates a Datasource RDS Table", - "githuburl":"" - }, - { - "uri":"dli_03_0251.html", - "node_id":"dli_03_0251.xml", - "product_code":"dli", - "code":"295", - "des":"The system failed to execute insert overwrite on the datasource GaussDB(DWS) table, and org.postgresql.util.PSQLException: ERROR: tuple concurrently updated was displayed", - "doc_type":"usermanual", - "kw":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed Wh", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed When the System Executes insert overwrite on a Datasource GaussDB(DWS) Table", - "githuburl":"" - }, - { - "uri":"dli_03_0252.html", - "node_id":"dli_03_0252.xml", - "product_code":"dli", - "code":"296", - "des":"A datasource table was used to import data to a CloudTable HBase table. This HBase table contains a column family and a rowkey for 100 million simulating data records. Th", - "doc_type":"usermanual", - "kw":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Datas", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Datasource Table", - "githuburl":"" - }, - { - "uri":"dli_03_0253.html", - "node_id":"dli_03_0253.xml", - "product_code":"dli", - "code":"297", - "des":"A table was created on GaussDB(DWS) and then a datasource connection was created on DLI to read and write data. An error message was displayed during data writing, indica", - "doc_type":"usermanual", - "kw":"A Null Value Is Written Into a Non-Null Field When a DLI Datasource Connection Is Used to Connect to", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"A Null Value Is Written Into a Non-Null Field When a DLI Datasource Connection Is Used to Connect to a GaussDB(DWS) Table", - "githuburl":"" - }, - { - "uri":"dli_03_0254.html", - "node_id":"dli_03_0254.xml", - "product_code":"dli", - "code":"298", - "des":"A datasource GaussDB(DWS) table and the datasource connection were created in DLI, and the schema of the source table in GaussDB(DWS) were updated. During the job executi", - "doc_type":"usermanual", - "kw":"An Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated,Datasource C", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"An Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated", + "title":"How Do I Do If I Receive a Message Saying I Don't Have Sufficient Permissions to Submit My Jobs to the Job Bucket?", "githuburl":"" }, { "uri":"dli_03_0056.html", "node_id":"dli_03_0056.xml", "product_code":"dli", - "code":"299", + "code":"283", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", - "kw":"APIs", + "kw":"DLI APIs", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" } ], - "title":"APIs", + "title":"DLI APIs", "githuburl":"" }, { "uri":"dli_03_0060.html", "node_id":"dli_03_0060.xml", "product_code":"dli", - "code":"300", + "code":"284", "des":"In the REST API provided by DLI, the request header can be added to the request URI, for example, Content-Type.Content-Type indicates the request body type or format. The", "doc_type":"usermanual", - "kw":"Why Is Error \"unsupported media Type\" Reported When I Subimt a SQL Job?,APIs,User Guide", + "kw":"Why Is Error \"unsupported media Type\" Reported When I Subimt a SQL Job?,DLI APIs,User Guide", "search_title":"", "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" } ], "title":"Why Is Error \"unsupported media Type\" Reported When I Subimt a SQL Job?", "githuburl":"" }, - { - "uri":"dli_03_0125.html", - "node_id":"dli_03_0125.xml", - "product_code":"dli", - "code":"301", - "des":"When different IAM users call an API under the same enterprise project in the same region, the project ID is the same.", - "doc_type":"usermanual", - "kw":"Is the Project ID Fixed when Different IAM Users Call an API?,APIs,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"Is the Project ID Fixed when Different IAM Users Call an API?", - "githuburl":"" - }, { "uri":"dli_03_0178.html", "node_id":"dli_03_0178.xml", "product_code":"dli", - "code":"302", + "code":"285", "des":"When the API call for submitting a SQL job times out, and the following error information is displayed:There are currently no resources tracked in the state, so there is ", "doc_type":"usermanual", "kw":"What Can I Do If an Error Is Reported When the Execution of the API for Creating a SQL Job Times Out", @@ -5652,71 +5841,19 @@ "metedata":[ { "prodname":"dli", - "documenttype":"usermanual" + "IsMulti":"No", + "documenttype":"usermanual", + "IsBot":"Yes" } ], "title":"What Can I Do If an Error Is Reported When the Execution of the API for Creating a SQL Job Times Out?", "githuburl":"" }, - { - "uri":"dli_03_0058.html", - "node_id":"dli_03_0058.xml", - "product_code":"dli", - "code":"303", - "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "doc_type":"usermanual", - "kw":"SDKs", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"SDKs", - "githuburl":"" - }, - { - "uri":"dli_03_0073.html", - "node_id":"dli_03_0073.xml", - "product_code":"dli", - "code":"304", - "des":"When you query the SQL job results using SDK, the system checks the job status when the job is submitted. The timeout interval set in the system is 300s. If the job is no", - "doc_type":"usermanual", - "kw":"How Do I Set the Timeout Duration for Querying SQL Job Results Using SDK?,SDKs,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Set the Timeout Duration for Querying SQL Job Results Using SDK?", - "githuburl":"" - }, - { - "uri":"dli_03_0255.html", - "node_id":"dli_03_0255.xml", - "product_code":"dli", - "code":"305", - "des":"Run the ping command to check whether dli.xxx can be accessed.If dli.xxx can be accessed, check whether DNS resolution is correctly configured.If dli.xxx can be accessed,", - "doc_type":"usermanual", - "kw":"How Do I Handle the dli.xxx,unable to resolve host address Error?,SDKs,User Guide", - "search_title":"", - "metedata":[ - { - "prodname":"dli", - "documenttype":"usermanual" - } - ], - "title":"How Do I Handle the dli.xxx,unable to resolve host address Error?", - "githuburl":"" - }, { "uri":"dli_01_00006.html", "node_id":"dli_01_00006.xml", "product_code":"dli", - "code":"306", + "code":"286", "des":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "doc_type":"usermanual", "kw":"Change History,User Guide", @@ -5724,9 +5861,10 @@ "metedata":[ { "prodname":"dli", - "IsMulti":"No", - "IsBot":"No", - "documenttype":"usermanual" + "opensource":"true", + "documenttype":"usermanual", + "IsMulti":"No;NO;Yes", + "IsBot":"Yes;No" } ], "title":"Change History", diff --git a/docs/dli/umn/CLASS.TXT.json b/docs/dli/umn/CLASS.TXT.json index 8f7006d15..48621c444 100644 --- a/docs/dli/umn/CLASS.TXT.json +++ b/docs/dli/umn/CLASS.TXT.json @@ -11,7 +11,7 @@ { "desc":"Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark and Apache Flink ecosystems. It frees you from managing an", "product_code":"dli", - "title":"What Is Data Lake Insight?", + "title":"What Is Data Lake Insight", "uri":"dli_01_0378.html", "doc_type":"usermanual", "p_code":"1", @@ -36,7 +36,7 @@ "code":"4" }, { - "desc":"Only the latest 100 jobs are displayed on DLI's SparkUI.A maximum of 1,000 job results can be displayed on the console. To view more or all jobs, export the job data to O", + "desc":"For more notes and constraints on elastic resource pools, see Notes and Constraints on Using an Elastic Resource Pool.For more notes and constraints on using a queue, see", "product_code":"dli", "title":"Notes and Constraints", "uri":"dli_07_0005.html", @@ -44,6 +44,15 @@ "p_code":"1", "code":"5" }, + { + "desc":"An elastic resource pool provides compute resources (CPU and memory) for running DLI jobs. The unit is CU. One CU contains one CPU and 4 GB memory.You can create multiple", + "product_code":"dli", + "title":"Specifications", + "uri":"dli_07_0027.html", + "doc_type":"usermanual", + "p_code":"1", + "code":"6" + }, { "desc":"If you need to assign different permissions to employees in your enterprise to access your DLI resources, IAM is a good choice for fine-grained permissions management. IA", "product_code":"dli", @@ -51,7 +60,7 @@ "uri":"dli_07_0006.html", "doc_type":"usermanual", "p_code":"1", - "code":"6" + "code":"7" }, { "desc":"A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.You can also request for an increased quota if your e", @@ -60,16 +69,16 @@ "uri":"dli_07_0009.html", "doc_type":"usermanual", "p_code":"1", - "code":"7" + "code":"8" }, { - "desc":"DLI allows multiple organizations, departments, or applications to share resources. A logical entity, also called a tenant, is provided to use diverse resources and servi", + "desc":"Dedicated computing resources. They are isolated by resource pools and can only be shared by queues in the same elastic resource pool. You can set scaling policies of dif", "product_code":"dli", "title":"Basic Concepts", "uri":"dli_07_0003.html", "doc_type":"usermanual", "p_code":"1", - "code":"8" + "code":"9" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -78,244 +87,235 @@ "uri":"dli_01_0220.html", "doc_type":"usermanual", "p_code":"", - "code":"9" - }, - { - "desc":"DLI can query data stored in OBS. This section describes how to us a Spark SQL job on DLI to query OBS data.You can use DLI to submit a Spark SQL job to query data. The g", - "product_code":"dli", - "title":"Creating and Submitting a Spark SQL Job", - "uri":"dli_01_0002.html", - "doc_type":"usermanual", - "p_code":"9", "code":"10" }, { - "desc":"DLI allows you to customize query templates or save frequently used SQL statements as templates to facilitate SQL operations. After templates are saved, you do not need t", + "desc":"DLI can query data stored in OBS. This section describes how to use DLI to submit a SQL job to query OBS data.To illustrate, create a new file called sampledata.csv and u", "product_code":"dli", - "title":"Developing and Submitting a Spark SQL Job Using the TPC-H Sample Template", - "uri":"dli_01_0512.html", + "title":"Using DLI to Submit a SQL Job to Query OBS Data", + "uri":"dli_01_0002.html", "doc_type":"usermanual", - "p_code":"9", + "p_code":"10", "code":"11" }, { - "desc":"DLI can query data stored in OBS. This section describes how to use a Spark Jar job on DLI to query OBS data in real time.You can use DLI to submit Spark jobs for real-ti", + "desc":"DLI can query data stored in RDS. This section describes how to use DLI to submit a SQL job to query RDS for MySQL data.In this example, we will create an RDS for MySQL D", "product_code":"dli", - "title":"Creating and Submitting a Spark Jar Job", - "uri":"dli_01_0375.html", + "title":"Using DLI to Submit a SQL Job to Query RDS for MySQL Data", + "uri":"dli_01_0020.html", "doc_type":"usermanual", - "p_code":"9", + "p_code":"10", "code":"12" }, { - "desc":"DLI Flink jobs can use other cloud services as data sources and sink streams for real-time compute. This example describes how to create and submit a Flink Opensource SQL", + "desc":"DLI Flink jobs can use other cloud services as data sources and sink streams for real-time compute.This example describes how to create and submit a Flink OpenSource SQL ", "product_code":"dli", - "title":"Creating and Submitting a Flink OpenSource SQL Job", + "title":"Using DLI to Submit a Flink OpenSource SQL Job to Query RDS for MySQL Data", "uri":"dli_01_0531.html", "doc_type":"usermanual", - "p_code":"9", + "p_code":"10", "code":"13" }, { - "desc":"The Overview page of the DLI console provides you with the DLI workflow and resource usage.The process of using DLI is as follows:Create a queue.Queues are DLI's compute ", + "desc":"Flink Jar jobs are suitable for data analysis scenarios that require custom stream processing logic, complex state management, or integration with specific libraries. You", "product_code":"dli", - "title":"DLI Console Overview", - "uri":"dli_01_0377.html", + "title":"Using DLI to Submit a Flink Jar Job", + "uri":"dli_01_0512.html", "doc_type":"usermanual", - "p_code":"", + "p_code":"10", "code":"14" }, { - "desc":"You can edit and run SQL statements in the SQL job editor to execute data query.The editor supports SQL:2003 and is compatible with Spark SQL. For details about the synta", + "desc":"DLI allows you to submit Spark jobs compiled as JAR files, which contain the necessary code and dependency information for executing the job. These files are used for spe", "product_code":"dli", - "title":"SQL Editor", - "uri":"dli_01_0320.html", + "title":"Using DLI to Submit a Spark Jar Job", + "uri":"dli_01_0375.html", "doc_type":"usermanual", - "p_code":"", + "p_code":"10", "code":"15" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "desc":"This chapter walks you through on how to develop a DLI job.To manage fine-grained permissions for your DLI resources using IAM, create an IAM user and grant them permissi", "product_code":"dli", - "title":"Job Management", + "title":"DLI Job Development Process", "uri":"dli_01_0001.html", "doc_type":"usermanual", "p_code":"", "code":"16" }, { - "desc":"DLI provides the following job types:SQL job: SQL jobs provide you with standard SQL statements and are compatible with Spark SQL and Presto SQL (based on Presto). You ca", + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Overview", - "uri":"dli_01_0567.html", + "title":"Preparations", + "uri":"dli_01_0611.html", "doc_type":"usermanual", - "p_code":"16", + "p_code":"", "code":"17" }, { - "desc":"SQL jobs allow you to execute SQL statements in the SQL job editing window, import data, and export data.SQL job management provides the following functions:Searching for", + "desc":"To use DLI, you must first configure permissions.This section applies to the following scenarios:If you use DLI for the first time, configure DLI agency permissions by re", "product_code":"dli", - "title":"SQL Job Management", - "uri":"dli_01_0017.html", + "title":"Configuring DLI Agency Permissions", + "uri":"dli_01_0618.html", "doc_type":"usermanual", - "p_code":"16", + "p_code":"17", "code":"18" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "desc":"You can use Identity and Access Management (IAM) to implement fine-grained permissions control on DLI resources. For details, see Overview.If your cloud account does not ", "product_code":"dli", - "title":"Flink Job Management", - "uri":"dli_01_0389.html", + "title":"Creating an IAM User and Granting Permissions", + "uri":"dli_01_0418.html", "doc_type":"usermanual", - "p_code":"16", + "p_code":"17", "code":"19" }, { - "desc":"On the Job Management page of Flink jobs, you can submit a Flink job. Currently, the following job types are supported:Flink SQL uses SQL statements to define jobs and ca", + "desc":"Before using DLI, you need to configure a DLI job bucket. The bucket is used to store temporary data generated during DLI job running, such as job logs and results.Config", "product_code":"dli", - "title":"Overview", - "uri":"dli_01_0403.html", + "title":"Configuring a DLI Job Bucket", + "uri":"dli_01_0536.html", "doc_type":"usermanual", - "p_code":"19", + "p_code":"17", "code":"20" }, - { - "desc":"You can isolate Flink jobs allocated to different users by setting permissions to ensure data query performance.The administrator and job creator have all permissions, wh", - "product_code":"dli", - "title":"Managing Flink Job Permissions", - "uri":"dli_01_0479.html", - "doc_type":"usermanual", - "p_code":"19", - "code":"21" - }, - { - "desc":"To create a Flink job, you need to enter the data source and data output channel, that is, source and sink. To use another service as the source or sink stream, you need ", - "product_code":"dli", - "title":"Preparing Flink Job Data", - "uri":"dli_01_0454.html", - "doc_type":"usermanual", - "p_code":"19", - "code":"22" - }, - { - "desc":"This section describes how to create a Flink OpenSource SQL job.DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink 1.10 and 1.12 provided by the ", - "product_code":"dli", - "title":"(Recommended) Creating a Flink OpenSource SQL Job", - "uri":"dli_01_0498.html", - "doc_type":"usermanual", - "p_code":"19", - "code":"23" - }, - { - "desc":"This section describes how to create a Flink SQL job. You can use Flink SQLs to develop jobs to meet your service requirements. Using SQL statements simplifies logic impl", - "product_code":"dli", - "title":"Creating a Flink SQL Job", - "uri":"dli_01_0455.html", - "doc_type":"usermanual", - "p_code":"19", - "code":"24" - }, - { - "desc":"This section describes how to create a Flink Jar job. You can perform secondary development based on Flink APIs, build your own JAR file, and submit the JAR file to DLI q", - "product_code":"dli", - "title":"Creating a Flink Jar Job", - "uri":"dli_01_0457.html", - "doc_type":"usermanual", - "p_code":"19", - "code":"25" - }, - { - "desc":"After a job is created, you can perform operations on the job as required.Editing a JobStarting a JobStopping a JobDeleting a JobExporting a JobImporting a JobModifying N", - "product_code":"dli", - "title":"Performing Operations on a Flink Job", - "uri":"dli_01_0461.html", - "doc_type":"usermanual", - "p_code":"19", - "code":"26" - }, - { - "desc":"After creating a job, you can view the job details to learn about the following information:Viewing Job DetailsChecking Job Monitoring InformationViewing the Task List of", - "product_code":"dli", - "title":"Flink Job Details", - "uri":"dli_01_0462.html", - "doc_type":"usermanual", - "p_code":"19", - "code":"27" - }, - { - "desc":"A tag is a key-value pair customized by users and used to identify cloud resources. It helps users to classify and search for cloud resources. A tag consists of a tag key", - "product_code":"dli", - "title":"Tag Management", - "uri":"dli_01_0463.html", - "doc_type":"usermanual", - "p_code":"19", - "code":"28" - }, - { - "desc":"In actual job operations, the compute resources required by a job vary depending on the data volume. As a result, compute resources are wasted when the volume is small an", - "product_code":"dli", - "title":"Enabling Dynamic Scaling for Flink Jobs", - "uri":"dli_01_0534.html", - "doc_type":"usermanual", - "p_code":"19", - "code":"29" - }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Spark Job Management", - "uri":"dli_01_0465.html", + "title":"Creating an Elastic Resource Pool and Queues Within It", + "uri":"dli_01_0508.html", "doc_type":"usermanual", - "p_code":"16", + "p_code":"", + "code":"21" + }, + { + "desc":"DLI compute resources are the foundation to run jobs. This section describes the modes of DLI compute resources and queue types.Before we dive into the compute resource m", + "product_code":"dli", + "title":"Overview of DLI Elastic Resource Pools and Queues", + "uri":"dli_01_0504.html", + "doc_type":"usermanual", + "p_code":"21", + "code":"22" + }, + { + "desc":"An elastic resource pool offers compute resources (CPU and memory) required for running DLI jobs, which can adapt to the changing demands of services.You can create multi", + "product_code":"dli", + "title":"Creating an Elastic Resource Pool and Creating Queues Within It", + "uri":"dli_01_0505.html", + "doc_type":"usermanual", + "p_code":"21", + "code":"23" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Managing Elastic Resource Pools", + "uri":"dli_01_0529.html", + "doc_type":"usermanual", + "p_code":"21", + "code":"24" + }, + { + "desc":"After creating an elastic resource pool, you can check and manage it on the management console.This section describes how to check basic information about an elastic reso", + "product_code":"dli", + "title":"Viewing Basic Information", + "uri":"dli_01_0622.html", + "doc_type":"usermanual", + "p_code":"24", + "code":"25" + }, + { + "desc":"Administrators can assign permissions of different operation scopes to users for each elastic resource pool.The administrator and elastic resource pool owner have all per", + "product_code":"dli", + "title":"Managing Permissions", + "uri":"dli_01_0526.html", + "doc_type":"usermanual", + "p_code":"24", + "code":"26" + }, + { + "desc":"If you want a queue to use resources in an elastic resource pool, bind the queue to the pool.You can click Associate Queue on the Resource Pool page to bind a queue to an", + "product_code":"dli", + "title":"Binding a Queue", + "uri":"dli_01_0530.html", + "doc_type":"usermanual", + "p_code":"24", + "code":"27" + }, + { + "desc":"CU settings are used to control the maximum and minimum CU ranges for elastic resource pools to avoid unlimited resource scaling.For example, an elastic resource pool has", + "product_code":"dli", + "title":"Setting CUs", + "uri":"dli_01_0507.html", + "doc_type":"usermanual", + "p_code":"24", + "code":"28" + }, + { + "desc":"If the current specifications of your elastic resource pool do not meet your service needs, you can modify them using the change yearly/monthly CUs function.In the naviga", + "product_code":"dli", + "title":"Modifying Specifications", + "uri":"dli_01_0524.html", + "doc_type":"usermanual", + "p_code":"24", + "code":"29" + }, + { + "desc":"A tag is a key-value pair that you can customize to identify cloud resources. It helps you to classify and search for cloud resources. A tag consists of a tag key and a t", + "product_code":"dli", + "title":"Managing Tags", + "uri":"dli_01_0525.html", + "doc_type":"usermanual", + "p_code":"24", "code":"30" }, { - "desc":"Based on the open-source Spark, DLI optimizes performance and reconstructs services to be compatible with the Apache Spark ecosystem and interfaces, and executes batch pr", + "desc":"Multiple queues can be added to an elastic resource pool. For how to add a queue, see Creating an Elastic Resource Pool and Creating Queues Within It. You can configure t", "product_code":"dli", - "title":"Spark Job Management", - "uri":"dli_01_0385.html", + "title":"Adjusting Scaling Policies for Queues in an Elastic Resource Pool", + "uri":"dli_01_0506.html", "doc_type":"usermanual", - "p_code":"30", + "p_code":"24", "code":"31" }, { - "desc":"DLI provides fully-managed Spark computing services by allowing you to execute Spark jobs.On the Overview page, click Create Job in the upper right corner of the Spark Jo", + "desc":"If you added a queue to or deleted one from an elastic resource pool, or you scaled an added queue, the CU quantity of the elastic resource pool may be changed. You can v", "product_code":"dli", - "title":"Creating a Spark Job", - "uri":"dli_01_0384.html", + "title":"Viewing Scaling History", + "uri":"dli_01_0532.html", "doc_type":"usermanual", - "p_code":"30", + "p_code":"24", "code":"32" }, { - "desc":"In actual job operations, jobs have different importance and urgency levels. So, the compute resources required for normal operations of important and urgent jobs need to", + "desc":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", "product_code":"dli", - "title":"Setting the Priority for a Job", - "uri":"dli_01_0535.html", + "title":"Allocating to an Enterprise Project", + "uri":"dli_01_0566.html", "doc_type":"usermanual", - "p_code":"16", + "p_code":"24", "code":"33" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Queue Management", - "uri":"dli_01_0012.html", + "title":"Managing Queues", + "uri":"dli_01_0528.html", "doc_type":"usermanual", - "p_code":"", + "p_code":"21", "code":"34" }, { - "desc":"Queues in DLI are computing resources, which are the basis for using DLI. All executed jobs require computing resources.Currently, DLI provides two types of queues: For S", + "desc":"This section walks you through how to view basic information about a queue on the management console, including the engine type and version.Log in to the DLI management c", "product_code":"dli", - "title":"Overview", - "uri":"dli_01_0402.html", + "title":"Viewing Basic Information About a Queue", + "uri":"dli_01_0663.html", "doc_type":"usermanual", "p_code":"34", "code":"35" }, { - "desc":"You can isolate queues allocated to different users by setting permissions to ensure data query performance.The administrator and queue owner have all permissions, which ", + "desc":"Administrators and queue owners have full operation permissions on queues. They can grant operation permissions to other users based on service needs. This ensures that u", "product_code":"dli", "title":"Queue Permission Management", "uri":"dli_01_0015.html", @@ -323,24 +323,6 @@ "p_code":"34", "code":"36" }, - { - "desc":"Before executing a job, you need to create a queue.If you use a sub-account to create a queue for the first time, log in to the DLI management console using the main acco", - "product_code":"dli", - "title":"Creating a Queue", - "uri":"dli_01_0363.html", - "doc_type":"usermanual", - "p_code":"34", - "code":"37" - }, - { - "desc":"You can delete a queue based on actual conditions.This operation will fail if there are jobs in the Submitting or Running state on this queue.Deleting a queue does not ca", - "product_code":"dli", - "title":"Deleting a Queue", - "uri":"dli_01_0016.html", - "doc_type":"usermanual", - "p_code":"34", - "code":"38" - }, { "desc":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", "product_code":"dli", @@ -348,43 +330,7 @@ "uri":"dli_01_0565.html", "doc_type":"usermanual", "p_code":"34", - "code":"39" - }, - { - "desc":"If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.If the queue whose CIDR block is to be modified", - "product_code":"dli", - "title":"Modifying the CIDR Block", - "uri":"dli_01_0443.html", - "doc_type":"usermanual", - "p_code":"34", - "code":"40" - }, - { - "desc":"Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.Queues with 16 CUs do not support scale-out or scale-in.Queues ", - "product_code":"dli", - "title":"Elastic Scaling of Queues", - "uri":"dli_01_0487.html", - "doc_type":"usermanual", - "p_code":"34", - "code":"41" - }, - { - "desc":"When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. ", - "product_code":"dli", - "title":"Scheduling CU Changes", - "uri":"dli_01_0488.html", - "doc_type":"usermanual", - "p_code":"34", - "code":"42" - }, - { - "desc":"It can be used to test the connectivity between the DLI queue and the peer IP address specified by the user in common scenarios, or the connectivity between the DLI queue", - "product_code":"dli", - "title":"Testing Address Connectivity", - "uri":"dli_01_0489.html", - "doc_type":"usermanual", - "p_code":"34", - "code":"43" + "code":"37" }, { "desc":"Once you have created an SMN topic, you can easily subscribe to it by going to the Topic Management > Topics page of the SMN console. You can choose to receive notificati", @@ -393,7 +339,7 @@ "uri":"dli_01_0421.html", "doc_type":"usermanual", "p_code":"34", - "code":"44" + "code":"38" }, { "desc":"A tag is a key-value pair that you can customize to identify cloud resources. It helps you to classify and search for cloud resources. A tag consists of a tag key and a t", @@ -402,7 +348,7 @@ "uri":"dli_01_0022.html", "doc_type":"usermanual", "p_code":"34", - "code":"45" + "code":"39" }, { "desc":"DLI allows you to set properties for queues.You can set Spark driver parameters to improve the scheduling efficiency of queues.This section describes how to set queue pro", @@ -411,259 +357,151 @@ "uri":"dli_01_0563.html", "doc_type":"usermanual", "p_code":"34", - "code":"46" + "code":"40" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "desc":"DLI's address connectivity testing feature can be used to verify network connectivity between DLI queues and destination addresses.This feature is typically utilized for ", "product_code":"dli", - "title":"Elastic Resource Pool", - "uri":"dli_01_0508.html", + "title":"Testing Address Connectivity", + "uri":"dli_01_0489.html", "doc_type":"usermanual", - "p_code":"", - "code":"47" + "p_code":"34", + "code":"41" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "desc":"You can delete a queue based on actual conditions.This operation will fail if there are jobs in the Submitting or Running state on this queue.Deleting a queue does not ca", "product_code":"dli", - "title":"Before You Start", - "uri":"dli_01_0528.html", + "title":"Deleting a Queue", + "uri":"dli_01_0016.html", "doc_type":"usermanual", - "p_code":"47", - "code":"48" + "p_code":"34", + "code":"42" }, { - "desc":"An elastic resource pool provides compute resources (CPU and memory) for running DLI jobs. The unit is CU. One CU contains one CPU and 4 GB memory.You can create multiple", + "desc":"Newly created queues need to run jobs before they can be scaled in or out.The operations described in this section only apply to standard queues.Queues with 16 CUs do not", "product_code":"dli", - "title":"Overview", - "uri":"dli_01_0504.html", + "title":"Auto Scaling of Standard Queues", + "uri":"dli_01_0487.html", "doc_type":"usermanual", - "p_code":"48", - "code":"49" + "p_code":"34", + "code":"43" + }, + { + "desc":"When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. ", + "product_code":"dli", + "title":"Setting a Scheduled Auto Scaling Task for a Standard Queue", + "uri":"dli_01_0488.html", + "doc_type":"usermanual", + "p_code":"34", + "code":"44" + }, + { + "desc":"If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.If the queue whose CIDR block is to be modified", + "product_code":"dli", + "title":"Changing the CIDR Block for a Standard Queue", + "uri":"dli_01_0443.html", + "doc_type":"usermanual", + "p_code":"34", + "code":"45" }, { "desc":"This section walks you through the procedure of adding a queue to an elastic resource pool and binding an enhanced datasource connection to the elastic resource pool.Proc", "product_code":"dli", - "title":"Creating an Elastic Resource Pool and Running a Job", + "title":"Example Use Case: Creating an Elastic Resource Pool and Running Jobs", "uri":"dli_01_0515.html", "doc_type":"usermanual", - "p_code":"48", - "code":"50" + "p_code":"21", + "code":"46" }, { "desc":"A company has multiple departments that perform data analysis in different periods during a day.Department A requires a large number of compute resources from 00:00 a.m. ", "product_code":"dli", - "title":"Configuring Scaling Policies for Queues", + "title":"Example Use Case: Configuring Scaling Policies for Queues in an Elastic Resource Pool", "uri":"dli_01_0516.html", "doc_type":"usermanual", - "p_code":"48", - "code":"51" + "p_code":"21", + "code":"47" + }, + { + "desc":"Queues in the non-elastic resource pool mode are the previous-gen of resource management for DLI. It involved purchasing and releasing resources based on usage demands, r", + "product_code":"dli", + "title":"Creating a Non-Elastic Resource Pool Queue (Discarded and Not Recommended)", + "uri":"dli_01_0363.html", + "doc_type":"usermanual", + "p_code":"21", + "code":"48" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Regular Operations", - "uri":"dli_01_0529.html", - "doc_type":"usermanual", - "p_code":"47", - "code":"52" - }, - { - "desc":"For details about the application scenarios of elastic resource pools, see the Overview. This section describes how to create an elastic resource pool.If you use an enhan", - "product_code":"dli", - "title":"Creating an Elastic Resource Pool", - "uri":"dli_01_0505.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"53" - }, - { - "desc":"Administrators can assign permissions of different operation scopes to users for each elastic resource pool.The administrator and elastic resource pool owner have all per", - "product_code":"dli", - "title":"Managing Permissions", - "uri":"dli_01_0526.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"54" - }, - { - "desc":"You can add one or more queues to an elastic resource pool to run jobs. This section describes how to add a queue to an elastic resource pool.Automatic scaling of an elas", - "product_code":"dli", - "title":"Adding a Queue", - "uri":"dli_01_0509.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"55" - }, - { - "desc":"If you want a queue to use resources in an elastic resource pool, bind the queue to the pool.You can click Associate Queue on the Resource Pool page to bind a queue to an", - "product_code":"dli", - "title":"Binding a Queue", - "uri":"dli_01_0530.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"56" - }, - { - "desc":"Multiple queues can be added to an elastic resource pool. For details about how to add a queue, see Adding a Queue. You can configure the number of CUs you want based on ", - "product_code":"dli", - "title":"Managing Queues", - "uri":"dli_01_0506.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"57" - }, - { - "desc":"CU settings are used to control the maximum and minimum CU ranges for elastic resource pools to avoid unlimited resource scaling.For example, an elastic resource pool has", - "product_code":"dli", - "title":"Setting CUs", - "uri":"dli_01_0507.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"58" - }, - { - "desc":"If the current specifications of your elastic resource pool do not meet your service needs, you can modify them using the change specifications function.In the navigation", - "product_code":"dli", - "title":"Modifying Specifications", - "uri":"dli_01_0524.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"59" - }, - { - "desc":"A tag is a key-value pair that you can customize to identify cloud resources. It helps you to classify and search for cloud resources. A tag consists of a tag key and a t", - "product_code":"dli", - "title":"Managing Tags", - "uri":"dli_01_0525.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"60" - }, - { - "desc":"If you added a queue to or deleted one from an elastic resource pool, or you scaled an added queue, the CU quantity of the elastic resource pool may be changed. You can v", - "product_code":"dli", - "title":"Viewing Scaling History", - "uri":"dli_01_0532.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"61" - }, - { - "desc":"You can create enterprise projects matching the organizational structure of your enterprises to centrally manage cloud resources across regions by project. Then you can c", - "product_code":"dli", - "title":"Allocating to an Enterprise Project", - "uri":"dli_01_0566.html", - "doc_type":"usermanual", - "p_code":"52", - "code":"62" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Data Management", - "uri":"dli_01_0004.html", - "doc_type":"usermanual", - "p_code":"", - "code":"63" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Databases and Tables", + "title":"Creating Databases and Tables", "uri":"dli_01_0390.html", "doc_type":"usermanual", - "p_code":"63", - "code":"64" + "p_code":"", + "code":"49" }, { - "desc":"DLI database and table management provide the following functions:Database Permission ManagementTable Permission ManagementCreating a Database or a TableDeleting a Databa", + "desc":"Databases and tables are the basis for developing SQL and Spark jobs. Before running a job, you need to define databases and tables based on your service scenarios.Flink ", "product_code":"dli", - "title":"Overview", + "title":"Understanding Data Catalogs, Databases, and Tables", "uri":"dli_01_0228.html", "doc_type":"usermanual", - "p_code":"64", - "code":"65" - }, - { - "desc":"By setting permissions, you can assign varying database permissions to different users.The administrator and database owner have all permissions, which cannot be set or m", - "product_code":"dli", - "title":"Managing Database Permissions", - "uri":"dli_01_0447.html", - "doc_type":"usermanual", - "p_code":"64", - "code":"66" - }, - { - "desc":"By setting permissions, you can assign varying table permissions to different users.The administrator and table owner have all permissions, which cannot be set or modifie", - "product_code":"dli", - "title":"Managing Table Permissions", - "uri":"dli_01_0448.html", - "doc_type":"usermanual", - "p_code":"64", - "code":"67" + "p_code":"49", + "code":"50" }, { "desc":"A database, built on the computer storage device, is a data warehouse where data is organized, stored, and managed based on its structure.The table is an important part o", "product_code":"dli", - "title":"Creating a Database or a Table", + "title":"Creating a Database and Table on the DLI Console", "uri":"dli_01_0005.html", "doc_type":"usermanual", - "p_code":"64", - "code":"68" - }, - { - "desc":"You can delete unnecessary databases and tables based on actual conditions.You are not allowed to delete databases or tables that are being used for running jobs.The admi", - "product_code":"dli", - "title":"Deleting a Database or a Table", - "uri":"dli_01_0011.html", - "doc_type":"usermanual", - "p_code":"64", - "code":"69" - }, - { - "desc":"During actual use, developers create databases and tables and submit them to test personnel for testing. After the test is complete, the databases and tables are transfer", - "product_code":"dli", - "title":"Modifying the Owners of Databases and Tables", - "uri":"dli_01_0376.html", - "doc_type":"usermanual", - "p_code":"64", - "code":"70" - }, - { - "desc":"You can import data from OBS to a table created in DLI.Only one path can be specified during data import. The path cannot contain commas (,).To import data in CSV format ", - "product_code":"dli", - "title":"Importing Data to the Table", - "uri":"dli_01_0253.html", - "doc_type":"usermanual", - "p_code":"64", - "code":"71" - }, - { - "desc":"You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.The exported file can be i", - "product_code":"dli", - "title":"Exporting Data from DLI to OBS", - "uri":"dli_01_0010.html", - "doc_type":"usermanual", - "p_code":"64", - "code":"72" + "p_code":"49", + "code":"51" }, { "desc":"Metadata is used to define data types. It describes information about the data, including the source, size, format, and other data features. In database fields, metadata ", "product_code":"dli", - "title":"Viewing Metadata", + "title":"Viewing Table Metadata", "uri":"dli_01_0008.html", "doc_type":"usermanual", - "p_code":"64", - "code":"73" + "p_code":"49", + "code":"52" }, { - "desc":"The Preview page displays the first 10 records in the table.You can preview data on either the Data Management page or the SQL Editor page.To preview data on the Data Man", + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Previewing Data", - "uri":"dli_01_0007.html", + "title":"Managing Database Resources on the DLI Console", + "uri":"dli_01_0004.html", "doc_type":"usermanual", - "p_code":"64", - "code":"74" + "p_code":"49", + "code":"53" + }, + { + "desc":"By setting permissions, you can assign varying database permissions to different users.The administrator and database owner have all permissions, which cannot be set or m", + "product_code":"dli", + "title":"Configuring Database Permissions on the DLI Console", + "uri":"dli_01_0447.html", + "doc_type":"usermanual", + "p_code":"53", + "code":"54" + }, + { + "desc":"You can delete databases as needed.You are not allowed to delete databases or tables that are being used for running jobs.The administrator, database owner, and users wit", + "product_code":"dli", + "title":"Deleting a Database on the DLI Console", + "uri":"dli_01_0011.html", + "doc_type":"usermanual", + "p_code":"53", + "code":"55" + }, + { + "desc":"In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed", + "product_code":"dli", + "title":"Changing the Database Owner on the DLI Console", + "uri":"dli_01_0376.html", + "doc_type":"usermanual", + "p_code":"53", + "code":"56" }, { "desc":"A tag is a key-value pair that you can customize to identify cloud resources. It helps you to classify and search for cloud resources. A tag consists of a tag key and a t", @@ -671,152 +509,125 @@ "title":"Managing Tags", "uri":"dli_01_0552.html", "doc_type":"usermanual", - "p_code":"64", - "code":"75" + "p_code":"53", + "code":"57" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Package Management", - "uri":"dli_01_0366.html", + "title":"Managing Table Resources on the DLI Console", + "uri":"dli_01_0625.html", "doc_type":"usermanual", - "p_code":"63", - "code":"76" + "p_code":"49", + "code":"58" }, { - "desc":"Package management provides the following functions:Managing Package PermissionsCreating a PackageDeleting a PackageYou can delete program packages in batches.You can del", + "desc":"By setting permissions, you can assign varying table permissions to different users.The administrator and table owner have all permissions, which cannot be set or modifie", + "product_code":"dli", + "title":"Configuring Table Permissions on the DLI Console", + "uri":"dli_01_0448.html", + "doc_type":"usermanual", + "p_code":"58", + "code":"59" + }, + { + "desc":"You can delete tables as needed.Databases or tables that are being used for running jobs cannot be deleted.Only administrators, table owners, and users with table deletio", + "product_code":"dli", + "title":"Deleting a Table on the DLI Console", + "uri":"dli_01_0626.html", + "doc_type":"usermanual", + "p_code":"58", + "code":"60" + }, + { + "desc":"In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed", + "product_code":"dli", + "title":"Changing the Table Owner on the DLI Console", + "uri":"dli_01_0627.html", + "doc_type":"usermanual", + "p_code":"58", + "code":"61" + }, + { + "desc":"This section describes how to import data stored in OBS to a table on the DLI console.Only one path can be specified during data import. The path cannot contain commas (,", + "product_code":"dli", + "title":"Importing OBS Data to DLI", + "uri":"dli_01_0253.html", + "doc_type":"usermanual", + "p_code":"58", + "code":"62" + }, + { + "desc":"You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.The exported file can be i", + "product_code":"dli", + "title":"Exporting DLI Table Data to OBS", + "uri":"dli_01_0010.html", + "doc_type":"usermanual", + "p_code":"58", + "code":"63" + }, + { + "desc":"The Preview page displays the first 10 records in the table.You can preview data on either the Data Management page or the SQL Editor page.To preview data on the Data Man", + "product_code":"dli", + "title":"Previewing Table Data on the DLI Console", + "uri":"dli_01_0007.html", + "doc_type":"usermanual", + "p_code":"58", + "code":"64" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Data Migration and Transmission", + "uri":"dli_01_0661.html", + "doc_type":"usermanual", + "p_code":"", + "code":"65" + }, + { + "desc":"DLI enables direct access to data stored in OBS for query and analysis, eliminating the need for data migration.To begin using DLI for data analysis, just import your loc", "product_code":"dli", "title":"Overview", - "uri":"dli_01_0407.html", + "uri":"dli_01_0662.html", "doc_type":"usermanual", - "p_code":"76", - "code":"77" - }, - { - "desc":"You can isolate package groups or packages allocated to different users by setting permissions to ensure data query performance.The administrator and the owner of a packa", - "product_code":"dli", - "title":"Managing Permissions on Packages and Package Groups", - "uri":"dli_01_0477.html", - "doc_type":"usermanual", - "p_code":"76", - "code":"78" - }, - { - "desc":"DLI allows you to submit program packages in batches to the general-use queue for running.If you need to update a package, you can use the same package or file to upload ", - "product_code":"dli", - "title":"Creating a Package", - "uri":"dli_01_0367.html", - "doc_type":"usermanual", - "p_code":"76", - "code":"79" - }, - { - "desc":"You can delete a package based on actual conditions.On the left of the management console, choose Data Management > Package Management.Click Delete in the Operation colum", - "product_code":"dli", - "title":"Deleting a Package", - "uri":"dli_01_0369.html", - "doc_type":"usermanual", - "p_code":"76", - "code":"80" - }, - { - "desc":"To change the owner of a package, click More > Modify Owner in the Operation column of a package on the Package Management page.If the package has been grouped, you can m", - "product_code":"dli", - "title":"Modifying the Owner", - "uri":"dli_01_0478.html", - "doc_type":"usermanual", - "p_code":"76", - "code":"81" - }, - { - "desc":"DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar ", - "product_code":"dli", - "title":"Built-in Dependencies", - "uri":"dli_01_0397.html", - "doc_type":"usermanual", - "p_code":"76", - "code":"82" + "p_code":"65", + "code":"66" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Job Templates", - "uri":"dli_01_0379.html", + "title":"Configuring DLI to Read and Write Data from and to External Data Sources", + "uri":"dli_01_0634.html", "doc_type":"usermanual", - "p_code":"", - "code":"83" + "p_code":"65", + "code":"67" }, { - "desc":"To facilitate SQL operation execution, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not n", + "desc":"To read and write external data sources when running DLI jobs, two conditions must be met:Establish network connectivity between DLI and the external data source to ensur", "product_code":"dli", - "title":"Managing SQL Templates", - "uri":"dli_01_0021.html", + "title":"Configuring DLI to Read and Write External Data Sources", + "uri":"dli_01_0635.html", "doc_type":"usermanual", - "p_code":"83", - "code":"84" - }, - { - "desc":"Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for edit", - "product_code":"dli", - "title":"Managing Flink Templates", - "uri":"dli_01_0464.html", - "doc_type":"usermanual", - "p_code":"83", - "code":"85" - }, - { - "desc":"You can modify a sample template to meet the Spark job requirements, saving time for editing SQL statements.Currently, the cloud platform does not provide preset Spark te", - "product_code":"dli", - "title":"Managing Spark SQL Templates", - "uri":"dli_01_0551.html", - "doc_type":"usermanual", - "p_code":"83", - "code":"86" + "p_code":"67", + "code":"68" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Appendix", - "uri":"dli_01_05110.html", - "doc_type":"usermanual", - "p_code":"83", - "code":"87" - }, - { - "desc":"TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and ", - "product_code":"dli", - "title":"TPC-H Sample Data in the SQL Template", - "uri":"dli_01_05111.html", - "doc_type":"usermanual", - "p_code":"87", - "code":"88" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Enhanced Datasource Connections", + "title":"Configuring the Network Connection Between DLI and Data Sources (Enhanced Datasource Connection)", "uri":"dli_01_0426.html", "doc_type":"usermanual", - "p_code":"", - "code":"89" + "p_code":"67", + "code":"69" }, { "desc":"In cross-source data analysis scenarios, DLI needs to connect to external data sources. However, due to the different VPCs between the data source and DLI, the network ca", "product_code":"dli", - "title":"Overview", + "title":"Overview of Enhanced Datasource Connections", "uri":"dli_01_0003.html", "doc_type":"usermanual", - "p_code":"89", - "code":"90" - }, - { - "desc":"If DLI needs to access external data sources, you need to establish enhanced datasource connections to enable the network between DLI and the data sources, and then devel", - "product_code":"dli", - "title":"Cross-Source Analysis Development Methods", - "uri":"dli_01_0410.html", - "doc_type":"usermanual", - "p_code":"89", - "code":"91" + "p_code":"69", + "code":"70" }, { "desc":"Create an enhanced datasource connection for DLI to access, import, query, and analyze data of other data sources.For example, to connect DLI to the MRS, RDS, CSS, Kafka,", @@ -824,98 +635,44 @@ "title":"Creating an Enhanced Datasource Connection", "uri":"dli_01_0006.html", "doc_type":"usermanual", - "p_code":"89", - "code":"92" + "p_code":"69", + "code":"71" }, { - "desc":"VPC sharing allows sharing VPC resources created in one account with other accounts using Resource Access Manager (RAM). For example, account A can share its VPC and subn", + "desc":"If DLI needs to access external data sources, you need to establish enhanced datasource connections to enable the network between DLI and the data sources, and then devel", "product_code":"dli", - "title":"Establishing a Network Connection Between DLI and Resources in a Shared VPC", - "uri":"dli_01_0624.html", + "title":"Common Development Methods for DLI Cross-Source Analysis", + "uri":"dli_01_0410.html", "doc_type":"usermanual", - "p_code":"89", - "code":"93" + "p_code":"69", + "code":"72" }, { - "desc":"Delete an enhanced datasource connection that is no longer used on the console.Log in to the DLI management console.In the left navigation pane, choose Datasource Connect", + "desc":"When using DLI to submit jobs that involve reading and writing data from external sources, it is crucial to securely access these sources by properly storing their access", "product_code":"dli", - "title":"Deleting an Enhanced Datasource Connection", - "uri":"dli_01_0553.html", + "title":"Using DEW to Manage Access Credentials for Data Sources", + "uri":"dli_01_0636.html", "doc_type":"usermanual", - "p_code":"89", - "code":"94" - }, - { - "desc":"Host information is the mapping between host IP addresses and domain names. After you configure host information, jobs can only use the configured domain names to access ", - "product_code":"dli", - "title":"Modifying Host Information in an Elastic Resource Pool", - "uri":"dli_01_0013.html", - "doc_type":"usermanual", - "p_code":"89", - "code":"95" - }, - { - "desc":"The CIDR block of the DLI queue that is bound with a datasource connection cannot overlap with that of the data source.The default queue cannot be bound with a connection", - "product_code":"dli", - "title":"Binding and Unbinding a Queue", - "uri":"dli_01_0514.html", - "doc_type":"usermanual", - "p_code":"89", - "code":"96" - }, - { - "desc":"A route is configured with the destination, next hop type, and next hop to determine where the network traffic is directed. Routes are classified into system routes and c", - "product_code":"dli", - "title":"Adding a Route", - "uri":"dli_01_0014.html", - "doc_type":"usermanual", - "p_code":"89", - "code":"97" - }, - { - "desc":"Delete a route that is no longer used.A custom route table cannot be deleted if it is associated with a subnet.Log in to the DLI management console.In the left navigation", - "product_code":"dli", - "title":"Deleting a Route", - "uri":"dli_01_0556.html", - "doc_type":"usermanual", - "p_code":"89", - "code":"98" - }, - { - "desc":"Enhanced connections support user authorization by project. After authorization, users in the project have the permission to perform operations on the enhanced connection", - "product_code":"dli", - "title":"Enhanced Connection Permission Management", - "uri":"dli_01_0018.html", - "doc_type":"usermanual", - "p_code":"89", - "code":"99" - }, - { - "desc":"A tag is a key-value pair customized by users and used to identify cloud resources. It helps users to classify and search for cloud resources. A tag consists of a tag key", - "product_code":"dli", - "title":"Enhanced Datasource Connection Tag Management", - "uri":"dli_01_0019.html", - "doc_type":"usermanual", - "p_code":"89", - "code":"100" + "p_code":"67", + "code":"73" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Datasource Authentication", + "title":"Using DLI Datasource Authentication to Manage Access Credentials for Data Sources", "uri":"dli_01_0422.html", "doc_type":"usermanual", - "p_code":"", - "code":"101" + "p_code":"67", + "code":"74" }, { - "desc":"When analyzing across multiple sources, it is not recommended to configure authentication information directly in a job as it can lead to password leakage. Instead, you a", + "desc":"When analyzing across multiple sources, you are advised not to configure authentication information directly in a job as it can lead to password leakage. Instead, you are", "product_code":"dli", "title":"Overview", "uri":"dli_01_0561.html", "doc_type":"usermanual", - "p_code":"101", - "code":"102" + "p_code":"74", + "code":"75" }, { "desc":"Create a CSS datasource authentication on the DLI console to store the authentication information of the CSS security cluster to DLI. This will allow you to access to the", @@ -923,8 +680,8 @@ "title":"Creating a CSS Datasource Authentication", "uri":"dli_01_0427.html", "doc_type":"usermanual", - "p_code":"101", - "code":"103" + "p_code":"74", + "code":"76" }, { "desc":"Create a Kerberos datasource authentication on the DLI console to store the authentication information of the data source to DLI. This will allow you to access to the dat", @@ -932,8 +689,8 @@ "title":"Creating a Kerberos Datasource Authentication", "uri":"dli_01_0558.html", "doc_type":"usermanual", - "p_code":"101", - "code":"104" + "p_code":"74", + "code":"77" }, { "desc":"Create a Kafka_SSL datasource authentication on the DLI console to store the Kafka authentication information to DLI. This will allow you to access to Kafka instances wit", @@ -941,8 +698,8 @@ "title":"Creating a Kafka_SSL Datasource Authentication", "uri":"dli_01_0560.html", "doc_type":"usermanual", - "p_code":"101", - "code":"105" + "p_code":"74", + "code":"78" }, { "desc":"Create a password datasource authentication on the DLI console to store passwords of the GaussDB(DWS), RDS, DCS, and DDS data sources to DLI. This will allow you to acces", @@ -950,8 +707,8 @@ "title":"Creating a Password Datasource Authentication", "uri":"dli_01_0559.html", "doc_type":"usermanual", - "p_code":"101", - "code":"106" + "p_code":"74", + "code":"79" }, { "desc":"Grant permissions on a datasource authentication to users so multiple user jobs can use the datasource authentication without affecting each other.The administrator and t", @@ -959,36 +716,450 @@ "title":"Datasource Authentication Permission Management", "uri":"dli_01_0480.html", "doc_type":"usermanual", - "p_code":"101", - "code":"107" + "p_code":"74", + "code":"80" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Global Configuration", - "uri":"dli_01_0485.html", + "title":"Managing Enhanced Datasource Connections", + "uri":"dli_01_0509.html", + "doc_type":"usermanual", + "p_code":"67", + "code":"81" + }, + { + "desc":"After creating an enhanced datasource connection, you can view and manage it on the management console.This section describes how to view basic information about an enhan", + "product_code":"dli", + "title":"Viewing Basic Information About an Enhanced Datasource Connection", + "uri":"dli_01_0623.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"82" + }, + { + "desc":"Enhanced connections support user authorization by project. After authorization, users in the project have the permission to perform operations on the enhanced connection", + "product_code":"dli", + "title":"Enhanced Connection Permission Management", + "uri":"dli_01_0018.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"83" + }, + { + "desc":"Bind an elastic resource pool to a queue to connect the queue to a target data source. When an enhanced datasource connection is no longer needed, unbind the queue that w", + "product_code":"dli", + "title":"Binding or Unbinding an Enhanced Datasource Connection to a Queue", + "uri":"dli_01_0514.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"84" + }, + { + "desc":"A route is configured with the destination, next hop type, and next hop to determine where the network traffic is directed. Routes are classified into system routes and c", + "product_code":"dli", + "title":"Adding a Route for an Enhanced Datasource Connection", + "uri":"dli_01_0014.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"85" + }, + { + "desc":"Delete a route that is no longer used.A custom route table cannot be deleted if it is associated with a subnet.Log in to the DLI management console.In the left navigation", + "product_code":"dli", + "title":"Deleting the Route for an Enhanced Datasource Connection", + "uri":"dli_01_0556.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"86" + }, + { + "desc":"Host information is the mapping between host IP addresses and domain names. After you configure host information, jobs can only use the configured domain names to access ", + "product_code":"dli", + "title":"Modifying Host Information in an Elastic Resource Pool", + "uri":"dli_01_0013.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"87" + }, + { + "desc":"A tag is a key-value pair customized by users and used to identify cloud resources. It helps users to classify and search for cloud resources. A tag consists of a tag key", + "product_code":"dli", + "title":"Enhanced Datasource Connection Tag Management", + "uri":"dli_01_0019.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"88" + }, + { + "desc":"Delete an enhanced datasource connection that is no longer used on the console.Log in to the DLI management console.In the left navigation pane, choose Datasource Connect", + "product_code":"dli", + "title":"Deleting an Enhanced Datasource Connection", + "uri":"dli_01_0553.html", + "doc_type":"usermanual", + "p_code":"81", + "code":"89" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Configuring an Agency to Allow DLI to Access Other Cloud Services", + "uri":"dli_01_0486.html", "doc_type":"usermanual", "p_code":"", + "code":"90" + }, + { + "desc":"Cloud services often interact with each other, with some of which dependent on other services. You can create an agency to delegate DLI to use other cloud services and pe", + "product_code":"dli", + "title":"DLI Agency Overview", + "uri":"dli_01_0419.html", + "doc_type":"usermanual", + "p_code":"90", + "code":"91" + }, + { + "desc":"When Flink 1.15, Spark 3.3, or a later version is used to execute jobs and the required agency is not included in the DLI system agency dli_management_agency, you need to", + "product_code":"dli", + "title":"Creating a Custom DLI Agency", + "uri":"dli_01_0616.html", + "doc_type":"usermanual", + "p_code":"90", + "code":"92" + }, + { + "desc":"This section provides agency permission policies for common scenarios, which can be used to configure agency permission policies when you customize your permissions. The ", + "product_code":"dli", + "title":"Agency Permission Policies in Common Scenarios", + "uri":"dli_01_0617.html", + "doc_type":"usermanual", + "p_code":"90", + "code":"93" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Submitting a SQL Job Using DLI", + "uri":"dli_01_0402.html", + "doc_type":"usermanual", + "p_code":"", + "code":"94" + }, + { + "desc":"The SQL Editor allows you to execute data query operations using SQL statements.It supports SQL:2003 and is compatible with Spark SQL.To access the SQL editor, choose SQL", + "product_code":"dli", + "title":"Creating and Submitting a SQL Job", + "uri":"dli_01_0320.html", + "doc_type":"usermanual", + "p_code":"94", + "code":"95" + }, + { + "desc":"Store the data results of analyzed SQL jobs in a specified location in the desired format.By default, DLI stores SQL job results in its job bucket. You can also download ", + "product_code":"dli", + "title":"Exporting SQL Job Results", + "uri":"dli_01_0621.html", + "doc_type":"usermanual", + "p_code":"94", + "code":"96" + }, + { + "desc":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", + "product_code":"dli", + "title":"Setting the Priority for a SQL Job", + "uri":"dli_01_0535.html", + "doc_type":"usermanual", + "p_code":"94", + "code":"97" + }, + { + "desc":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", + "product_code":"dli", + "title":"Querying Logs for SQL Jobs", + "uri":"dli_01_0637.html", + "doc_type":"usermanual", + "p_code":"94", + "code":"98" + }, + { + "desc":"The SQL Jobs page displays all SQL jobs, which may span multiple pages if there are many jobs. You can navigate to a specific page as needed. DLI allows you to view jobs ", + "product_code":"dli", + "title":"Managing SQL Jobs", + "uri":"dli_01_0017.html", + "doc_type":"usermanual", + "p_code":"94", + "code":"99" + }, + { + "desc":"A SQL execution plan is a logical flowchart of a database query that shows how a database management system executes a specific SQL query. The execution plan details the ", + "product_code":"dli", + "title":"Viewing a SQL Execution Plan", + "uri":"dli_01_0638.html", + "doc_type":"usermanual", + "p_code":"94", + "code":"100" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Creating and Managing SQL Job Templates", + "uri":"dli_01_05110.html", + "doc_type":"usermanual", + "p_code":"94", + "code":"101" + }, + { + "desc":"To facilitate SQL operation execution, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not n", + "product_code":"dli", + "title":"Creating a SQL Job Template", + "uri":"dli_01_0021.html", + "doc_type":"usermanual", + "p_code":"101", + "code":"102" + }, + { + "desc":"DLI allows you to create custom templates or save currently used SQL statements as templates for quick and convenient SQL operations. Once a template is saved, you can ex", + "product_code":"dli", + "title":"Developing and Submitting a SQL Job Using a SQL Job Template", + "uri":"dli_01_0639.html", + "doc_type":"usermanual", + "p_code":"101", + "code":"103" + }, + { + "desc":"TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and ", + "product_code":"dli", + "title":"TPC-H Sample Data in the SQL Templates Preset on DLI", + "uri":"dli_01_05111.html", + "doc_type":"usermanual", + "p_code":"101", + "code":"104" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Submitting a Flink Job Using DLI", + "uri":"dli_01_0389.html", + "doc_type":"usermanual", + "p_code":"", + "code":"105" + }, + { + "desc":"DLI supports two types of Flink jobs:Flink OpenSource SQL job:It is fully compatible with Flink of the community edition, ensuring that jobs can run smoothly on these Fli", + "product_code":"dli", + "title":"Flink Job Overview", + "uri":"dli_01_0403.html", + "doc_type":"usermanual", + "p_code":"105", + "code":"106" + }, + { + "desc":"This section describes how to create a Flink OpenSource SQL job.DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink provided by the community. In ", + "product_code":"dli", + "title":"Creating a Flink OpenSource SQL Job", + "uri":"dli_01_0498.html", + "doc_type":"usermanual", + "p_code":"105", + "code":"107" + }, + { + "desc":"A Flink Jar job involves developing a custom application Jar package based on Flink's capabilities and submitting it to a DLI queue for execution.To create a Flink Jar jo", + "product_code":"dli", + "title":"Creating a Flink Jar Job", + "uri":"dli_01_0457.html", + "doc_type":"usermanual", + "p_code":"105", "code":"108" }, { - "desc":"DLI allows you to set variables that are frequently used during job development as global variables on the DLI management console. This avoids repeated definitions during", + "desc":"You can isolate Flink jobs allocated to different users by setting permissions to ensure data query performance.The administrator and job creator have all permissions, wh", "product_code":"dli", - "title":"Global Variables", - "uri":"dli_01_0476.html", + "title":"Configuring Flink Job Permissions", + "uri":"dli_01_0479.html", "doc_type":"usermanual", - "p_code":"108", + "p_code":"105", "code":"109" }, { - "desc":"You can grant permissions on a global variable to users.The administrator and the global variable owner have all permissions. You do not need to set permissions for them,", + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Permission Management for Global Variables", - "uri":"dli_01_0533.html", + "title":"Managing Flink Jobs", + "uri":"dli_01_0377.html", "doc_type":"usermanual", - "p_code":"108", + "p_code":"105", "code":"110" }, + { + "desc":"After creating a Flink job, you can check the basic information, job details, task list, and execution plan of the job on the DLI console.This section describes how to ch", + "product_code":"dli", + "title":"Viewing Flink Job Details", + "uri":"dli_01_0462.html", + "doc_type":"usermanual", + "p_code":"110", + "code":"111" + }, + { + "desc":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", + "product_code":"dli", + "title":"Setting the Priority for a Flink Job", + "uri":"dli_01_0650.html", + "doc_type":"usermanual", + "p_code":"110", + "code":"112" + }, + { + "desc":"In actual job operations, the compute resources required by a job vary depending on the data volume. As a result, compute resources are wasted when the volume is small an", + "product_code":"dli", + "title":"Enabling Dynamic Scaling for Flink Jobs", + "uri":"dli_01_0534.html", + "doc_type":"usermanual", + "p_code":"110", + "code":"113" + }, + { + "desc":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", + "product_code":"dli", + "title":"Querying Logs for Flink Jobs", + "uri":"dli_01_0651.html", + "doc_type":"usermanual", + "p_code":"110", + "code":"114" + }, + { + "desc":"After creating a job, you can manage it by performing various operations such as editing its basic information, starting or stopping it, and importing or exporting it.You", + "product_code":"dli", + "title":"Common Operations of Flink Jobs", + "uri":"dli_01_0461.html", + "doc_type":"usermanual", + "p_code":"110", + "code":"115" + }, + { + "desc":"Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for edit", + "product_code":"dli", + "title":"Managing Flink Job Templates", + "uri":"dli_01_0464.html", + "doc_type":"usermanual", + "p_code":"105", + "code":"116" + }, + { + "desc":"A tag is a key-value pair customized by users and used to identify cloud resources. It helps users to classify and search for cloud resources. A tag consists of a tag key", + "product_code":"dli", + "title":"Adding Tags to a Flink Job", + "uri":"dli_01_0463.html", + "doc_type":"usermanual", + "p_code":"105", + "code":"117" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Submitting a Spark Job Using DLI", + "uri":"dli_01_0465.html", + "doc_type":"usermanual", + "p_code":"", + "code":"118" + }, + { + "desc":"DLI provides fully-managed Spark computing services by allowing you to execute Spark jobs.On the Overview page, click Create Job in the upper right corner of the Spark Jo", + "product_code":"dli", + "title":"Creating a Spark Job", + "uri":"dli_01_0384.html", + "doc_type":"usermanual", + "p_code":"118", + "code":"119" + }, + { + "desc":"In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. Th", + "product_code":"dli", + "title":"Setting the Priority for a Spark Job", + "uri":"dli_01_0652.html", + "doc_type":"usermanual", + "p_code":"118", + "code":"120" + }, + { + "desc":"DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.This section describes how to configure a bucket for DLI j", + "product_code":"dli", + "title":"Querying Logs for Spark Jobs", + "uri":"dli_01_0379.html", + "doc_type":"usermanual", + "p_code":"118", + "code":"121" + }, + { + "desc":"On the Overview page, click Spark Jobs to go to the SQL job management page. Alternatively, you can click Job Management > Spark Jobs. The page displays all Spark jobs. I", + "product_code":"dli", + "title":"Managing Spark Jobs", + "uri":"dli_01_0385.html", + "doc_type":"usermanual", + "p_code":"118", + "code":"122" + }, + { + "desc":"You can modify a sample template to meet the Spark job requirements, saving time for editing SQL statements.Currently, the cloud platform does not provide preset Spark te", + "product_code":"dli", + "title":"Managing Spark Job Templates", + "uri":"dli_01_0551.html", + "doc_type":"usermanual", + "p_code":"118", + "code":"123" + }, + { + "desc":"This section describes metrics reported by DLI to Cloud Eye as well as their namespaces and dimensions. You can use the management console or APIs provided by Cloud Eye t", + "product_code":"dli", + "title":"Using Cloud Eye to Monitor DLI", + "uri":"dli_01_0445.html", + "doc_type":"usermanual", + "p_code":"", + "code":"124" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Using AOM to Monitor DLI", + "uri":"dli_01_0664.html", + "doc_type":"usermanual", + "p_code":"", + "code":"125" + }, + { + "desc":"AOM's Prometheus Monitoring is a comprehensive solution that integrates with the open source Prometheus ecosystem. It supports monitoring of various types of components, ", + "product_code":"dli", + "title":"Connecting DLI to AOM's Prometheus Monitoring", + "uri":"dli_01_0665.html", + "doc_type":"usermanual", + "p_code":"125", + "code":"126" + }, + { + "desc":"When connecting DLI to AOM's Prometheus Monitoring, the system automatically configures the parameters listed in table 1 in Configuration Items for Connecting DLI to AOM'", + "product_code":"dli", + "title":"Configuration Items for Connecting DLI to AOM's Prometheus Monitoring", + "uri":"dli_01_0666.html", + "doc_type":"usermanual", + "p_code":"125", + "code":"127" + }, + { + "desc":"Table 1 lists the basic monitoring metrics that are reported by DLI to Prometheus. AOM Prometheus can store these basic metrics for free.", + "product_code":"dli", + "title":"Basic Monitoring Metrics Reported by DLI to Prometheus", + "uri":"dli_01_0667.html", + "doc_type":"usermanual", + "p_code":"125", + "code":"128" + }, + { + "desc":"With CTS, you can log operations related to DLI, making it easier to search, audit, and trace in the future.", + "product_code":"dli", + "title":"Using CTS to Audit DLI", + "uri":"dli_01_0318.html", + "doc_type":"usermanual", + "p_code":"", + "code":"129" + }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", @@ -996,7 +1167,7 @@ "uri":"dli_01_0408.html", "doc_type":"usermanual", "p_code":"", - "code":"111" + "code":"130" }, { "desc":"DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM", @@ -1004,17 +1175,8 @@ "title":"Overview", "uri":"dli_01_0440.html", "doc_type":"usermanual", - "p_code":"111", - "code":"112" - }, - { - "desc":"You can use Identity and Access Management (IAM) to implement fine-grained permissions control on DLI resources. For details, see Overview.If your cloud account does not ", - "product_code":"dli", - "title":"Creating an IAM User and Granting Permissions", - "uri":"dli_01_0418.html", - "doc_type":"usermanual", - "p_code":"111", - "code":"113" + "p_code":"130", + "code":"131" }, { "desc":"Custom policies can be created as a supplement to the system policies of DLI. You can add actions to custom policies. For the actions supported for custom policies, see \"", @@ -1022,8 +1184,8 @@ "title":"Creating a Custom Policy", "uri":"dli_01_0451.html", "doc_type":"usermanual", - "p_code":"111", - "code":"114" + "p_code":"130", + "code":"132" }, { "desc":"A resource is an object that exists within a service. You can select DLI resources by specifying their paths.", @@ -1031,8 +1193,8 @@ "title":"DLI Resources", "uri":"dli_01_0417.html", "doc_type":"usermanual", - "p_code":"111", - "code":"115" + "p_code":"130", + "code":"133" }, { "desc":"Request conditions are useful in determining when a custom policy takes effect. A request condition consists of a condition key and operator. Condition keys are either gl", @@ -1040,8 +1202,8 @@ "title":"DLI Request Conditions", "uri":"dli_01_0475.html", "doc_type":"usermanual", - "p_code":"111", - "code":"116" + "p_code":"130", + "code":"134" }, { "desc":"Table 1 lists the common operations supported by each system policy of DLI. Choose proper system policies according to this table. For details about the SQL statement per", @@ -1049,53 +1211,107 @@ "title":"Common Operations Supported by DLI System Policy", "uri":"dli_01_0441.html", "doc_type":"usermanual", - "p_code":"111", - "code":"117" + "p_code":"130", + "code":"135" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Other Common Operations", - "uri":"dli_01_0513.html", + "title":"Common DLI Management Operations", + "uri":"dli_01_0567.html", "doc_type":"usermanual", "p_code":"", - "code":"118" + "code":"136" }, { - "desc":"On the DLI management console, you can import data stored in OBS into DLI tables.To import OBS data to a DLI table, either choose Data Management > Databases and Tables i", + "desc":"To enhance the functions and performance of Spark and Flink jobs, you can create custom images by downloading the base images provided by DLI and adding dependencies (fil", "product_code":"dli", - "title":"Importing Data to a DLI Table", - "uri":"dli_01_0420.html", + "title":"Using a Custom Image to Enhance the Job Running Environment", + "uri":"dli_01_0494.html", "doc_type":"usermanual", - "p_code":"118", - "code":"119" + "p_code":"136", + "code":"137" }, { - "desc":"This section describes metrics reported by DLI to Cloud Eye as well as their namespaces and dimensions. You can use the management console or APIs provided by Cloud Eye t", + "desc":"DLI allows you to set variables that are frequently used during job development as global variables on the DLI management console. This avoids repeated definitions during", "product_code":"dli", - "title":"Viewing Monitoring Metrics", - "uri":"dli_01_0445.html", + "title":"Managing DLI Global Variables", + "uri":"dli_01_0476.html", "doc_type":"usermanual", - "p_code":"118", - "code":"120" + "p_code":"136", + "code":"138" }, { - "desc":"With CTS, you can record operations associated with DLI for later query, audit, and backtrack operations.", + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"DLI Operations That Can Be Recorded by CTS", - "uri":"dli_01_0318.html", + "title":"Managing Program Packages of Jar Jobs", + "uri":"dli_01_0366.html", "doc_type":"usermanual", - "p_code":"118", - "code":"121" + "p_code":"136", + "code":"139" + }, + { + "desc":"Before running DLI jobs, UDF JAR files or Jar job packages need to be uploaded to the cloud platform for unified management and maintenance.There are two ways to manage p", + "product_code":"dli", + "title":"Package Management Overview", + "uri":"dli_01_0407.html", + "doc_type":"usermanual", + "p_code":"139", + "code":"140" + }, + { + "desc":"DLI allows you to submit program packages in batches to the general-use queue for running.If you need to update a package, you can use the same package or file to upload ", + "product_code":"dli", + "title":"Creating a DLI Package", + "uri":"dli_01_0367.html", + "doc_type":"usermanual", + "p_code":"139", + "code":"141" + }, + { + "desc":"By configuring permissions, you can grant different package groups or packages to various users, ensuring that job efficiency remains unaffected and job performance is ma", + "product_code":"dli", + "title":"Configuring DLI Package Permissions", + "uri":"dli_01_0477.html", + "doc_type":"usermanual", + "p_code":"139", + "code":"142" + }, + { + "desc":"DLI allows you to change the owner of a package group or package.Log in to the DLI management console and choose Data Management > Package Management.On the Package Manag", + "product_code":"dli", + "title":"Changing the DLI Package Owner", + "uri":"dli_01_0478.html", + "doc_type":"usermanual", + "p_code":"139", + "code":"143" + }, + { + "desc":"Tags are key-value pairs that you can define to identify cloud resources. They assist you in categorizing and searching for cloud resources. A tag consists of a key and a", + "product_code":"dli", + "title":"Managing DLI Package Tags", + "uri":"dli_01_0369.html", + "doc_type":"usermanual", + "p_code":"139", + "code":"144" + }, + { + "desc":"DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar ", + "product_code":"dli", + "title":"DLI Built-in Dependencies", + "uri":"dli_01_0397.html", + "doc_type":"usermanual", + "p_code":"139", + "code":"145" }, { "desc":"A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.You can also request for an increased quota if your e", "product_code":"dli", - "title":"Quota Management", + "title":"Managing DLI Resource Quotas", "uri":"dli_01_0550.html", "doc_type":"usermanual", - "p_code":"118", - "code":"122" + "p_code":"136", + "code":"146" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -1104,412 +1320,412 @@ "uri":"dli_01_0539.html", "doc_type":"usermanual", "p_code":"", - "code":"123" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Flink Jobs", - "uri":"dli_03_0037.html", - "doc_type":"usermanual", - "p_code":"123", - "code":"124" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Usage", - "uri":"dli_03_0137.html", - "doc_type":"usermanual", - "p_code":"124", - "code":"125" - }, - { - "desc":"DLI Flink jobs support the following data formats:Avro, Avro_merge, BLOB, CSV, EMAIL, JSON, ORC, Parquet, and XML.DLI Flink jobs support data from the following data sour", - "product_code":"dli", - "title":"What Data Formats and Data Sources Are Supported by DLI Flink Jobs?", - "uri":"dli_03_0083.html", - "doc_type":"usermanual", - "p_code":"125", - "code":"126" - }, - { - "desc":"A sub-user can view queues but cannot view Flink jobs. You can authorize the sub-user using DLI or IAM.Authorization on DLILog in to the DLI console using a tenant accoun", - "product_code":"dli", - "title":"How Do I Authorize a Subuser to View Flink Jobs?", - "uri":"dli_03_0139.html", - "doc_type":"usermanual", - "p_code":"125", - "code":"127" - }, - { - "desc":"DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are r", - "product_code":"dli", - "title":"How Do I Set Auto Restart upon Exception for a Flink Job?", - "uri":"dli_03_0090.html", - "doc_type":"usermanual", - "p_code":"125", - "code":"128" - }, - { - "desc":"When you create a Flink SQL job or Flink Jar job, you can select Save Job Log on the job editing page to save job running logs to OBS.To set the OBS bucket for storing th", - "product_code":"dli", - "title":"How Do I Save Flink Job Logs?", - "uri":"dli_03_0099.html", - "doc_type":"usermanual", - "p_code":"125", - "code":"129" - }, - { - "desc":"DLI can output Flink job results to DIS. You can view the results in DIS. For details, see \"Obtaining Data from DIS\" in Data Ingestion Service User Guide.DLI can output F", - "product_code":"dli", - "title":"How Can I Check Flink Job Results?", - "uri":"dli_03_0043.html", - "doc_type":"usermanual", - "p_code":"125", - "code":"130" - }, - { - "desc":"Choose Job Management > Flink Jobs. In the Operation column of the target job, choose More > Permissions. When a new user is authorized, No such user. userName:xxxx. is d", - "product_code":"dli", - "title":"Why Is Error \"No such user. userName:xxxx.\" Reported on the Flink Job Management Page When I Grant Permission to a User?", - "uri":"dli_03_0160.html", - "doc_type":"usermanual", - "p_code":"125", - "code":"131" - }, - { - "desc":"Checkpoint was enabled when a Flink job is created, and the OBS bucket for storing checkpoints was specified. After a Flink job is manually stopped, no message is display", - "product_code":"dli", - "title":"How Do I Know Which Checkpoint the Flink Job I Stopped Will Be Restored to When I Start the Job Again?", - "uri":"dli_03_0180.html", - "doc_type":"usermanual", - "p_code":"125", - "code":"132" - }, - { - "desc":"When you set running parameters of a DLI Flink job, you can enable Alarm Generation upon Job Exception to receive alarms when the job runs abnormally or is in arrears.If ", - "product_code":"dli", - "title":"Why Is a Message Displayed Indicating That the SMN Topic Does Not Exist When I Use the SMN Topic in DLI?", - "uri":"dli_03_0036.html", - "doc_type":"usermanual", - "p_code":"125", - "code":"133" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Flink SQL", - "uri":"dli_03_0131.html", - "doc_type":"usermanual", - "p_code":"124", - "code":"134" - }, - { - "desc":"The consumption capability of a Flink SQL job depends on the data source transmission, queue size, and job parameter settings. The peak consumption is 10 Mbit/s.", - "product_code":"dli", - "title":"How Much Data Can Be Processed in a Day by a Flink SQL Job?", - "uri":"dli_03_0130.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"135" - }, - { - "desc":"The temp stream in Flink SQL is similar to a subquery. It is a logical stream used to simplify the SQL logic and does not generate data storage. Therefore, there is no ne", - "product_code":"dli", - "title":"Does Data in the Temporary Stream of Flink SQL Need to Be Cleared Periodically? How Do I Clear the Data?", - "uri":"dli_03_0061.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"136" - }, - { - "desc":"When you create a Flink SQL job and configure the parameters, you select an OBS bucket you have created. The system displays a message indicating that the OBS bucket is n", - "product_code":"dli", - "title":"Why Is a Message Displayed Indicating That the OBS Bucket Is Not Authorized When I Select an OBS Bucket for a Flink SQL Job?", - "uri":"dli_03_0138.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"137" - }, - { - "desc":"When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.In the following example, the day field is used as the partition fie", - "product_code":"dli", - "title":"How Do I Create an OBS Partitioned Table for a Flink SQL Job?", - "uri":"dli_03_0089.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"138" - }, - { - "desc":"In this example, the day field is used as the partition field with the parquet encoding format (only the parquet format is supported currently) to dump car_info data to O", - "product_code":"dli", - "title":"How Do I Dump Data to OBS and Create an OBS Partitioned Table?", - "uri":"dli_03_0075.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"139" - }, - { - "desc":"When I run the creation statement with an EL expression in the table name in a Flink SQL job, the following error message is displayed:DLI.0005: AnalysisException: t_user", - "product_code":"dli", - "title":"Why Is Error Message \"DLI.0005\" Displayed When I Use an EL Expression to Create a Table in a Flink SQL Job?", - "uri":"dli_03_0167.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"140" - }, - { - "desc":"After data is written to OBS through the Flink job output stream, data cannot be queried from the DLI table created in the OBS file path.For example, use the following Fl", - "product_code":"dli", - "title":"Why Is No Data Queried in the DLI Table Created Using the OBS File Path When Data Is Written to OBS by a Flink Job Output Stream?", - "uri":"dli_03_0168.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"141" - }, - { - "desc":"After a Flink SQL job is submitted on DLI, the job fails to be executed. The following error information is displayed in the job log:connect to DIS failed java.lang.Illeg", - "product_code":"dli", - "title":"Why Does a Flink SQL Job Fails to Be Executed, and Is \"connect to DIS failed java.lang.IllegalArgumentException: Access key cannot be null\" Displayed in the Log?", - "uri":"dli_03_0174.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"142" - }, - { - "desc":"Semantic verification for a Flink SQL job (reading DIS data) fails. The following information is displayed when the job fails:Get dis channel xxxinfo failed. error info: ", - "product_code":"dli", - "title":"Why Is Error \"Not authorized\" Reported When a Flink SQL Job Reads DIS Data?", - "uri":"dli_03_0176.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"143" - }, - { - "desc":"After a Flink SQL job consumed Kafka and sent data to the Elasticsearch cluster, the job was successfully executed, but no data is available.Possible causes are as follow", - "product_code":"dli", - "title":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster", - "uri":"dli_03_0232.html", - "doc_type":"usermanual", - "p_code":"134", - "code":"144" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Flink Jar Jobs", - "uri":"dli_03_0132.html", - "doc_type":"usermanual", - "p_code":"124", - "code":"145" - }, - { - "desc":"You can upload configuration files for custom jobs (Jar).Upload the configuration file to DLI through Package Management.In the Other Dependencies area of the Flink Jar j", - "product_code":"dli", - "title":"Can I Upload Configuration Files for Flink Jar Jobs?", - "uri":"dli_03_0044.html", - "doc_type":"usermanual", - "p_code":"145", - "code":"146" - }, - { - "desc":"The dependency of your Flink job conflicts with a built-in dependency of the DLI Flink platform. As a result, the job submission fails.Delete your JAR file that is the sa", - "product_code":"dli", - "title":"Why Does a Flink Jar Package Conflict Result in Submission Failure?", - "uri":"dli_03_0119.html", - "doc_type":"usermanual", - "p_code":"145", "code":"147" }, { - "desc":"When a Flink Jar job is submitted to access GaussDB(DWS), an error message is displayed indicating that the job fails to be started. The job log contains the following er", + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Why Does a Flink Jar Job Fail to Access GaussDB(DWS) and a Message Is Displayed Indicating Too Many Client Connections?", - "uri":"dli_03_0161.html", + "title":"DLI Basics", + "uri":"dli_03_0001.html", "doc_type":"usermanual", - "p_code":"145", + "p_code":"147", "code":"148" }, { - "desc":"An exception occurred when a Flink Jar job is running. The following error information is displayed in the job log:org.apache.flink.shaded.curator.org.apache.curator.Conn", + "desc":"Both DLI and MRS support Spark, but there are some differences in service mode, interface, application scenarios, and performance characteristics.The Spark component of D", "product_code":"dli", - "title":"Why Is Error Message \"Authentication failed\" Displayed During Flink Jar Job Running?", - "uri":"dli_03_0165.html", + "title":"What Are the Differences Between MRS Spark and DLI Spark?", + "uri":"dli_03_0115.html", "doc_type":"usermanual", - "p_code":"145", + "p_code":"148", "code":"149" }, { - "desc":"The storage path of the Flink Jar job checkpoints was set to an OBS bucket. The job failed to be submitted, and an error message indicating an invalid OBS bucket name was", + "desc":"Supported data formats:ParquetCSVORCJSONAvroOBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.DLI: The column-based Parq", "product_code":"dli", - "title":"Why Is Error Invalid OBS Bucket Name Reported After a Flink Job Submission Failed?", - "uri":"dli_03_0233.html", + "title":"Where Can Data Be Stored in DLI?", + "uri":"dli_03_0029.html", "doc_type":"usermanual", - "p_code":"145", + "p_code":"148", "code":"150" }, { - "desc":"Flink Job submission failed. The exception information is as follows:Flink JAR files conflicted. The submitted Flink JAR file conflicted with the HDFS JAR file of the DLI", + "desc":"DLI supports importing data from OBS buckets shared by IAM users under the same tenant, but not from OBS buckets shared by other tenants.This ensures data security and is", "product_code":"dli", - "title":"Why Does the Flink Submission Fail Due to Hadoop JAR File Conflict?", - "uri":"dli_03_0234.html", + "title":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?", + "uri":"dli_03_0129.html", "doc_type":"usermanual", - "p_code":"145", + "p_code":"148", "code":"151" }, { - "desc":"You can use Flink Jar to connect to Kafka with SASL SSL authentication enabled.", + "desc":"No, a global variable can only be used by the user who created it. Global variables can be used to simplify complex parameters. For example, long and difficult variables ", "product_code":"dli", - "title":"How Do I Connect a Flink jar Job to SASL_SSL?", - "uri":"dli_03_0266.html", + "title":"Can a Member Account Use Global Variables Created by Other Member Accounts?", + "uri":"dli_03_0263.html", "doc_type":"usermanual", - "p_code":"145", + "p_code":"148", "code":"152" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "desc":"No.The spark.acls.enable configuration item is not used in DLI. The Apache Spark command injection vulnerability (CVE-2022-33891) does not exist in DLI.This vulnerability", "product_code":"dli", - "title":"Performance Tuning", - "uri":"dli_03_0133.html", + "title":"Is DLI Affected by the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?", + "uri":"dli_03_0260.html", "doc_type":"usermanual", - "p_code":"124", + "p_code":"148", "code":"153" }, { - "desc":"Data Stacking in a Consumer GroupThe accumulated data of a consumer group can be calculated by the following formula: Total amount of data to be consumed by the consumer ", + "desc":"To manage a large number of DLI jobs, you can use the following methods:Manage jobs by group.Group tens of thousands of jobs by type and run each group on a queue.Group t", "product_code":"dli", - "title":"How Do I Optimize Performance of a Flink Job?", - "uri":"dli_03_0106.html", + "title":"How Do I Manage Jobs Running on DLI?", + "uri":"dli_03_0126.html", "doc_type":"usermanual", - "p_code":"153", + "p_code":"148", "code":"154" }, { - "desc":"Add the following SQL statements to the Flink job:", + "desc":"DLI does not support directly changing the field names of a table. However, you can solve this issue by migrating the table data using the following steps:Create a table:", "product_code":"dli", - "title":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?", - "uri":"dli_03_0048.html", + "title":"How Do I Change the Field Names of an Existing Table on DLI?", + "uri":"dli_03_0162.html", "doc_type":"usermanual", - "p_code":"153", + "p_code":"148", "code":"155" }, { - "desc":"The DLI Flink checkpoint/savepoint mechanism is complete and reliable. You can use this mechanism to prevent data loss when a job is manually restarted or restarted due t", + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"How Do I Prevent Data Loss After Flink Job Restart?", - "uri":"dli_03_0096.html", + "title":"DLI Elastic Resource Pools and Queues", + "uri":"dli_03_0049.html", "doc_type":"usermanual", - "p_code":"153", + "p_code":"147", "code":"156" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "desc":"In daily big data analysis work, it is important to allocate and manage compute resources properly to provide a good job execution environment.You can allocate resources ", "product_code":"dli", - "title":"O&M Guide", - "uri":"dli_03_0135.html", + "title":"How Can I Check the Actual and Used CUs for an Elastic Resource Pool as Well as the Required CUs for a Job?", + "uri":"dli_03_0276.html", "doc_type":"usermanual", - "p_code":"124", + "p_code":"156", "code":"157" }, { - "desc":"On the Flink job management page, hover the cursor on the status of the job that fails to be submitted to view the brief information about the failure.The possible causes", + "desc":"You need to check the large number of jobs in the Submitting and Running states on the queue.Use Cloud Eye to view jobs in different states on the queue. The procedure is", "product_code":"dli", - "title":"How Do I Locate a Flink Job Submission Error?", - "uri":"dli_03_0103.html", + "title":"How Do I Check for a Backlog of Jobs in the Current DLI Queue?", + "uri":"dli_03_0183.html", "doc_type":"usermanual", - "p_code":"157", + "p_code":"156", "code":"158" }, { - "desc":"On the Flink job management, click Edit in the Operation column of the target job. On the displayed page, check whether Save Job Log in the Running Parameters tab is enab", + "desc":"To check the running status of the DLI queue and determine whether to run more jobs on that queue, you need to check the queue load.Search for Cloud Eye on the console.In", "product_code":"dli", - "title":"How Do I Locate a Flink Job Running Error?", - "uri":"dli_03_0105.html", + "title":"How Do I View the Load of a DLI Queue?", + "uri":"dli_03_0095.html", "doc_type":"usermanual", - "p_code":"157", + "p_code":"156", "code":"159" }, { - "desc":"Flink's checkpointing is a fault tolerance and recovery mechanism. This mechanism ensures that real-time programs can self-recover in case of exceptions or machine issues", + "desc":"DLI allows you to subscribe to an SMN topic for failed jobs.Log in to the DLI console.In the navigation pane on the left, choose Queue Management.On the Queue Management ", "product_code":"dli", - "title":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?", - "uri":"dli_03_0136.html", + "title":"How Do I Monitor Job Exceptions on a DLI Queue?", + "uri":"dli_03_0098.html", "doc_type":"usermanual", - "p_code":"157", + "p_code":"156", "code":"160" }, { - "desc":"To rectify this fault, perform the following steps:Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statemen", + "desc":"Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earl", "product_code":"dli", - "title":"Why Does DIS Stream Not Exist During Job Semantic Check?", - "uri":"dli_03_0040.html", + "title":"How Do I Migrate an Old Version Spark Queue to a General-Purpose Queue?", + "uri":"dli_03_0065.html", "doc_type":"usermanual", - "p_code":"157", + "p_code":"156", "code":"161" }, { - "desc":"If the OBS bucket selected for a job is not authorized, perform the following steps:Select Enable Checkpointing or Save Job Log.Specify OBS Bucket.Select Authorize OBS.", + "desc":"After a SQL job was submitted to the default queue, the job runs abnormally. The job log reported that the execution timed out. The exception logs are as follows:[ERROR] ", "product_code":"dli", - "title":"Why Is the OBS Bucket Selected for Job Not Authorized?", - "uri":"dli_03_0045.html", + "title":"How Do I Do If I Encounter a Timeout Exception When Executing DLI SQL Statements on the default Queue?", + "uri":"dli_03_0171.html", "doc_type":"usermanual", - "p_code":"157", + "p_code":"156", "code":"162" }, { - "desc":"Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:If the submission fails, a submission log is genera", + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Why Are Logs Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?", - "uri":"dli_03_0064.html", + "title":"DLI Databases and Tables", + "uri":"dli_03_0211.html", "doc_type":"usermanual", - "p_code":"157", + "p_code":"147", "code":"163" }, { - "desc":"The Flink/Spark UI was displayed with incomplete information.When the queue is used to run a job, the system releases the cluster and takes about 10 minutes to create a n", + "desc":"A DLI table exists but cannot be queried on the DLI console.If a table exists but cannot be queried, there is a high probability that the current user does not have the p", "product_code":"dli", - "title":"Why Is Information Displayed on the FlinkUI/Spark UI Page Incomplete?", - "uri":"dli_03_0235.html", + "title":"Why Am I Unable to Query a Table on the DLI Console?", + "uri":"dli_03_0184.html", "doc_type":"usermanual", - "p_code":"157", + "p_code":"163", "code":"164" }, { - "desc":"JobManager and TaskManager heartbeats timed out. As a result, the Flink job is abnormal.Check whether the network is intermittently disconnected and whether the cluster l", + "desc":"When submitting a job to import data into a DLI table, if the compression rate of the Parquet/ORC file corresponding to the OBS table is high, exceeding 5 times the compr", "product_code":"dli", - "title":"Why Is the Flink Job Abnormal Due to Heartbeat Timeout Between JobManager and TaskManager?", - "uri":"dli_03_0236.html", + "title":"How Do I Do If the Compression Rate of an OBS Table Is High?", + "uri":"dli_03_0013.html", "doc_type":"usermanual", - "p_code":"157", + "p_code":"163", "code":"165" }, { - "desc":"Test address connectivity.If the network is unreachable, rectify the network connection first. Ensure that the network connection between the DLI queue and the external d", + "desc":"To avoid garbled characters caused by inconsistent character encoding, you are advised to unify the encoding format of your data source when executing jobs in DLI.DLI onl", "product_code":"dli", - "title":"Why Is Error \"Timeout expired while fetching topic metadata\" Repeatedly Reported in Flink JobManager Logs?", - "uri":"dli_03_0265.html", + "title":"How Do I Do If Inconsistent Character Encoding Leads to Garbled Characters?", + "uri":"dli_03_0009.html", "doc_type":"usermanual", - "p_code":"157", + "p_code":"163", "code":"166" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "desc":"User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table a", "product_code":"dli", - "title":"Problems Related to SQL Jobs", - "uri":"dli_03_0020.html", + "title":"Do I Need to to Regrant Permissions to Users and Projects After Deleting and Recreating a Table With the Same Name?", + "uri":"dli_03_0175.html", "doc_type":"usermanual", - "p_code":"123", + "p_code":"163", "code":"167" }, + { + "desc":"A CSV file is imported to a DLI partitioned table, but the imported file data does not contain the data in the partitioning column. The partitioning column needs to be sp", + "product_code":"dli", + "title":"How Do I Do If Files Imported Into a DLI Partitioned Table Lack Data for the Partition Columns, Causing Query Failures After the Import Is Completed?", + "uri":"dli_03_0177.html", + "doc_type":"usermanual", + "p_code":"163", + "code":"168" + }, + { + "desc":"When an OBS foreign table is created, a field in the specified OBS file contains a carriage return line feed (CRLF) character. As a result, the data is incorrect.The stat", + "product_code":"dli", + "title":"How Do I Fix Incorrect Data in an OBS Foreign Table Caused by Newline Characters in OBS File Fields?", + "uri":"dli_03_0181.html", + "doc_type":"usermanual", + "p_code":"163", + "code":"169" + }, + { + "desc":"The on clause was not added to the SQL statement for joining tables. As a result, the Cartesian product query occurs due to multi-table association, and the queue resourc", + "product_code":"dli", + "title":"How Do I Prevent a Cartesian Product Query and Resource Overload Due to Missing \"ON\" Conditions in Table Joins?", + "uri":"dli_03_0187.html", + "doc_type":"usermanual", + "p_code":"163", + "code":"170" + }, + { + "desc":"Partition data is manually uploaded to a partition of an OBS table. However, the data cannot be queried using DLI SQL editor.After manually adding partition data, you nee", + "product_code":"dli", + "title":"How Do I Do If I Can't Query Data After Manually Adding It to the Partition Directory of an OBS Table?", + "uri":"dli_03_0190.html", + "doc_type":"usermanual", + "p_code":"163", + "code":"171" + }, + { + "desc":"When using the INSERT OVERWRITE statement to overwrite data in a partitioned table, if you find that it overwrites all data instead of the expected partitioned data, it m", + "product_code":"dli", + "title":"Why Does the \"insert overwrite\" Operation Affect All Data in a Partitioned Table Instead of Just the Targeted Partition?", + "uri":"dli_03_0212.html", + "doc_type":"usermanual", + "p_code":"163", + "code":"172" + }, + { + "desc":"Spark does not have the datetime type and uses the TIMESTAMP type instead.You can use a function to convert data types.The following is an example.select cast(create_date", + "product_code":"dli", + "title":"Why Does the \"create_date\" Field in an RDS Table (Datetime Data Type) Appear as a Timestamp in DLI Queries?", + "uri":"dli_03_0214.html", + "doc_type":"usermanual", + "p_code":"163", + "code":"173" + }, + { + "desc":"Changing a table name immediately after executing a SQL job may result in an incorrect data size for the table.This is because DLI updates the metadata of the table when ", + "product_code":"dli", + "title":"How Do I Do If Renaming a Table After a SQL Job Causes Incorrect Data Size?", + "uri":"dli_03_0215.html", + "doc_type":"usermanual", + "p_code":"163", + "code":"174" + }, + { + "desc":"When DLI is used to insert data into an OBS temporary table, only part of data is imported.Possible causes are as follows:The amount of data read during job execution is ", + "product_code":"dli", + "title":"How Can I Resolve Data Inconsistencies When Importing Data from DLI to OBS?", + "uri":"dli_03_0231.html", + "doc_type":"usermanual", + "p_code":"163", + "code":"175" + }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Usage", - "uri":"dli_03_0216.html", + "title":"Enhanced Datasource Connections", + "uri":"dli_03_0022.html", "doc_type":"usermanual", - "p_code":"167", - "code":"168" + "p_code":"147", + "code":"176" + }, + { + "desc":"The outbound rule had been configured for the security group of the queue associated with the enhanced datasource connection. The datasource authentication used a passwor", + "product_code":"dli", + "title":"How Do I Resolve a Failure in Connecting DLI to GaussDB(DWS) Through an Enhanced Datasource Connection?", + "uri":"dli_03_0238.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"177" + }, + { + "desc":"A datasource connection is created and bound to a queue. The connectivity test fails and the following error information is displayed:failed to connect to specified addre", + "product_code":"dli", + "title":"How Do I Do If the Datasource Connection Is Successfully Created but the Network Connectivity Test Fails?", + "uri":"dli_03_0179.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"178" + }, + { + "desc":"Configuring the Connection Between a DLI Queue and a Data Source in a Private NetworkIf your DLI job needs to connect to a data source, for example, MRS, RDS, CSS, Kafka,", + "product_code":"dli", + "title":"How Do I Configure Network Connectivity Between a DLI Queue and a Data Source?", + "uri":"dli_03_0186.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"179" + }, + { + "desc":"The main reason for creating a VPC peering connection for DLI's enhanced datasource connection is to establish network connectivity between DLI and data sources in differ", + "product_code":"dli", + "title":"Why Is Creating a VPC Peering Connection Necessary for Enhanced Datasource Connections in DLI?", + "uri":"dli_03_0128.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"180" + }, + { + "desc":"The possible causes and solutions are as follows:If you have created a queue, do not bind it to a datasource connection immediately. Wait for 5 to 10 minutes. After the c", + "product_code":"dli", + "title":"How Do I Do If Creating a Datasource Connection in DLI Gets Stuck in the \"Creating\" State When Binding It to a Queue?", + "uri":"dli_03_0257.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"181" + }, + { + "desc":"The network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.Example: When you create an RDS d", + "product_code":"dli", + "title":"How Do I Resolve the \"communication link failure\" Error When Using a Newly Created Datasource Connection That Appears to Be Activated?", + "uri":"dli_03_0047.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"182" + }, + { + "desc":"If you have not added the cluster host information to the datasource connection, it can lead to KRB authentication failure, resulting in a connection timeout. In this cas", + "product_code":"dli", + "title":"How Do I Troubleshoot a Connection Timeout Issue That Isn't Recorded in Logs When Accessing MRS HBase Through a Datasource Connection?", + "uri":"dli_03_0080.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"183" + }, + { + "desc":"A datasource RDS table was created in the DataArts Studio, and the insert overwrite statement was executed to write data into RDS. DLI.0999: BatchUpdateException: Incorre", + "product_code":"dli", + "title":"How Do I Do If I Encounter the \"Incorrect string value\" Error When Executing insert overwrite on a Datasource RDS Table?", + "uri":"dli_03_0239.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"184" + }, + { + "desc":"The system failed to create a datasource RDS table, and null pointer error was reported.The following table creation statement was used:The RDS database is in a PostGre c", + "product_code":"dli", + "title":"How Do I Resolve the Null Pointer Error When Creating an RDS Datasource Table?", + "uri":"dli_03_0250.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"185" + }, + { + "desc":"The system failed to execute insert overwrite on the datasource GaussDB(DWS) table, and org.postgresql.util.PSQLException: ERROR: tuple concurrently updated was displayed", + "product_code":"dli", + "title":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed When the System Executes insert overwrite on a Datasource GaussDB(DWS) Table", + "uri":"dli_03_0251.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"186" + }, + { + "desc":"A datasource table was used to import data to a CloudTable HBase table. This HBase table contains a column family and a rowkey for 100 million simulating data records. Th", + "product_code":"dli", + "title":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Datasource Table", + "uri":"dli_03_0252.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"187" + }, + { + "desc":"A table was created on GaussDB(DWS) and then a datasource connection was created on DLI to read and write data. An error message was displayed during data writing, indica", + "product_code":"dli", + "title":"How Do I Do If A Null Value Is Written Into a Non-Null Field When Using a DLI Datasource Connection to Connect to a GaussDB(DWS) Table?", + "uri":"dli_03_0253.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"188" + }, + { + "desc":"A datasource GaussDB(DWS) table and the datasource connection were created in DLI, and the schema of the source table in GaussDB(DWS) were updated. During the job executi", + "product_code":"dli", + "title":"How Do I Do If an Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated?", + "uri":"dli_03_0254.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"189" + }, + { + "desc":"When creating an associated RDS table in DLI, if the RDS table contains an auto-increment primary key or other auto-populated fields, you can take the following measures ", + "product_code":"dli", + "title":"How Do I Insert Data into an RDS Table with an Auto-Increment Primary Key Using DLI?", + "uri":"dli_03_0028.html", + "doc_type":"usermanual", + "p_code":"176", + "code":"190" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"SQL Jobs", + "uri":"dli_03_0020.html", + "doc_type":"usermanual", + "p_code":"147", + "code":"191" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"SQL Job Development", + "uri":"dli_03_0204.html", + "doc_type":"usermanual", + "p_code":"191", + "code":"192" }, { "desc":"A temporary table is used to store intermediate results. When a transaction or session ends, the data in the temporary table can be automatically deleted. For example, in", @@ -1517,26 +1733,17 @@ "title":"SQL Jobs", "uri":"dli_03_0200.html", "doc_type":"usermanual", - "p_code":"168", - "code":"169" + "p_code":"192", + "code":"193" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Job Development", - "uri":"dli_03_0204.html", - "doc_type":"usermanual", - "p_code":"167", - "code":"170" - }, - { - "desc":"If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.Set t", + "desc":"If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.You a", "product_code":"dli", "title":"How Do I Merge Small Files?", "uri":"dli_03_0086.html", "doc_type":"usermanual", - "p_code":"170", - "code":"171" + "p_code":"192", + "code":"194" }, { "desc":"When creating an OBS table, you must specify a table path in the database. The path format is as follows: obs://xxx/database name/table name.If the specified path is akdc", @@ -1544,26 +1751,17 @@ "title":"How Do I Specify an OBS Path When Creating an OBS Table?", "uri":"dli_03_0092.html", "doc_type":"usermanual", - "p_code":"170", - "code":"172" + "p_code":"192", + "code":"195" }, { - "desc":"DLI allows you to associate JSON data in an OBS bucket to create tables in asynchronous mode.The statement for creating the table is as follows:", + "desc":"To associate JSON data nested in an OBS bucket, you can create a table in asynchronous mode.The following is an example of a table creation statement that shows how to us", "product_code":"dli", "title":"How Do I Create a Table Using JSON Data in an OBS Bucket?", "uri":"dli_03_0108.html", "doc_type":"usermanual", - "p_code":"170", - "code":"173" - }, - { - "desc":"You can use the where condition statement in the select statement to filter data. For example:", - "product_code":"dli", - "title":"How Do I Set Local Variables in SQL Statements?", - "uri":"dli_03_0087.html", - "doc_type":"usermanual", - "p_code":"170", - "code":"174" + "p_code":"192", + "code":"196" }, { "desc":"The correct method for using the count function to perform aggregation is as follows:OrIf an incorrect method is used, an error will be reported.", @@ -1571,161 +1769,26 @@ "title":"How Can I Use the count Function to Perform Aggregation?", "uri":"dli_03_0069.html", "doc_type":"usermanual", - "p_code":"170", - "code":"175" + "p_code":"192", + "code":"197" }, { "desc":"You can use the cross-region replication function of OBS. The procedure is as follows:Export the DLI table data in region 1 to the user-defined OBS bucket.Use the OBS cro", "product_code":"dli", - "title":"How Do I Synchronize DLI Table Data from One Region to Another?", + "title":"How Do I Synchronize DLI Table Data Across Regions?", "uri":"dli_03_0072.html", "doc_type":"usermanual", - "p_code":"170", - "code":"176" + "p_code":"192", + "code":"198" }, { - "desc":"Currently, DLI does not allow you to insert table data into specific fields. To insert table data, you must insert data of all table fields at a time.", + "desc":"If you need to insert data into a table but only want to specify certain fields, you can use the INSERT INTO statement combined with the SELECT clause.However, DLI curren", "product_code":"dli", "title":"How Do I Insert Table Data into Specific Fields of a Table Using a SQL Job?", "uri":"dli_03_0191.html", "doc_type":"usermanual", - "p_code":"170", - "code":"177" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Job O&M Errors", - "uri":"dli_03_0206.html", - "doc_type":"usermanual", - "p_code":"167", - "code":"178" - }, - { - "desc":"Create an OBS directory with a unique name. Alternatively, you can manually delete the existing OBS directory and submit the job again. However, exercise caution when del", - "product_code":"dli", - "title":"Why Is Error \"path obs://xxx already exists\" Reported When Data Is Exported to OBS?", - "uri":"dli_03_0014.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"179" - }, - { - "desc":"This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.For example, ta", - "product_code":"dli", - "title":"Why Is Error \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Displayed When Two Tables Are Joined?", - "uri":"dli_03_0066.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"180" - }, - { - "desc":"Check if your account is in arrears and top it up if necessary.If the same error message persists after the top-up, log out of your account and log back in.", - "product_code":"dli", - "title":"Why Is Error \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget.\" Reported when a SQL Statement Is Executed?", - "uri":"dli_03_0071.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"181" - }, - { - "desc":"Cause AnalysisWhen you query the partitioned table XX.YYY, the partition column is not specified in the search criteria.A partitioned table can be queried only when the q", - "product_code":"dli", - "title":"Why Is Error \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" Reported When a Query Statement Is Executed?", - "uri":"dli_03_0145.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"182" - }, - { - "desc":"The following error message is displayed when the LOAD DATA command is executed by a Spark SQL job to import data to a DLI table:In some cases ,the following error messag", - "product_code":"dli", - "title":"Why Is Error \"IllegalArgumentException: Buffer size too small. size\" Reported When Data Is Loaded to an OBS Foreign Table?", - "uri":"dli_03_0169.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"183" - }, - { - "desc":"An error is reported during SQL job execution:Please contact DLI service. DLI.0002: FileNotFoundException: getFileStatus on obs://xxx: status [404]Check whether there is ", - "product_code":"dli", - "title":"Why Is Error \"DLI.0002 FileNotFoundException\" Reported During SQL Job Running?", - "uri":"dli_03_0189.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"184" - }, - { - "desc":"Currently, DLI supports the Hive syntax for creating tables of the TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET file types. If the file format specified for cre", - "product_code":"dli", - "title":"Why Is a Schema Parsing Error Reported When I Create a Hive Table Using CTAS?", - "uri":"dli_03_0046.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"185" - }, - { - "desc":"When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:DLI.0999: RuntimeException: o", - "product_code":"dli", - "title":"Why Is Error \"org.apache.hadoop.fs.obs.OBSIOException\" Reported When I Run DLI SQL Scripts on DataArts Studio?", - "uri":"dli_03_0173.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"186" - }, - { - "desc":"After the migration job is submitted, the following error information is displayed in the log:org.apache.sqoop.common.SqoopException:UQUERY_CONNECTOR_0001:Invoke DLI serv", - "product_code":"dli", - "title":"Why Is Error \"UQUERY_CONNECTOR_0001:Invoke DLI service api failed\" Reported in the Job Log When I Use CDM to Migrate Data to DLI?", - "uri":"dli_03_0172.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"187" - }, - { - "desc":"Error message \"File not Found\" is displayed when a SQL job is accessed.Generally, the file cannot be found due to a read/write conflict. Check whether a job is overwritin", - "product_code":"dli", - "title":"Why Is Error \"File not Found\" Reported When I Access a SQL Job?", - "uri":"dli_03_0207.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"188" - }, - { - "desc":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.Check the OBS bucket written in the AccessControlException to confirm if your ", - "product_code":"dli", - "title":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?", - "uri":"dli_03_0208.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"189" - }, - { - "desc":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Yo", - "product_code":"dli", - "title":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" Reported When I Access a SQL Job?", - "uri":"dli_03_0209.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"190" - }, - { - "desc":"Error message \"The current account does not have permission to perform this operation,the current account was restricted.\" is reported during SQL statement execution.Chec", - "product_code":"dli", - "title":"Why Is Error \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget\" Reported During SQL Statement Execution? Restricted for no budget.", - "uri":"dli_03_0210.html", - "doc_type":"usermanual", - "p_code":"178", - "code":"191" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"O&M Guide", - "uri":"dli_03_0211.html", - "doc_type":"usermanual", - "p_code":"167", - "code":"192" + "p_code":"192", + "code":"199" }, { "desc":"If the job runs slowly, perform the following steps to find the causes and rectify the fault:Check whether the problem is caused by FullGC.Log in to the DLI console. In t", @@ -1734,7 +1797,7 @@ "uri":"dli_03_0196.html", "doc_type":"usermanual", "p_code":"192", - "code":"193" + "code":"200" }, { "desc":"You can view SQL job logs for routine O&M.Obtain the ID of the DLI job executed on the DataArts Studio console.Job IDOn the DLI console, choose Job Management > SQL Jobs.", @@ -1743,16 +1806,16 @@ "uri":"dli_03_0091.html", "doc_type":"usermanual", "p_code":"192", - "code":"194" + "code":"201" }, { "desc":"You can view the job execution records when a job is running.Log in to the DLI management console.In the navigation pane on the left, choose Job Management > SQL Jobs.Ent", "product_code":"dli", - "title":"How Do I View SQL Execution Records?", + "title":"How Do I View SQL Execution Records in DLI?", "uri":"dli_03_0116.html", "doc_type":"usermanual", "p_code":"192", - "code":"195" + "code":"202" }, { "desc":"Data skew is a common issue during the execution of SQL jobs. When data is unevenly distributed, some compute nodes process significantly more data than others, which can", @@ -1761,61 +1824,7 @@ "uri":"dli_03_0093.html", "doc_type":"usermanual", "p_code":"192", - "code":"196" - }, - { - "desc":"A DLI table exists but cannot be queried on the DLI console.If a table exists but cannot be queried, there is a high probability that the current user does not have the p", - "product_code":"dli", - "title":"What Can I Do If a Table Cannot Be Queried on the DLI Console?", - "uri":"dli_03_0184.html", - "doc_type":"usermanual", - "p_code":"192", - "code":"197" - }, - { - "desc":"A high compression ratio of OBS tables in the Parquet or ORC format (for example, a compression ratio of 5 or higher compared with text compression) will lead to large da", - "product_code":"dli", - "title":"The Compression Ratio of OBS Tables Is Too High", - "uri":"dli_03_0013.html", - "doc_type":"usermanual", - "p_code":"192", - "code":"198" - }, - { - "desc":"DLI supports only UTF-8-encoded texts. Ensure that data is encoded using UTF-8 during table creation and import.", - "product_code":"dli", - "title":"How Can I Avoid Garbled Characters Caused by Inconsistent Character Codes?", - "uri":"dli_03_0009.html", - "doc_type":"usermanual", - "p_code":"192", - "code":"199" - }, - { - "desc":"User A created the testTable table in a database through a SQL job and granted user B the permission to insert and delete table data. User A deleted the testTable table a", - "product_code":"dli", - "title":"Do I Need to Grant Table Permissions to a User and Project After I Delete a Table and Create One with the Same Name?", - "uri":"dli_03_0175.html", - "doc_type":"usermanual", - "p_code":"192", - "code":"200" - }, - { - "desc":"A CSV file is imported to a DLI partitioned table, but the imported file data does not contain the data in the partitioning column. The partitioning column needs to be sp", - "product_code":"dli", - "title":"Why Can't I Query Table Data After Data Is Imported to a DLI Partitioned Table Because the File to Be Imported Does Not Contain Data in the Partitioning Column?", - "uri":"dli_03_0177.html", - "doc_type":"usermanual", - "p_code":"192", - "code":"201" - }, - { - "desc":"When an OBS foreign table is created, a field in the specified OBS file contains a carriage return line feed (CRLF) character. As a result, the data is incorrect.The stat", - "product_code":"dli", - "title":"How Do I Fix the Data Error Caused by CRLF Characters in a Field of the OBS File Used to Create an External OBS Table?", - "uri":"dli_03_0181.html", - "doc_type":"usermanual", - "p_code":"192", - "code":"202" + "code":"203" }, { "desc":"A SQL job contains join operations. After the job is submitted, it is stuck in the Running state and no result is returned.When a Spark SQL job has join operations on sma", @@ -1824,35 +1833,8 @@ "uri":"dli_03_0182.html", "doc_type":"usermanual", "p_code":"192", - "code":"203" - }, - { - "desc":"The on clause was not added to the SQL statement for joining tables. As a result, the Cartesian product query occurs due to multi-table association, and the queue resourc", - "product_code":"dli", - "title":"The on Clause Is Not Added When Tables Are Joined. Cartesian Product Query Causes High Resource Usage of the Queue, and the Job Fails to Be Executed", - "uri":"dli_03_0187.html", - "doc_type":"usermanual", - "p_code":"192", "code":"204" }, - { - "desc":"Partition data is manually uploaded to a partition of an OBS table. However, the data cannot be queried using DLI SQL editor.After manually adding partition data, you nee", - "product_code":"dli", - "title":"Why Can't I Query Data After I Manually Add Data to the Partition Directory of an OBS Table?", - "uri":"dli_03_0190.html", - "doc_type":"usermanual", - "p_code":"192", - "code":"205" - }, - { - "desc":"To dynamically overwrite the specified partitioned data in the DataSource table, set dli.sql.dynamicPartitionOverwrite.enabled to true and then run the insert overwrite s", - "product_code":"dli", - "title":"Why Is All Data Overwritten When insert overwrite Is Used to Overwrite Partitioned Table?", - "uri":"dli_03_0212.html", - "doc_type":"usermanual", - "p_code":"192", - "code":"206" - }, { "desc":"The possible causes and solutions are as follows:After you purchase a DLI queue and submit a SQL job for the first time, wait for 5 to 10 minutes. After the cluster is st", "product_code":"dli", @@ -1860,52 +1842,439 @@ "uri":"dli_03_0213.html", "doc_type":"usermanual", "p_code":"192", + "code":"205" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"SQL Job O&M", + "uri":"dli_03_0206.html", + "doc_type":"usermanual", + "p_code":"191", + "code":"206" + }, + { + "desc":"This message indicates that you are exporting data to an existing OBS path.Solution:Create an OBS directory.You can create an OBS directory that does not exist to store t", + "product_code":"dli", + "title":"Why Is Error \"path obs://xxx already exists\" Reported When Data Is Exported to OBS?", + "uri":"dli_03_0014.html", + "doc_type":"usermanual", + "p_code":"206", "code":"207" }, { - "desc":"Spark does not have the datetime type and uses the TIMESTAMP type instead.You can use a function to convert data types.The following is an example.select cast(create_date", + "desc":"This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.For example, ta", "product_code":"dli", - "title":"Why Is the create_date Field in the RDS Table Is a Timestamp in the DLI query result?", - "uri":"dli_03_0214.html", + "title":"Why Is Error \"SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;\" Displayed When Two Tables Are Joined?", + "uri":"dli_03_0066.html", "doc_type":"usermanual", - "p_code":"192", + "p_code":"206", "code":"208" }, { - "desc":"If the table name is changed immediately after SQL statements are executed, the data size of the table may be incorrect.If you need to change the table name, change it 5 ", + "desc":"This message indicates that your operation is restricted because your account is in arrears or there is insufficient balance in your account.Solution:Check the account st", "product_code":"dli", - "title":"What Can I Do If datasize Cannot Be Changed After the Table Name Is Changed in a Finished SQL Job?", - "uri":"dli_03_0215.html", + "title":"Why Is Error \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget.\" Reported when a SQL Statement Is Executed?", + "uri":"dli_03_0071.html", "doc_type":"usermanual", - "p_code":"192", + "p_code":"206", "code":"209" }, { - "desc":"When DLI is used to insert data into an OBS temporary table, only part of data is imported.Possible causes are as follows:The amount of data read during job execution is ", + "desc":"When you query the partitioned table XX.YYY, the partition column is not specified in the search criteria.A partitioned table can be queried only when the query condition", "product_code":"dli", - "title":"Why Is the Data Volume Changes When Data Is Imported from DLI to OBS?", - "uri":"dli_03_0231.html", + "title":"Why Is Error \"There should be at least one partition pruning predicate on partitioned table XX.YYY\" Reported When a Query Statement Is Executed?", + "uri":"dli_03_0145.html", "doc_type":"usermanual", - "p_code":"192", + "p_code":"206", "code":"210" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "desc":"The following error message is displayed when the LOAD DATA command is executed by a Spark SQL job to import data to a DLI table:In some cases ,the following error messag", "product_code":"dli", - "title":"Problems Related to Spark Jobs", - "uri":"dli_03_0021.html", + "title":"Why Is Error \"IllegalArgumentException: Buffer size too small. size\" Reported When Data Is Loaded to an OBS Foreign Table?", + "uri":"dli_03_0169.html", "doc_type":"usermanual", - "p_code":"123", + "p_code":"206", "code":"211" }, + { + "desc":"An error is reported during SQL job execution:Please contact DLI service. DLI.0002: FileNotFoundException: getFileStatus on obs://xxx: status [404]Check whether there is ", + "product_code":"dli", + "title":"Why Is Error \"DLI.0002 FileNotFoundException\" Reported During SQL Job Running?", + "uri":"dli_03_0189.html", + "doc_type":"usermanual", + "p_code":"206", + "code":"212" + }, + { + "desc":"Currently, DLI supports the creation of TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET tables using the Hive syntax.If you create a table using CTAS statements, s", + "product_code":"dli", + "title":"Why Is a Schema Parsing Error Reported When I Create a Hive Table Using CTAS?", + "uri":"dli_03_0046.html", + "doc_type":"usermanual", + "p_code":"206", + "code":"213" + }, + { + "desc":"When you run a DLI SQL script on DataArts Studio, the log shows that the statements fail to be executed. The error information is as follows:DLI.0999: RuntimeException: o", + "product_code":"dli", + "title":"Why Is Error \"org.apache.hadoop.fs.obs.OBSIOException\" Reported When I Run DLI SQL Scripts on DataArts Studio?", + "uri":"dli_03_0173.html", + "doc_type":"usermanual", + "p_code":"206", + "code":"214" + }, + { + "desc":"Error message \"File not Found\" is displayed when a SQL job is accessed.The system may not be able to locate the specified file path or file due to an incorrect file path ", + "product_code":"dli", + "title":"Why Is Error \"File not Found\" Reported When I Access a SQL Job?", + "uri":"dli_03_0207.html", + "doc_type":"usermanual", + "p_code":"206", + "code":"215" + }, + { + "desc":"Error message \"DLI.0003: AccessControlException XXX\" is reported when a SQL job is accessed.Check the permissions of the OBS bucket to ensure that the account has access ", + "product_code":"dli", + "title":"Why Is Error \"DLI.0003: AccessControlException XXX\" Reported When I Access a SQL Job?", + "uri":"dli_03_0208.html", + "doc_type":"usermanual", + "p_code":"206", + "code":"216" + }, + { + "desc":"Error message \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" is reported when a SQL job is Accessed.Ch", + "product_code":"dli", + "title":"Why Is Error \"DLI.0001: org.apache.hadoop.security.AccessControlException: verifyBucketExists on {{bucket name}}: status [403]\" Reported When I Access a SQL Job?", + "uri":"dli_03_0209.html", + "doc_type":"usermanual", + "p_code":"206", + "code":"217" + }, + { + "desc":"Error message \"The current account does not have permission to perform this operation,the current account was restricted.\" is reported during SQL statement execution.Chec", + "product_code":"dli", + "title":"Why Am I Seeing the Error Message \"The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget\" When Executing a SQL Statement?", + "uri":"dli_03_0210.html", + "doc_type":"usermanual", + "p_code":"206", + "code":"218" + }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Usage", - "uri":"dli_03_0163.html", + "title":"Flink Jobs", + "uri":"dli_03_0037.html", "doc_type":"usermanual", - "p_code":"211", - "code":"212" + "p_code":"147", + "code":"219" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Flink Job Consulting", + "uri":"dli_03_0137.html", + "doc_type":"usermanual", + "p_code":"219", + "code":"220" + }, + { + "desc":"A sub-user can view queues but cannot view Flink jobs. You can authorize the sub-user using DLI or IAM.Authorization on DLILog in to the DLI console using a tenant accoun", + "product_code":"dli", + "title":"How Do I Authorize a Subuser to View Flink Jobs?", + "uri":"dli_03_0139.html", + "doc_type":"usermanual", + "p_code":"220", + "code":"221" + }, + { + "desc":"DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are r", + "product_code":"dli", + "title":"How Do I Configure Auto Restart upon Exception for a Flink Job?", + "uri":"dli_03_0090.html", + "doc_type":"usermanual", + "p_code":"220", + "code":"222" + }, + { + "desc":"When you create a Flink SQL job or Flink Jar job, you can select Save Job Log on the job editing page to save job running logs to OBS.To set the OBS bucket for storing th", + "product_code":"dli", + "title":"How Do I Save Logs for Flink Jobs?", + "uri":"dli_03_0099.html", + "doc_type":"usermanual", + "p_code":"220", + "code":"223" + }, + { + "desc":"Choose Job Management > Flink Jobs. In the Operation column of the target job, choose More > Permissions. When a new user is authorized, No such user. userName:xxxx. is d", + "product_code":"dli", + "title":"Why Is Error \"No such user. userName:xxxx.\" Reported on the Flink Job Management Page When I Grant Permission to a User?", + "uri":"dli_03_0160.html", + "doc_type":"usermanual", + "p_code":"220", + "code":"224" + }, + { + "desc":"Checkpoint was enabled when a Flink job is created, and the OBS bucket for storing checkpoints was specified. I am not sure how to restore a Flink job from a specific che", + "product_code":"dli", + "title":"How Do I Restore a Flink Job from a Specific Checkpoint After Manually Stopping the Job?", + "uri":"dli_03_0180.html", + "doc_type":"usermanual", + "p_code":"220", + "code":"225" + }, + { + "desc":"When you set running parameters of a DLI Flink job, you can enable Alarm Generation upon Job Exception to receive alarms when the job runs abnormally or is in arrears.If ", + "product_code":"dli", + "title":"Why Is a Message Displayed Indicating That the SMN Topic Does Not Exist When I Use the SMN Topic in DLI?", + "uri":"dli_03_0036.html", + "doc_type":"usermanual", + "p_code":"220", + "code":"226" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Flink SQL Jobs", + "uri":"dli_03_0131.html", + "doc_type":"usermanual", + "p_code":"219", + "code":"227" + }, + { + "desc":"When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.In the following example, the day field is used as the partition fie", + "product_code":"dli", + "title":"How Do I Map an OBS Table to a DLI Partitioned Table?", + "uri":"dli_03_0089.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"228" + }, + { + "desc":"You used Flink 1.10 to run a Flink Opensource SQL job. You set the number of Kafka partitions for the job a small value at the beginning and need to increase the number n", + "product_code":"dli", + "title":"How Do I Change the Number of Kafka Partitions in a Flink SQL Job Without Stopping It?", + "uri":"dli_03_0120.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"229" + }, + { + "desc":"When I run the creation statement with an EL expression in the table name in a Flink SQL job, the following error message is displayed:DLI.0005: AnalysisException: t_user", + "product_code":"dli", + "title":"How Do I Fix the DLI.0005 Error When Using EL Expressions to Create a Table in a Flink SQL Job?", + "uri":"dli_03_0167.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"230" + }, + { + "desc":"After data is written to OBS through the Flink job output stream, data cannot be queried from the DLI table created in the OBS file path.For example, use the following Fl", + "product_code":"dli", + "title":"Why Is No Data Queried in the DLI Table Created Using the OBS File Path When Data Is Written to OBS by a Flink Job Output Stream?", + "uri":"dli_03_0168.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"231" + }, + { + "desc":"After a Flink SQL job is submitted on DLI, the job fails to be executed. The following error information is displayed in the job log:connect to DIS failed java.lang.Illeg", + "product_code":"dli", + "title":"Why Does a Flink SQL Job Fails to Be Executed, and Is \"connect to DIS failed java.lang.IllegalArgumentException: Access key cannot be null\" Displayed in the Log?", + "uri":"dli_03_0174.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"232" + }, + { + "desc":"After a Flink SQL job consumed Kafka and sent data to the Elasticsearch cluster, the job was successfully executed, but no data is available.Possible causes are as follow", + "product_code":"dli", + "title":"Data Writing Fails After a Flink SQL Job Consumed Kafka and Sank Data to the Elasticsearch Cluster", + "uri":"dli_03_0232.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"233" + }, + { + "desc":"You used Flink 1.10 to run a Flink Opensource SQL job. The job failed after the following error was reported when Flink Sink wrote data to Kafka.The CPU usage is too high", + "product_code":"dli", + "title":"How Do I Configure Connection Retries for Kafka Sink If it is Disconnected?", + "uri":"dli_03_0121.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"234" + }, + { + "desc":"In a Flink job, you can use CREATE statements to define source and sink tables, and specify their connector types and related attributes.If you need to write data to diff", + "product_code":"dli", + "title":"How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?", + "uri":"dli_03_0048.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"235" + }, + { + "desc":"To rectify this fault, perform the following steps:Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statemen", + "product_code":"dli", + "title":"Why Does DIS Stream Not Exist During Job Semantic Check?", + "uri":"dli_03_0040.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"236" + }, + { + "desc":"If the Flink JobManager prompts \"Timeout expired while fetching topic metadata\", it means that the Flink job timed out while trying to fetch metadata for the Kafka topic.", + "product_code":"dli", + "title":"Why Is Error \"Timeout expired while fetching topic metadata\" Repeatedly Reported in Flink JobManager Logs?", + "uri":"dli_03_0265.html", + "doc_type":"usermanual", + "p_code":"227", + "code":"237" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Flink Jar Jobs", + "uri":"dli_03_0132.html", + "doc_type":"usermanual", + "p_code":"219", + "code":"238" + }, + { + "desc":"You can upload configuration files for custom jobs (Jar).Upload the configuration file to DLI through Package Management.In the Other Dependencies area of the Flink Jar j", + "product_code":"dli", + "title":"Can I Upload Configuration Files for Flink Jar Jobs?", + "uri":"dli_03_0044.html", + "doc_type":"usermanual", + "p_code":"238", + "code":"239" + }, + { + "desc":"The dependency of your Flink job conflicts with a built-in dependency of the DLI Flink platform. As a result, the job submission fails.Check whether there are conflicting", + "product_code":"dli", + "title":"Why Does a Flink Jar Package Conflict Result in Job Submission Failure?", + "uri":"dli_03_0119.html", + "doc_type":"usermanual", + "p_code":"238", + "code":"240" + }, + { + "desc":"When a Flink Jar job is submitted to access GaussDB(DWS), an error message is displayed indicating that the job fails to be started. The job log contains the following er", + "product_code":"dli", + "title":"Why Does a Flink Jar Job Fail to Access GaussDB(DWS) and a Message Is Displayed Indicating Too Many Client Connections?", + "uri":"dli_03_0161.html", + "doc_type":"usermanual", + "p_code":"238", + "code":"241" + }, + { + "desc":"The storage path of the Flink Jar job checkpoints was set to an OBS bucket. The job failed to be submitted, and an error message indicating an invalid OBS bucket name was", + "product_code":"dli", + "title":"Why Is Error Invalid OBS Bucket Name Reported After a Flink Job Submission Failed?", + "uri":"dli_03_0233.html", + "doc_type":"usermanual", + "p_code":"238", + "code":"242" + }, + { + "desc":"Flink Job submission failed. The exception information is as follows:Flink JAR files conflicted. The submitted Flink JAR file conflicted with the HDFS JAR file of the DLI", + "product_code":"dli", + "title":"Why Does the Flink Submission Fail Due to Hadoop JAR File Conflict?", + "uri":"dli_03_0234.html", + "doc_type":"usermanual", + "p_code":"238", + "code":"243" + }, + { + "desc":"On the Flink job management page, hover the cursor on the status of the job that fails to be submitted to view the brief information about the failure.The possible causes", + "product_code":"dli", + "title":"How Do I Locate a Flink Job Submission Error?", + "uri":"dli_03_0103.html", + "doc_type":"usermanual", + "p_code":"238", + "code":"244" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Flink Job Performance Tuning", + "uri":"dli_03_0133.html", + "doc_type":"usermanual", + "p_code":"219", + "code":"245" + }, + { + "desc":"Data Stacking in a Consumer GroupThe accumulated data of a consumer group can be calculated by the following formula: Total amount of data to be consumed by the consumer ", + "product_code":"dli", + "title":"Flink Job Performance Tuning", + "uri":"dli_03_0106.html", + "doc_type":"usermanual", + "p_code":"245", + "code":"246" + }, + { + "desc":"The DLI Flink checkpoint/savepoint mechanism is complete and reliable. You can use this mechanism to prevent data loss when a job is manually restarted or restarted due t", + "product_code":"dli", + "title":"How Do I Prevent Data Loss After Flink Job Restart?", + "uri":"dli_03_0096.html", + "doc_type":"usermanual", + "p_code":"245", + "code":"247" + }, + { + "desc":"On the Flink job management, click Edit in the Operation column of the target job. On the displayed page, check whether Save Job Log in the Running Parameters tab is enab", + "product_code":"dli", + "title":"How Do I Locate a Flink Job Running Error?", + "uri":"dli_03_0105.html", + "doc_type":"usermanual", + "p_code":"245", + "code":"248" + }, + { + "desc":"Flink's checkpointing is a fault tolerance and recovery mechanism. This mechanism ensures that real-time programs can self-recover in case of exceptions or machine issues", + "product_code":"dli", + "title":"How Can I Check if a Flink Job Can Be Restored From a Checkpoint After Restarting It?", + "uri":"dli_03_0136.html", + "doc_type":"usermanual", + "p_code":"245", + "code":"249" + }, + { + "desc":"Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:If the submission fails, a submission log is genera", + "product_code":"dli", + "title":"Why Are Logs Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?", + "uri":"dli_03_0064.html", + "doc_type":"usermanual", + "p_code":"245", + "code":"250" + }, + { + "desc":"JobManager and TaskManager heartbeats timed out. As a result, the Flink job is abnormal.Check whether the network is intermittently disconnected and whether the cluster l", + "product_code":"dli", + "title":"Why Is the Flink Job Abnormal Due to Heartbeat Timeout Between JobManager and TaskManager?", + "uri":"dli_03_0236.html", + "doc_type":"usermanual", + "p_code":"245", + "code":"251" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Spark Jobs", + "uri":"dli_03_0021.html", + "doc_type":"usermanual", + "p_code":"147", + "code":"252" + }, + { + "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "product_code":"dli", + "title":"Spark Job Development", + "uri":"dli_03_0217.html", + "doc_type":"usermanual", + "p_code":"252", + "code":"253" }, { "desc":"DLI Spark does not support job scheduling. You can use other services, such as DataArts Studio, or use APIs or SDKs to customize job schedule.The Spark SQL syntax does no", @@ -1913,17 +2282,8 @@ "title":"Spark Jobs", "uri":"dli_03_0201.html", "doc_type":"usermanual", - "p_code":"212", - "code":"213" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Job Development", - "uri":"dli_03_0217.html", - "doc_type":"usermanual", - "p_code":"211", - "code":"214" + "p_code":"253", + "code":"254" }, { "desc":"To use Spark to write data into a DLI table, configure the following parameters:fs.obs.access.keyfs.obs.secret.keyfs.obs.implfs.obs.endpointThe following is an example:", @@ -1931,17 +2291,17 @@ "title":"How Do I Use Spark to Write Data into a DLI Table?", "uri":"dli_03_0107.html", "doc_type":"usermanual", - "p_code":"214", - "code":"215" + "p_code":"253", + "code":"255" }, { "desc":"To obtain the AK/SK, set the parameters as follows:Create a SparkContext using code.val sc: SparkContext = new SparkContext()\nsc.hadoopConfiguration.set(\"fs.obs.access.ke", "product_code":"dli", - "title":"How Do I Set the AK/SK for a Queue to Operate an OBS Table?", + "title":"How Do I Set Up AK/SK So That a General Queue Can Access Tables Stored in OBS?", "uri":"dli_03_0017.html", "doc_type":"usermanual", - "p_code":"214", - "code":"216" + "p_code":"253", + "code":"256" }, { "desc":"Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs. In the job list, locate the target job and click next to Job ID to view the parame", @@ -1949,8 +2309,8 @@ "title":"How Do I View the Resource Usage of DLI Spark Jobs?", "uri":"dli_03_0102.html", "doc_type":"usermanual", - "p_code":"214", - "code":"217" + "p_code":"253", + "code":"257" }, { "desc":"If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management p", @@ -1958,8 +2318,8 @@ "title":"How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the Spark Job Results Stored in MySQL?", "uri":"dli_03_0076.html", "doc_type":"usermanual", - "p_code":"214", - "code":"218" + "p_code":"253", + "code":"258" }, { "desc":"DLI natively supports PySpark.For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are pac", @@ -1967,17 +2327,8 @@ "title":"How Do I Run a Complex PySpark Program in DLI?", "uri":"dli_03_0082.html", "doc_type":"usermanual", - "p_code":"214", - "code":"219" - }, - { - "desc":"You can use DLI Spark jobs to access data in the MySQL database using either of the following methods:Solution 1: Buy a queue, create an enhanced datasource connection, a", - "product_code":"dli", - "title":"How Does a Spark Job Access a MySQL Database?", - "uri":"dli_03_0127.html", - "doc_type":"usermanual", - "p_code":"214", - "code":"220" + "p_code":"253", + "code":"259" }, { "desc":"When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.To solve this problem, you can config", @@ -1985,8 +2336,8 @@ "title":"How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?", "uri":"dli_03_0068.html", "doc_type":"usermanual", - "p_code":"214", - "code":"221" + "p_code":"253", + "code":"260" }, { "desc":"You can use SparkFiles to read the file submitted using –-file form a local path: SparkFiles.get(\"Name of the uploaded file\").The file path in the Driver is different fro", @@ -1994,71 +2345,8 @@ "title":"How Do I Read Uploaded Files for a Spark Jar Job?", "uri":"dli_03_0118.html", "doc_type":"usermanual", - "p_code":"214", - "code":"222" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Job O&M Errors", - "uri":"dli_03_0218.html", - "doc_type":"usermanual", - "p_code":"211", - "code":"223" - }, - { - "desc":"The following error is reported when a Spark job accesses OBS data:Set the AK/SK to enable Spark jobs to access OBS data. For details, see How Do I Set the AK/SK for a Qu", - "product_code":"dli", - "title":"Why Are Errors \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Reported When a Spark Job Accesses OBS Data?", - "uri":"dli_03_0156.html", - "doc_type":"usermanual", - "p_code":"223", - "code":"224" - }, - { - "desc":"Check whether the OBS bucket is used to store DLI logs on the Global Configuration > Job Configurations page. The job log bucket cannot be used for other purpose.", - "product_code":"dli", - "title":"Why Is Error \"verifyBucketExists on XXXX: status [403]\" Reported When I Use a Spark Job to Access an OBS Bucket That I Have Access Permission?", - "uri":"dli_03_0164.html", - "doc_type":"usermanual", - "p_code":"223", - "code":"225" - }, - { - "desc":"When a Spark job accesses a large amount of data, for example, accessing data in a GaussDB(DWS) database, you are advised to set the number of concurrent tasks and enable", - "product_code":"dli", - "title":"Why Is a Job Running Timeout Reported When a Spark Job Runs a Large Amount of Data?", - "uri":"dli_03_0157.html", - "doc_type":"usermanual", - "p_code":"223", - "code":"226" - }, - { - "desc":"Spark jobs cannot access SFTP. Upload the files you want to access to OBS and then you can analyze the data using Spark jobs.", - "product_code":"dli", - "title":"Why Does the Job Fail to Be Executed and the Log Shows that the File Directory Is Abnormal When I Use a Spark Job to Access Files in SFTP?", - "uri":"dli_03_0188.html", - "doc_type":"usermanual", - "p_code":"223", - "code":"227" - }, - { - "desc":"When a Spark job is running, an error message is displayed, indicating that the user does not have the database permission. The error information is as follows:org.apache", - "product_code":"dli", - "title":"Why Does the Job Fail to Be Executed Due to Insufficient Database and Table Permissions?", - "uri":"dli_03_0192.html", - "doc_type":"usermanual", - "p_code":"223", - "code":"228" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"O&M Guide", - "uri":"dli_03_0219.html", - "doc_type":"usermanual", - "p_code":"211", - "code":"229" + "p_code":"253", + "code":"261" }, { "desc":"I cannot find the specified Python environment after adding the Python 3 package.Set spark.yarn.appMasterEnv.PYSPARK_PYTHON to python3 in the conf file to specify the Pyt", @@ -2066,8 +2354,8 @@ "title":"Why Can't I Find the Specified Python Environment After Adding the Python Package?", "uri":"dli_03_0077.html", "doc_type":"usermanual", - "p_code":"229", - "code":"230" + "p_code":"253", + "code":"262" }, { "desc":"The remaining CUs in the queue may be insufficient. As a result, the job cannot be submitted.To view the remaining CUs of a queue, perform the following steps:Check the C", @@ -2075,233 +2363,134 @@ "title":"Why Is a Spark Jar Job Stuck in the Submitting State?", "uri":"dli_03_0220.html", "doc_type":"usermanual", - "p_code":"229", - "code":"231" + "p_code":"253", + "code":"263" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Product Consultation", - "uri":"dli_03_0001.html", + "title":"Spark Job O&M", + "uri":"dli_03_0218.html", "doc_type":"usermanual", - "p_code":"123", - "code":"232" + "p_code":"252", + "code":"264" + }, + { + "desc":"The following error is reported when a Spark job accesses OBS data:Set the AK/SK to enable Spark jobs to access OBS data.For details, see How Do I Set Up AK/SK So That a ", + "product_code":"dli", + "title":"Why Do I Get \"ResponseCode: 403\" and \"ResponseStatus: Forbidden\" Errors When a Spark Job Accesses OBS Data?", + "uri":"dli_03_0156.html", + "doc_type":"usermanual", + "p_code":"264", + "code":"265" + }, + { + "desc":"This error message may be due to the OBS bucket being set as the DLI log bucket, which cannot be used for other purposes.You can follow these steps to check:Check if the ", + "product_code":"dli", + "title":"Why Do I Encounter the Error \"verifyBucketExists on XXXX: status [403]\" When Using a Spark Job to Access an OBS Bucket That I Have Permission to Access?", + "uri":"dli_03_0164.html", + "doc_type":"usermanual", + "p_code":"264", + "code":"266" + }, + { + "desc":"When running large amounts of data in a Spark job, if a timeout exception error occurs, it is usually due to insufficient resource configuration, data skew, network issue", + "product_code":"dli", + "title":"Why Does a Job Running Timeout Occur When Processing a Large Amount of Data with a Spark Job?", + "uri":"dli_03_0157.html", + "doc_type":"usermanual", + "p_code":"264", + "code":"267" + }, + { + "desc":"Spark jobs cannot access SFTP. Upload the files you want to access to OBS and then you can analyze the data using Spark jobs.Upload data to an OBS bucket: Upload data sto", + "product_code":"dli", + "title":"Why Does a Spark Job Fail to Execute with an Abnormal Access Directory Error When Accessing Files in SFTP?", + "uri":"dli_03_0188.html", + "doc_type":"usermanual", + "p_code":"264", + "code":"268" + }, + { + "desc":"When a Spark job is running, an error message is displayed, indicating that the user does not have the database permission. The error information is as follows:org.apache", + "product_code":"dli", + "title":"Why Does the Job Fail to Be Executed Due to Insufficient Database and Table Permissions?", + "uri":"dli_03_0192.html", + "doc_type":"usermanual", + "p_code":"264", + "code":"269" + }, + { + "desc":"I cannot find the global_temp database in the Spark 3.x job log.The global_temp database is the default built-in database of Spark 3.x and is Spark's global temporary vie", + "product_code":"dli", + "title":"Why Is the global_temp Database Missing in the Job Log of Spark 3.x?", + "uri":"dli_03_0272.html", + "doc_type":"usermanual", + "p_code":"264", + "code":"270" + }, + { + "desc":"I failed to use the DataSource syntax to create an OBS table in Avro format when selecting Spark to access metadata.Spark 2.3.x does not support creating OBS tables in Av", + "product_code":"dli", + "title":"Why Does Using DataSource Syntax to Create an OBS Table of Avro Type Fail When Accessing Metadata With Spark 2.3.x?", + "uri":"dli_03_0275.html", + "doc_type":"usermanual", + "p_code":"264", + "code":"271" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Usage", - "uri":"dli_03_0221.html", + "title":"DLI Resource Quotas", + "uri":"dli_03_0053.html", "doc_type":"usermanual", - "p_code":"232", - "code":"233" - }, - { - "desc":"DLI supports the following data formats:ParquetCSVORCJsonAvro", - "product_code":"dli", - "title":"Which Data Formats Does DLI Support?", - "uri":"dli_03_0025.html", - "doc_type":"usermanual", - "p_code":"233", - "code":"234" - }, - { - "desc":"The Spark component of DLI is a fully managed service. You can only use the DLI Spark through its APIs. .The Spark component of MRS is built on the VM in an MRS cluster. ", - "product_code":"dli", - "title":"What Are the Differences Between MRS Spark and DLI Spark?", - "uri":"dli_03_0115.html", - "doc_type":"usermanual", - "p_code":"233", - "code":"235" - }, - { - "desc":"DLI data can be stored in either of the following:OBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.DLI: The column-base", - "product_code":"dli", - "title":"Where Can DLI Data Be Stored?", - "uri":"dli_03_0029.html", - "doc_type":"usermanual", - "p_code":"233", - "code":"236" - }, - { - "desc":"DLI tables store data within the DLI service, and you do not need to know the data storage path.OBS tables store data in your OBS buckets, and you need to manage the sour", - "product_code":"dli", - "title":"What Are the Differences Between DLI Tables and OBS Tables?", - "uri":"dli_03_0117.html", - "doc_type":"usermanual", - "p_code":"233", - "code":"237" - }, - { - "desc":"Currently, DLI supports analysis only on the data uploaded to the cloud. In scenarios where regular (for example, on a per day basis) one-off analysis on incremental data", - "product_code":"dli", - "title":"How Can I Use DLI If Data Is Not Uploaded to OBS?", - "uri":"dli_03_0010.html", - "doc_type":"usermanual", - "p_code":"233", - "code":"238" - }, - { - "desc":"Data in the OBS bucket shared by IAM users under the same account can be imported. You cannot import data in the OBS bucket shared with other IAM account.", - "product_code":"dli", - "title":"Can I Import OBS Bucket Data Shared by Other Tenants into DLI?", - "uri":"dli_03_0129.html", - "doc_type":"usermanual", - "p_code":"233", - "code":"239" + "p_code":"147", + "code":"272" }, { "desc":"Log in to the management console.Click in the upper left corner and select a region and a project.Click the My Quota icon in the upper right corner of the page.The Serv", "product_code":"dli", - "title":"Why Is Error \"Failed to create the database. {\"error_code\":\"DLI.1028\";\"error_msg\":\"Already reached the maximum quota of databases:XXX\".\" Reported?", - "uri":"dli_03_0264.html", - "doc_type":"usermanual", - "p_code":"233", - "code":"240" - }, - { - "desc":"No, a global variable can only be used by the user who created it. Global variables can be used to simplify complex parameters. For example, long and difficult variables ", - "product_code":"dli", - "title":"Can a Member Account Use Global Variables Created by Other Member Accounts?", - "uri":"dli_03_0263.html", - "doc_type":"usermanual", - "p_code":"233", - "code":"241" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Job Management", - "uri":"dli_03_0222.html", - "doc_type":"usermanual", - "p_code":"232", - "code":"242" - }, - { - "desc":"If you are suggested to perform following operations to run a large number of DLI jobs:Group the DLI jobs by type, and run each group on a queue.Alternatively, create IAM", - "product_code":"dli", - "title":"How Do I Manage Tens of Thousands of Jobs Running on DLI?", - "uri":"dli_03_0126.html", - "doc_type":"usermanual", - "p_code":"242", - "code":"243" - }, - { - "desc":"The field names of tables that have been created cannot be changed.You can create a table, define new table fields, and migrate data from the old table to the new one.", - "product_code":"dli", - "title":"How Do I Change the Name of a Field in a Created Table?", - "uri":"dli_03_0162.html", - "doc_type":"usermanual", - "p_code":"242", - "code":"244" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Privacy and Security", - "uri":"dli_03_0261.html", - "doc_type":"usermanual", - "p_code":"232", - "code":"245" - }, - { - "desc":"No. The spark.acls.enable configuration item is not used in DLI. The Apache Spark command injection vulnerability (CVE-2022-33891) does not exist in DLI.", - "product_code":"dli", - "title":"Does DLI Have the Apache Spark Command Injection Vulnerability (CVE-2022-33891)?", - "uri":"dli_03_0260.html", - "doc_type":"usermanual", - "p_code":"245", - "code":"246" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Quota", - "uri":"dli_03_0053.html", - "doc_type":"usermanual", - "p_code":"123", - "code":"247" - }, - { - "desc":"Log in to the management console.Click in the upper left corner and select Region and Project.Click (the My Quotas icon) in the upper right corner.The Service Quota pag", - "product_code":"dli", "title":"How Do I View My Quotas?", "uri":"dli_03_0031.html", "doc_type":"usermanual", - "p_code":"247", - "code":"248" + "p_code":"272", + "code":"273" }, { "desc":"The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your applica", "product_code":"dli", - "title":"How Do I Increase a Quota?", + "title":"How Do I Apply for a Higher Quota?", "uri":"dli_03_0032.html", "doc_type":"usermanual", - "p_code":"247", - "code":"249" + "p_code":"272", + "code":"274" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Permission", + "title":"DLI Permissions Management", "uri":"dli_03_0054.html", "doc_type":"usermanual", - "p_code":"123", - "code":"250" + "p_code":"147", + "code":"275" }, { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", + "desc":"You are unable to perform permission operations on the partition columns of partitioned tables.However, when you grant the permission of any non-partition column in a par", "product_code":"dli", - "title":"Usage", - "uri":"dli_03_0223.html", - "doc_type":"usermanual", - "p_code":"250", - "code":"251" - }, - { - "desc":"DLI has a comprehensive permission control mechanism and supports fine-grained authentication through Identity and Access Management (IAM). You can create policies in IAM", - "product_code":"dli", - "title":"How Do I Manage Fine-Grained DLI Permissions?", - "uri":"dli_03_0100.html", - "doc_type":"usermanual", - "p_code":"251", - "code":"252" - }, - { - "desc":"You cannot perform permission-related operations on the partition column of a partitioned table.However, when you grant the permission of any non-partition column in a pa", - "product_code":"dli", - "title":"What Is Column Permission Granting of a DLI Partition Table?", + "title":"What Is Column-Level Authorization for DLI Partitioned Tables?", "uri":"dli_03_0008.html", "doc_type":"usermanual", - "p_code":"251", - "code":"253" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"O&M Guide", - "uri":"dli_03_0226.html", - "doc_type":"usermanual", - "p_code":"250", - "code":"254" - }, - { - "desc":"When you submit a job, a message is displayed indicating that the job fails to be submitted due to insufficient permission caused by arrears. In this case, you need to ch", - "product_code":"dli", - "title":"Why Does My Account Have Insufficient Permissions Due to Arrears?", - "uri":"dli_03_0140.html", - "doc_type":"usermanual", - "p_code":"254", - "code":"255" + "p_code":"275", + "code":"276" }, { "desc":"When the user update an existing program package, the following error information is displayed:\"error_code\"*DLI.0003\",\"error_msg\":\"Permission denied for resource 'resourc", "product_code":"dli", - "title":"Why Does the System Display a Message Indicating Insufficient Permissions When I Update a Program Package?", + "title":"How Do I Do If I Encounter Insufficient Permissions While Updating Packages?", "uri":"dli_03_0195.html", "doc_type":"usermanual", - "p_code":"254", - "code":"256" + "p_code":"275", + "code":"277" }, { "desc":"When the SQL query statement is executed, the system displays a message indicating that the user does not have the permission to query resources.Error information: DLI.00", @@ -2309,26 +2498,26 @@ "title":"Why Is Error \"DLI.0003: Permission denied for resource...\" Reported When I Run a SQL Statement?", "uri":"dli_03_0227.html", "doc_type":"usermanual", - "p_code":"254", - "code":"257" + "p_code":"275", + "code":"278" }, { - "desc":"The table permission has been granted and verified. However, after a period of time, an error is reported indicating that the table query fails.There are two possible rea", + "desc":"If you have already granted authorization to a table and a test query was successful, but you encounter an error when trying to query it again after some time, you should", "product_code":"dli", - "title":"Why Can't I Query Table Data After I've Been Granted Table Permissions?", + "title":"How Do I Do If I Can't Query Table Data After Being Granted Table Permissions?", "uri":"dli_03_0228.html", "doc_type":"usermanual", - "p_code":"254", - "code":"258" + "p_code":"275", + "code":"279" }, { - "desc":"If a table inherits database permissions, you do not need to regrant the inherited permissions to the table.When you grant permissions on a table on the console:If you se", + "desc":"If a table inherits database permissions, you do not need to regrant the inherited permissions to the table.Re-authorizing may cause confusion in table permission managem", "product_code":"dli", - "title":"Will an Error Be Reported if the Inherited Permissions Are Regranted to a Table That Inherits Database Permissions?", + "title":"Will Granting Duplicate Permissions to a Table After Inheriting Database Permissions Cause an Error?", "uri":"dli_03_0057.html", "doc_type":"usermanual", - "p_code":"254", - "code":"259" + "p_code":"275", + "code":"280" }, { "desc":"User A created Table1.User B created View1 based on Table1.After the Select Table permission on Table1 is granted to user C, user C fails to query View1.User B does not h", @@ -2336,359 +2525,26 @@ "title":"Why Can't I Query a View After I'm Granted the Select Table Permission on the View?", "uri":"dli_03_0067.html", "doc_type":"usermanual", - "p_code":"254", - "code":"260" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Queue", - "uri":"dli_03_0049.html", - "doc_type":"usermanual", - "p_code":"123", - "code":"261" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Usage", - "uri":"dli_03_0229.html", - "doc_type":"usermanual", - "p_code":"261", - "code":"262" - }, - { - "desc":"Currently, you are not allowed to modify the description of a created queue. You can add the description when purchasing the queue.", - "product_code":"dli", - "title":"Does the Description of a DLI Queue Can Be Modified?", - "uri":"dli_03_0109.html", - "doc_type":"usermanual", - "p_code":"262", - "code":"263" - }, - { - "desc":"Deleting a queue does not cause table data loss in your database.", - "product_code":"dli", - "title":"Will Table Data in My Database Be Lost If I Delete a Queue?", - "uri":"dli_03_0166.html", - "doc_type":"usermanual", - "p_code":"262", - "code":"264" - }, - { - "desc":"You need to develop a mechanism to retry failed jobs. When a faulty queue is recovered, your application tries to submit the failed jobs to the queue again.", - "product_code":"dli", - "title":"How Does DLI Ensure the Reliability of Spark Jobs When a Queue Is Abnormal?", - "uri":"dli_03_0170.html", - "doc_type":"usermanual", - "p_code":"262", - "code":"265" - }, - { - "desc":"DLI allows you to subscribe to an SMN topic for failed jobs.Log in to the DLI console.In the navigation pane on the left, choose Queue Management.On the Queue Management ", - "product_code":"dli", - "title":"How Do I Monitor Queue Exceptions?", - "uri":"dli_03_0098.html", - "doc_type":"usermanual", - "p_code":"262", - "code":"266" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"O&M Guide", - "uri":"dli_03_0230.html", - "doc_type":"usermanual", - "p_code":"261", - "code":"267" - }, - { - "desc":"To check the running status of the DLI queue and determine whether to run more jobs on that queue, you need to check the queue load.Search for Cloud Eye on the console.In", - "product_code":"dli", - "title":"How Do I View DLI Queue Load?", - "uri":"dli_03_0095.html", - "doc_type":"usermanual", - "p_code":"267", - "code":"268" - }, - { - "desc":"You need to check the large number of jobs in the Submitting and Running states on the queue.Use Cloud Eye to view jobs in different states on the queue. The procedure is", - "product_code":"dli", - "title":"How Do I Determine Whether There Are Too Many Jobs in the Current Queue?", - "uri":"dli_03_0183.html", - "doc_type":"usermanual", - "p_code":"267", - "code":"269" - }, - { - "desc":"Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earl", - "product_code":"dli", - "title":"How Do I Switch an Earlier-Version Spark Queue to a General-Purpose Queue?", - "uri":"dli_03_0065.html", - "doc_type":"usermanual", - "p_code":"267", - "code":"270" - }, - { - "desc":"DLI queues do not use resources or bandwidth when no job is running. In this case, the running status of DLI queues is not displayed on CES.", - "product_code":"dli", - "title":"Why Cannot I View the Resource Running Status of DLI Queues on Cloud Eye?", - "uri":"dli_03_0193.html", - "doc_type":"usermanual", - "p_code":"267", - "code":"271" - }, - { - "desc":"In DLI, 64 CU = 64 cores and 256 GB memory.In a Spark job, if the driver occupies 4 cores and 16 GB memory, the executor can occupy 60 cores and 240 GB memory.", - "product_code":"dli", - "title":"How Do I Allocate Queue Resources for Running Spark Jobs If I Have Purchased 64 CUs?", - "uri":"dli_03_0088.html", - "doc_type":"usermanual", - "p_code":"267", - "code":"272" - }, - { - "desc":"Queue plans create failed. The plan xxx target cu is out of quota is displayed when you create a scheduled scaling task.The CU quota of the current account is insufficien", - "product_code":"dli", - "title":"Why Is Error \"Queue plans create failed. The plan xxx target cu is out of quota\" Reported When I Schedule CU Changes?", - "uri":"dli_03_0159.html", - "doc_type":"usermanual", - "p_code":"267", - "code":"273" - }, - { - "desc":"After a SQL job was submitted to the default queue, the job runs abnormally. The job log reported that the execution timed out. The exception logs are as follows:[ERROR] ", - "product_code":"dli", - "title":"Why Is a Timeout Exception Reported When a DLI SQL Statement Fails to Be Executed on the Default Queue?", - "uri":"dli_03_0171.html", - "doc_type":"usermanual", - "p_code":"267", - "code":"274" - }, - { - "desc":"In daily big data analysis work, it is important to allocate and manage compute resources properly to provide a good job execution environment.You can allocate resources ", - "product_code":"dli", - "title":"How Can I Check the Actual and Used CUs for an Elastic Resource Pool as Well as the Required CUs for a Job?", - "uri":"dli_03_0276.html", - "doc_type":"usermanual", - "p_code":"267", - "code":"275" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Datasource Connections", - "uri":"dli_03_0022.html", - "doc_type":"usermanual", - "p_code":"123", - "code":"276" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Datasource Connections", - "uri":"dli_03_0110.html", - "doc_type":"usermanual", - "p_code":"276", - "code":"277" - }, - { - "desc":"You need to create a VPC peering connection to enable network connectivity. Take MRS as an example. If DLI and MRS clusters are in the same VPC, and the security group is", - "product_code":"dli", - "title":"Why Do I Need to Create a VPC Peering Connection for an Enhanced Datasource Connection?", - "uri":"dli_03_0128.html", - "doc_type":"usermanual", - "p_code":"277", - "code":"278" - }, - { - "desc":"An enhanced datasource connection failed to pass the network connectivity test. Datasource connection cannot be bound to a queue. The following error information is displ", - "product_code":"dli", - "title":"Failed to Bind a Queue to an Enhanced Datasource Connection", - "uri":"dli_03_0237.html", - "doc_type":"usermanual", - "p_code":"277", - "code":"279" - }, - { - "desc":"The outbound rule had been configured for the security group of the queue associated with the enhanced datasource connection. The datasource authentication used a passwor", - "product_code":"dli", - "title":"DLI Failed to Connect to GaussDB(DWS) Through an Enhanced Datasource Connection", - "uri":"dli_03_0238.html", - "doc_type":"usermanual", - "p_code":"277", - "code":"280" - }, - { - "desc":"A datasource connection is created and bound to a queue. The connectivity test fails and the following error information is displayed:failed to connect to specified addre", - "product_code":"dli", - "title":"How Do I Do if the Datasource Connection Is Created But the Network Connectivity Test Fails?", - "uri":"dli_03_0179.html", - "doc_type":"usermanual", - "p_code":"277", + "p_code":"275", "code":"281" }, { - "desc":"Configuring the Connection Between a DLI Queue and a Data Source in a Private NetworkIf your DLI job needs to connect to a data source, for example, MRS, RDS, CSS, Kafka,", + "desc":"Despite configuring a job bucket and authorizing DLI to access it, I still receive an error message stating that DLI is not authorized to access the bucket when attemptin", "product_code":"dli", - "title":"How Do I Configure the Network Between a DLI Queue and a Data Source?", - "uri":"dli_03_0186.html", + "title":"How Do I Do If I Receive a Message Saying I Don't Have Sufficient Permissions to Submit My Jobs to the Job Bucket?", + "uri":"dli_03_0225.html", "doc_type":"usermanual", - "p_code":"277", + "p_code":"275", "code":"282" }, - { - "desc":"The possible causes and solutions are as follows:If you have created a queue, do not bind it to a datasource connection immediately. Wait for 5 to 10 minutes. After the c", - "product_code":"dli", - "title":"What Can I Do If a Datasource Connection Is Stuck in Creating State When I Try to Bind a Queue to It?", - "uri":"dli_03_0257.html", - "doc_type":"usermanual", - "p_code":"277", - "code":"283" - }, - { - "desc":"DLI enhanced datasource connection uses VPC peering to directly connect the VPC networks of the desired data sources for point-to-point data exchanges.", - "product_code":"dli", - "title":"How Do I Connect DLI to Data Sources?", - "uri":"dli_03_0259.html", - "doc_type":"usermanual", - "p_code":"277", - "code":"284" - }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", "product_code":"dli", - "title":"Cross-Source Analysis", - "uri":"dli_03_0112.html", - "doc_type":"usermanual", - "p_code":"276", - "code":"285" - }, - { - "desc":"To perform query on data stored on services rather than DLI, perform the following steps:Assume that the data to be queried is stored on multiple services (for example, O", - "product_code":"dli", - "title":"How Can I Perform Query on Data Stored on Services Rather Than DLI?", - "uri":"dli_03_0011.html", - "doc_type":"usermanual", - "p_code":"285", - "code":"286" - }, - { - "desc":"Connect VPCs in different regions.Create an enhanced datasource connection on DLI and bind it to a queue.Add a DLI route.", - "product_code":"dli", - "title":"How Can I Access Data Across Regions?", - "uri":"dli_03_0085.html", - "doc_type":"usermanual", - "p_code":"285", - "code":"287" - }, - { - "desc":"When data is inserted into DLI, set the ID field to NULL.", - "product_code":"dli", - "title":"How Do I Set the Auto-increment Primary Key or Other Fields That Are Automatically Filled in the RDS Table When Creating a DLI and Associating It with the RDS Table?", - "uri":"dli_03_0028.html", - "doc_type":"usermanual", - "p_code":"285", - "code":"288" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"Datasource Connection O&M", - "uri":"dli_03_0256.html", - "doc_type":"usermanual", - "p_code":"276", - "code":"289" - }, - { - "desc":"Possible CausesThe network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.The network connec", - "product_code":"dli", - "title":"Why Is the Error Message \"communication link failure\" Displayed When I Use a Newly Activated Datasource Connection?", - "uri":"dli_03_0047.html", - "doc_type":"usermanual", - "p_code":"289", - "code":"290" - }, - { - "desc":"The cluster host information is not added to the datasource connection. As a result, the KRB authentication fails, the connection times out, and no error is recorded in l", - "product_code":"dli", - "title":"Connection Times Out During MRS HBase Datasource Connection, and No Error Is Recorded in Logs", - "uri":"dli_03_0080.html", - "doc_type":"usermanual", - "p_code":"289", - "code":"291" - }, - { - "desc":"When you create a VPC peering connection for the datasource connection, the following error information is displayed:Before you create a datasource connection, check whet", - "product_code":"dli", - "title":"Why Can't I Find the Subnet When Creating a DLI Datasource Connection?", - "uri":"dli_03_0111.html", - "doc_type":"usermanual", - "p_code":"289", - "code":"292" - }, - { - "desc":"A datasource RDS table was created in the DataArts Studio, and the insert overwrite statement was executed to write data into RDS. DLI.0999: BatchUpdateException: Incorre", - "product_code":"dli", - "title":"Error Message \"Incorrect string value\" Is Displayed When insert overwrite Is Executed on a Datasource RDS Table", - "uri":"dli_03_0239.html", - "doc_type":"usermanual", - "p_code":"289", - "code":"293" - }, - { - "desc":"The system failed to create a datasource RDS table, and null pointer error was reported.The following table creation statement was used:The RDS database is in a PostGre c", - "product_code":"dli", - "title":"Null Pointer Error Is Displayed When the System Creates a Datasource RDS Table", - "uri":"dli_03_0250.html", - "doc_type":"usermanual", - "p_code":"289", - "code":"294" - }, - { - "desc":"The system failed to execute insert overwrite on the datasource GaussDB(DWS) table, and org.postgresql.util.PSQLException: ERROR: tuple concurrently updated was displayed", - "product_code":"dli", - "title":"Error Message \"org.postgresql.util.PSQLException: ERROR: tuple concurrently updated\" Is Displayed When the System Executes insert overwrite on a Datasource GaussDB(DWS) Table", - "uri":"dli_03_0251.html", - "doc_type":"usermanual", - "p_code":"289", - "code":"295" - }, - { - "desc":"A datasource table was used to import data to a CloudTable HBase table. This HBase table contains a column family and a rowkey for 100 million simulating data records. Th", - "product_code":"dli", - "title":"RegionTooBusyException Is Reported When Data Is Imported to a CloudTable HBase Table Through a Datasource Table", - "uri":"dli_03_0252.html", - "doc_type":"usermanual", - "p_code":"289", - "code":"296" - }, - { - "desc":"A table was created on GaussDB(DWS) and then a datasource connection was created on DLI to read and write data. An error message was displayed during data writing, indica", - "product_code":"dli", - "title":"A Null Value Is Written Into a Non-Null Field When a DLI Datasource Connection Is Used to Connect to a GaussDB(DWS) Table", - "uri":"dli_03_0253.html", - "doc_type":"usermanual", - "p_code":"289", - "code":"297" - }, - { - "desc":"A datasource GaussDB(DWS) table and the datasource connection were created in DLI, and the schema of the source table in GaussDB(DWS) were updated. During the job executi", - "product_code":"dli", - "title":"An Insert Operation Failed After the Schema of the GaussDB(DWS) Source Table Is Updated", - "uri":"dli_03_0254.html", - "doc_type":"usermanual", - "p_code":"289", - "code":"298" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"APIs", + "title":"DLI APIs", "uri":"dli_03_0056.html", "doc_type":"usermanual", - "p_code":"123", - "code":"299" + "p_code":"147", + "code":"283" }, { "desc":"In the REST API provided by DLI, the request header can be added to the request URI, for example, Content-Type.Content-Type indicates the request body type or format. The", @@ -2696,17 +2552,8 @@ "title":"Why Is Error \"unsupported media Type\" Reported When I Subimt a SQL Job?", "uri":"dli_03_0060.html", "doc_type":"usermanual", - "p_code":"299", - "code":"300" - }, - { - "desc":"When different IAM users call an API under the same enterprise project in the same region, the project ID is the same.", - "product_code":"dli", - "title":"Is the Project ID Fixed when Different IAM Users Call an API?", - "uri":"dli_03_0125.html", - "doc_type":"usermanual", - "p_code":"299", - "code":"301" + "p_code":"283", + "code":"284" }, { "desc":"When the API call for submitting a SQL job times out, and the following error information is displayed:There are currently no resources tracked in the state, so there is ", @@ -2714,35 +2561,8 @@ "title":"What Can I Do If an Error Is Reported When the Execution of the API for Creating a SQL Job Times Out?", "uri":"dli_03_0178.html", "doc_type":"usermanual", - "p_code":"299", - "code":"302" - }, - { - "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", - "product_code":"dli", - "title":"SDKs", - "uri":"dli_03_0058.html", - "doc_type":"usermanual", - "p_code":"123", - "code":"303" - }, - { - "desc":"When you query the SQL job results using SDK, the system checks the job status when the job is submitted. The timeout interval set in the system is 300s. If the job is no", - "product_code":"dli", - "title":"How Do I Set the Timeout Duration for Querying SQL Job Results Using SDK?", - "uri":"dli_03_0073.html", - "doc_type":"usermanual", - "p_code":"303", - "code":"304" - }, - { - "desc":"Run the ping command to check whether dli.xxx can be accessed.If dli.xxx can be accessed, check whether DNS resolution is correctly configured.If dli.xxx can be accessed,", - "product_code":"dli", - "title":"How Do I Handle the dli.xxx,unable to resolve host address Error?", - "uri":"dli_03_0255.html", - "doc_type":"usermanual", - "p_code":"303", - "code":"305" + "p_code":"283", + "code":"285" }, { "desc":"HUAWEI CLOUD Help Center presents technical documents to help you quickly get started with HUAWEI CLOUD services. The technical documents include Service Overview, Price Details, Purchase Guide, User Guide, API Reference, Best Practices, FAQs, and Videos.", @@ -2751,6 +2571,6 @@ "uri":"dli_01_00006.html", "doc_type":"usermanual", "p_code":"", - "code":"306" + "code":"286" } ] \ No newline at end of file diff --git a/docs/dli/umn/dli_01_00006.html b/docs/dli/umn/dli_01_00006.html index cbbcb3e17..6cff00ac0 100644 --- a/docs/dli/umn/dli_01_00006.html +++ b/docs/dli/umn/dli_01_00006.html @@ -8,22 +8,29 @@ -

2024-07-11

+

2025-02-08

+ +

Modified the following sections:

+ + + +

2024-07-11

Modified the following section:

- +

2024-04-28

Added the following section:

- +

2024-02-27

Added the following content:

- +

Modified the following section:

Changed DDS MongoDB to DDS in Creating a Flink Jar Job.

@@ -31,31 +38,31 @@

2023-11-01

Modified the following content:

- +

2023-10-08

Modified the following content:

- +

2023-09-07

Added the following sections:

- +

Deleted the following section:

  • Deleted the content related to Flink job debugging.

Modified the following section:

-
  • Performing Operations on a Flink Job: Modified the description of the Auto Restart upon Exception parameter in the runtime configuration. Flink SQL jobs and Flink OpenSource SQL jobs cannot be restored from checkpoints.
-
Optimized the following sections: +
  • Common Operations of Flink Jobs: Modified the description of the Auto Restart upon Exception parameter in the runtime configuration. Flink SQL jobs and Flink OpenSource SQL jobs cannot be restored from checkpoints.
+
Optimized the following sections:

2023-05-11

Modified the following content:

- +

2023-01-30

diff --git a/docs/dli/umn/dli_01_0001.html b/docs/dli/umn/dli_01_0001.html index 5d9350688..4ea540017 100644 --- a/docs/dli/umn/dli_01_0001.html +++ b/docs/dli/umn/dli_01_0001.html @@ -1,19 +1,72 @@ -

Job Management

-
-
- +

DLI Job Development Process

+

This chapter walks you through on how to develop a DLI job.

+

Creating an IAM User and Granting Permissions

  • To manage fine-grained permissions for your DLI resources using IAM, create an IAM user and grant them permissions to DLI if you are an enterprise user. For details, see Creating an IAM User and Granting Permissions.
  • When using DLI for the first time, you need to update the DLI agency according to the console's guidance so that DLI can use other cloud services and perform resource O&M operations on your behalf. The agency includes permissions to obtain IAM user information, access and use VPCs, CIDR blocks, routes, and peering connections, and send notifications via SMN in case of job execution failure.

    For more information on the specific permissions included in the agency, refer to Configuring DLI Agency Permissions.

    +
+
+

Creating Compute Resources and Metadata Required for Running Jobs

  • Before submitting a job using DLI, you need to create an elastic resource pool and create queues within it. This will provide the necessary compute resources for running the job. For how to create an elastic resource pool and create queues within it, see Overview of DLI Elastic Resource Pools and Queues.

    Alternatively, you can enhance DLI's computing environment by creating custom images. Specifically, to enhance the functions and performance of Spark and Flink jobs, you can create custom images by downloading the base images provided by DLI and adding dependencies (files, JAR files, or software) and private capabilities required for job execution. This changes the container runtime environment for the jobs.

    +

    For example, you can add a Python package or C library related to machine learning to a custom image to help you extend functions. For how to create a custom image, see Using a Custom Image to Enhance the Job Running Environment.

    +
  • DLI metadata is the basis for developing SQL and Spark jobs. Before executing a job, you need to define databases and tables based on your business scenario.

    Flink allows for dynamic data types, enabling the definition of data structures at runtime without the need for predefined metadata.

    +
    + +
+
+

Importing Data to DLI

+
+

Submitting a Job Using DLI

  • DLI offers a serverless service that integrates stream processing, batch processing, and interactive analytics. It supports various job types to meet different data processing needs. +
    + + + + + + + + + + + + + + + + + +
    Table 1 Job types supported by DLI

    Job Type

    +

    Description

    +

    Use Case

    +

    SQL job

    +

    This type is suitable for scenarios where standard SQL statements are used for querying. It is typically used for querying and analyzing structured data.

    +

    For details, see Creating and Submitting a SQL Job.

    +

    It applies to scenarios such as data warehouse query, report generation, and online analytical processing (OLAP).

    +

    Flink job

    +

    This type is specifically designed for real-time data stream processing, making it ideal for scenarios that require low latency and quick response. It is well-suited for real-time monitoring and online analysis.

    +
    • Flink OpenSource job: DLI provides standard connectors and various APIs to facilitate quick integration with other data systems. For details, see Creating a Flink OpenSource SQL Job.
    • Flink Jar job: allows you to submit Flink jobs compiled into JAR files, providing greater flexibility and customization capabilities.

      It is suitable for complex data processing scenarios that require user-defined functions (UDFs) or specific library integration. The Flink ecosystem can be utilized to implement advanced stream processing logic and status management. For details, see Creating a Flink Jar Job.

      +
    +

    It applies to scenarios that require quick response, such as real-time data monitoring and real-time recommender systems.

    +

    Flink Jar jobs are suitable for data analysis scenarios that require custom stream processing logic, complex state management, or integration with specific libraries.

    +

    Spark job

    +

    Compute jobs can be submitted through interactive sessions or batch processing. Jobs are submitted to queues created within an elastic resource pool, simplifying resource management and job scheduling.

    +

    It supports multiple data sources and formats, providing rich data processing capabilities, including but not limited to SQL queries and machine learning. For details, see Creating a Spark Job.

    +

    It is suitable for large-scale data processing and analysis, such as machine learning training, log analysis, and large-scale data mining.

    +
    +
    +
  • Manage program packages of Jar jobs.

    DLI allows you to submit Flink or Spark jobs compiled as JAR files, which contain the necessary code and dependency information for executing the job. These files are used for specific data processing tasks such as data query, analysis, and machine learning. You can manage program packages required for jobs on the DLI console.

    +

    To submit a Spark Jar or Flink Jar job, you must first upload the program package to OBS, create a program package in DLI, and then submit the program package, data, and job parameters to run the job. For details, see Managing Program Packages of Jar Jobs.

    +

    For Spark 3.3.1 or later and Flink 1.15 or later, when creating a Jar job, you can directly configure the program package in OBS. Program packages cannot be read from DLI.

    +
    +
+
+

Using Cloud Eye to Monitor DLI

You can query DLI monitoring metrics and alarms through Cloud Eye management console or APIs.

+

For example, you can monitor the resource usage and job status of a DLI queue. For details about DLI metrics, see Using Cloud Eye to Monitor DLI.

+
+

Using CTS to Audit DLI

With CTS, you can log operations related to DLI, making it easier to search, audit, and trace in the future. For the supported operations, see Using CTS to Audit DLI.

+
diff --git a/docs/dli/umn/dli_01_0002.html b/docs/dli/umn/dli_01_0002.html index 57841538a..6f2b56e17 100644 --- a/docs/dli/umn/dli_01_0002.html +++ b/docs/dli/umn/dli_01_0002.html @@ -1,44 +1,211 @@ -

Creating and Submitting a Spark SQL Job

-

Scenario

DLI can query data stored in OBS. This section describes how to us a Spark SQL job on DLI to query OBS data.

+

Using DLI to Submit a SQL Job to Query OBS Data

+

Scenario

DLI can query data stored in OBS. This section describes how to use DLI to submit a SQL job to query OBS data.

+

To illustrate, create a new file called sampledata.csv and upload it to an OBS bucket. Then, create an elastic resource pool, create queues within it, and use DLI to create a database and table. Finally, use DLI's SQL editor to query 1,000 data records from the table.

-

Procedure

You can use DLI to submit a Spark SQL job to query data. The general procedure is as follows:

-

Step 1: Upload Data to OBS

-

Step 2: Create a Queue

-

Step 3: Create a Database

-

Step 4: Create a Table

-

Step 5: Query Data

+

Procedure

Table 1 shows the process for submitting a SQL job to query OBS data.

+ +
+ + + + + + + + + + + + + + + + + + + +
Table 1 Procedure for using DLI to submit a SQL job to query OBS data

Procedure

+

Description

+

Step 1: Upload Data to OBS

+

Upload data files to OBS.

+

Step 2: Create an Elastic Resource Pool and Add Queues to the Pool

+

Create compute resources required for submitting jobs.

+

Step 3: Create a Database

+

DLI metadata forms the foundation for SQL job development. Before executing a job, you need to define databases and tables based on your business scenario.

+

Step 4: Create a Table

+

Use the sample data stored in OBS to create tables in the db1 database.

+

Step 5: Query Data

+

Use standard SQL statements to query and analyze data.

+
-

Step 1: Upload Data to OBS

Before you use DLI to query and analyze data, upload data files to OBS.

-
  1. Go to the DLI console.
  2. In the service list, click Object Storage Service under Storage. The OBS console page is displayed.
  3. Create a bucket. In this example, the bucket name is obs1.
    1. Click Create Bucket in the upper right corner.
    2. On the displayed Create Bucket page, enter the Bucket Name. Retain the default values for other parameters or adjust them as needed.

      You must select the same region as the DLI management console.

      +
      +

      Step 1: Upload Data to OBS

      Upload data files to OBS.

      +
      1. Log in to the OBS management console.
      2. Create a bucket. In this example, the bucket name is obs1.
        1. Click Create Bucket in the upper right corner.
        2. On the displayed Create Bucket page, enter the Bucket Name. Retain the default values for other parameters or adjust them as needed.

          Select a region that matches the location of the DLI console.

        3. Click Create Now.
        -
      3. Click obs1 to access its Objects tab page.
      4. Click Upload Object. In the displayed dialog box, drag a desired file or folder, for example, sampledata.csv to the Upload Object area. Then, click Upload.
        You can create a sampledata.txt file, copy the following content separated by commas (,), and save the file as sampledata.csv.
        12,test
        -
        +
      5. Click obs1 to access its Objects tab page.
      6. Click Upload Object. In the displayed dialog box, drag a desired file or folder, for example, sampledata.csv to the Upload Object area. Then, click Upload.

        You can create a sampledata.txt file, copy the following content separated by commas (,), and save the file as sampledata.csv.

        +
        product_id,product_name
+113,office_13
+22,book_2
+29,book_9

        After the file is uploaded successfully, the file path is obs://obs1/sampledata.csv.

        -
        • For more information about OBS operations, see the Object Storage Service Console Operation Guide.
        • For more information about the tool, see the OBS Tool Guide.
        • You are advised to use an OBS tool, such as OBS Browser+, to upload large files because OBS Console has restrictions on the file size and quantity.
          • OBS Browser+ is a graphical tool that provides complete functions for managing your buckets and objects in OBS.
          -
        -
        +

        For more operations on the OBS console, see the Object Storage Service User Guide.

      -

      Step 2: Create a Queue

      A queue is the basis for using DLI. Before executing a SQL job, you need to create a queue.

      -
      • DLI provides a preconfigured queue named default.
      • You can also create queues as needed.
        1. Log in to the DLI management console.
        2. In the left navigation pane of the DLI management console, choose SQL Editor.
        3. On the left pane, select the Queues tab, and click next to Queues.

          For details, see Creating a Queue.

          -

          -
        -
      +

      Step 2: Create an Elastic Resource Pool and Add Queues to the Pool

      In this example, the elastic resource pool dli_resource_pool and queue dli_queue_01 are created.
      1. Log in to the DLI management console.
      2. In the navigation pane on the left, choose Resources > Resource Pool.
      3. On the displayed page, click Buy Resource Pool in the upper right corner.
      4. On the displayed page, set the parameters.
        Table 2 describes the parameters. +
        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        Table 2 Parameters

        Parameter

        +

        Description

        +

        Example Value

        +

        Region

        +

        Select a region where you want to buy the elastic resource pool.

        +

        _

        +

        Project

        +

        Project uniquely preset by the system for each region

        +

        Default

        +

        Name

        +

        Name of the elastic resource pool

        +

        dli_resource_pool

        +

        Specifications

        +

        Specifications of the elastic resource pool

        +

        Standard

        +

        CU Range

        +

        The maximum and minimum CUs allowed for the elastic resource pool

        +

        64-64

        +

        CIDR Block

        +

        CIDR block the elastic resource pool belongs to. If you use an enhanced datasource connection, this CIDR block cannot overlap that of the data source. Once set, this CIDR block cannot be changed.

        +

        172.16.0.0/19

        +

        Enterprise Project

        +

        Select an enterprise project for the elastic resource pool.

        +

        default

        +
        +
        +
        +
      5. Click Buy.
      6. Click Submit.
      7. In the elastic resource pool list, locate the pool you just created and click Add Queue in the Operation column.
      8. Set the basic parameters listed below. +
        + + + + + + + + + + + + + + + + + +
        Table 3 Basic parameters for adding a queue

        Parameter

        +

        Description

        +

        Example Value

        +

        Name

        +

        Name of the queue to add

        +

        dli_queue_01

        +

        Type

        +

        Type of the queue

        +
        • To execute SQL jobs, select For SQL.
        • To execute Flink or Spark jobs, select For general purpose.
        +

        _

        +

        Enterprise Project

        +

        Select an enterprise project.

        +

        default

        +
        +
        +
      9. Click Next and configure scaling policies for the queue.

        Click Create to add a scaling policy with varying priority, period, minimum CUs, and maximum CUs.

        +

        + +
        + + + + + + + + + + + + + + + + + + + + + +
        Table 4 Scaling policy parameters

        Parameter

        +

        Description

        +

        Example Value

        +

        Priority

        +

        Priority of the scaling policy in the current elastic resource pool. A larger value indicates a higher priority. In this example, only one scaling policy is configured, so its priority is set to 1 by default.

        +

        1

        +

        Period

        +

        The first scaling policy is the default policy, and its Period parameter configuration cannot be deleted or modified.

        +

        The period for the scaling policy is from 00 to 24.

        +

        00–24

        +

        Min CU

        +

        Minimum number of CUs allowed by the scaling policy

        +

        16

        +

        Max CU

        +

        Maximum number of CUs allowed by the scaling policy

        +

        64

        +
        +
        +
      10. Click OK.
      +

      Step 3: Create a Database

      Before querying data, create a database, for example, db1.

      The default database is a built-in database. You cannot create the default. database.

      1. In the left navigation pane of the DLI management console, choose SQL Editor.
      2. In the editing window on the right of the SQL Editor page, enter the following SQL statement and click Execute. Read and agree to the privacy agreement, and click OK.
        create database db1;
        -

        After the database is successfully created, click in the middle pane to refresh the database list. The new database db1 is displayed in the list.

        -

        When you execute a query on the DLI management console for the first time, you need to read the privacy agreement. You can perform operations only after you agree to the agreement. For later queries, you will not need to read the privacy agreement again.

        -
        +

        After the database is successfully created, click in the middle pane to refresh the database list. The new database db1 is displayed in the list.

      Step 4: Create a Table

      After database db1 is created, create a table (for example, table1) containing data in the sample file obs://obs1/sampledata.csv stored on OBS in db1.

      -
      1. In the SQL editing window of the SQL Editor page, select the default queue and database db1.
      2. Enter the following SQL statement in the job editor window and click Execute:
        create table table1 (id int, name string) using csv options (path 'obs://obs1/sampledata.csv');
        +
        1. In the SQL editing window of the SQL Editor page, select the default queue and database db1.
        2. Enter the following SQL statement in the job editor window and click Execute:
          create table table1 (product_id int, product_name string) using csv options (path 'obs://obs1');
          +

          When creating a table, you only need to specify the OBS storage path where the data file is located, without specifying the file name at the end of the directory.

          After the table is successfully created, click the Databases tab then db1. The created table table1 is displayed in the table list.

      diff --git a/docs/dli/umn/dli_01_0003.html b/docs/dli/umn/dli_01_0003.html index 35c6b2bf1..67edbadd3 100644 --- a/docs/dli/umn/dli_01_0003.html +++ b/docs/dli/umn/dli_01_0003.html @@ -1,18 +1,47 @@ -

      Overview

      -

      What Is Enhanced Datasource Connection?

      In cross-source data analysis scenarios, DLI needs to connect to external data sources. However, due to the different VPCs between the data source and DLI, the network cannot be connected, which results in DLI being unable to read data from the data source. DLI's enhanced datasource connection feature enables network connectivity between DLI and the data source.

      +

      Overview of Enhanced Datasource Connections

      +

      Why Create Enhanced Datasource Connections?

      In cross-source data analysis scenarios, DLI needs to connect to external data sources. However, due to the different VPCs between the data source and DLI, the network cannot be connected, which results in DLI being unable to read data from the data source. DLI's enhanced datasource connection feature enables network connectivity between DLI and the data source.

      This section will introduce a solution for cross-VPC data source network connectivity:

      • Creating an enhanced datasource connection: Establish a VPC peering connection to connect DLI and the data source's VPC network.
      • Testing network connectivity: Verify the connectivity between the queue and the data source's network.
      -

      For details about the data sources that support cross-source access, see Cross-Source Analysis Development Methods.

      -

      In cross-source development scenarios, there is a risk of password leakage if datasource authentication information is directly configured. You are advised to use the datasource authentication provided by DLI. For details, see Overview.

      +

      For details about the data sources that support cross-source access, see Common Development Methods for DLI Cross-Source Analysis.

      +

      In cross-source development scenarios, there is a risk of password leakage if datasource authentication information is directly configured. You are advised to use Data Encryption Workshop (DEW) to store authentication information of data sources when Spark 3.3.1 or later and Flink 1.15 or later jobs access data sources using datasource connections. This will help you address issues related to data security, key security, and complex key management. For details, see Using DEW to Manage Access Credentials for Data Sources.

      -

      Constraints

      • Datasource connections cannot be created for the default queue.
      • Flink jobs can directly access DIS, OBS, and SMN data sources without using datasource connections.
      • VPC Administrator permissions are required for enhanced connections to use VPCs, subnets, routes, VPC peering connections.
      • If you use an enhanced datasource connection, the CIDR block of the elastic resource pool or queue cannot overlap with that of the data source.
      • Only queues bound with datasource connections can access datasource tables.
      • Datasource tables do not support the preview function.
      • When checking the connectivity of datasource connections, the notes and constraints on IP addresses are:
        • The IP address must be valid, which consists of four decimal numbers separated by periods (.). The value ranges from 0 to 255.
        • During the test, you can add a port after the IP address and separate them with colons (:). The port can contain a maximum of five digits. The value ranges from 0 to 65535.

          For example, 192.168.xx.xx or 192.168.xx.xx:8181.

          +

          Notes and Constraints

          +
          + + + + + + + + + + + + + + + + +
          Table 1 Notes and constraints on enhanced datasource connections

          Item

          +

          Description

          +

          Use case

          +
          • Datasource connections cannot be created for the default queue.
          • Flink jobs can directly access DIS, OBS, and SMN data sources without using datasource connections.
          +

          Permission

          +
          • VPC Administrator permissions are required for enhanced connections to use VPCs, subnets, routes, VPC peering connections.
          +

          Usage

          +
          • If you use an enhanced datasource connection, the CIDR block of the elastic resource pool or queue cannot overlap with that of the data source.
          • Only queues bound with datasource connections can access datasource tables.
          • Datasource tables do not support the preview function.
          +

          Connectivity check

          +
          • When checking the connectivity of datasource connections, the notes and constraints on IP addresses are:
            • The IP address must be valid, which consists of four decimal numbers separated by periods (.). The value ranges from 0 to 255.
            • During the test, you can add a port after the IP address and separate them with colons (:). The port can contain a maximum of five digits. The value ranges from 0 to 65535.

              For example, 192.168.xx.xx or 192.168.xx.xx:8181.

            -
          • When checking the connectivity of datasource connections, the notes and constraints on domain names are:
            • The domain name can contain 1 to 255 characters. Only letters, digits, underscores (_), and hyphens (-) are allowed.
            • The top-level domain name must contain at least two letters, for example, .com, .net, and .cn.
            • During the test, you can add a port after the domain name and separate them with colons (:). The port can contain a maximum of five digits. The value ranges from 0 to 65535.

              For example, example.com:8080.

              +
            • When checking the connectivity of datasource connections, the notes and constraints on domain names are:
              • The domain name can contain 1 to 255 characters. Only letters, numbers, underscores (_), and hyphens (-) are allowed.
              • The top-level domain name must contain at least two letters, for example, .com, .net, and .cn.
              • During the test, you can add a port after the domain name and separate them with colons (:). The port can contain a maximum of five digits. The value ranges from 0 to 65535.

                Example: example.com:8080

            +
          +

          Cross-Source Analysis Process

          To use DLI for cross-source analysis, you need to create a datasource connection to connect DLI to the data source, and then develop jobs to access the data source.

          Figure 1 Cross-source analysis flowchart
          @@ -20,7 +49,7 @@
          diff --git a/docs/dli/umn/dli_01_0004.html b/docs/dli/umn/dli_01_0004.html index 1d24cba4b..1f4245345 100644 --- a/docs/dli/umn/dli_01_0004.html +++ b/docs/dli/umn/dli_01_0004.html @@ -1,13 +1,21 @@ -

          Data Management

          +

          Managing Database Resources on the DLI Console

          diff --git a/docs/dli/umn/dli_01_0005.html b/docs/dli/umn/dli_01_0005.html index 6f1a0004b..35dd222c2 100644 --- a/docs/dli/umn/dli_01_0005.html +++ b/docs/dli/umn/dli_01_0005.html @@ -1,46 +1,45 @@ -

          Creating a Database or a Table

          -

          Definition of Database and Table in DLI

          A database, built on the computer storage device, is a data warehouse where data is organized, stored, and managed based on its structure.

          +

          Creating a Database and Table on the DLI Console

          +

          A database, built on the computer storage device, is a data warehouse where data is organized, stored, and managed based on its structure.

          The table is an important part of the database. It consists of rows and columns. Each column functions as a field. Each value in a field (column) represents a type of data.

          The database is a framework and the table contains data content. A database has one or more tables.

          You can create databases and tables on the management console or using SQL statements. This section describes how to create a database and a table on the management console.

          A view can be created only by using SQL statements. You cannot create a view on the Create Table page.

          -

          Precautions

          • If a folder and a file have the same name in the OBS directory, the file path is preferred as the path of the OBS table to be created.

          Creating a Database

          1. You can create a database on either the Data Management page or the SQL Editor page.

            • To create a database on the Data Management page:
              1. On the left of the management console, choose Data Management > Databases and Tables.
              2. In the upper right corner of the Databases and Tables page, click Create Database to create a database.
            • To create a database on the SQL Editor page:
              1. On the left of the management console, click SQL Editor.
              2. In the navigation pane on the left, click next to Databases.

          2. In the displayed Create Database dialog box, specify Name and Description by referring to Table 1.

            -

            Table 1 Description

            Parameter

            +
            - - - - - - - @@ -52,174 +51,174 @@

            Creating a Table

            Before creating a table, ensure that a database has been created.

            -
            1. You can create a table on either the Databases and Tables page or the SQL Editor page.

              Datasource connection tables, such as View tables, HBase (MRS) tables, OpenTSDB (MRS) tables, GaussDB(DWS) tables, RDS tables, and CSS tables, cannot be created. You can use SQL to create views and datasource connection tables. For details, see sections Creating a View and Creating a Datasource Connection Table in the Data Lake Insight SQL Syntax Reference.

              +
              1. You can create a table on either the Databases and Tables page or the SQL Editor page.

                Datasource connection tables, such as View tables, HBase (MRS) tables, OpenTSDB (MRS) tables, GaussDB(DWS) tables, RDS tables, and CSS tables, cannot be created. You can use SQL to create views and datasource connection tables. For details, see sections Creating a View and Creating a Datasource Connection Table in the Data Lake Insight SQL Syntax Reference.

                -
                • To create a table on the Data Management page:
                  1. On the left of the management console, choose Data Management > Databases and Tables.
                  2. On the Databases and Tables page, select the database for which you want to create a table. In the Operation column, click More > Create Table to create a table in the current database.
                  +
                  • To create a table on the Data Management page:
                    1. On the left of the management console, choose Data Management > Databases and Tables.
                    2. On the Databases and Tables page, select the database for which you want to create a table. In the Operation column, click More > Create Table to create a table in the current database.
                  • To create a table on the SQL Editor page:
                    1. On the left of the management console, click SQL Editor.
                    2. In the navigation pane of the displayed SQL Editor page, click Databases. You can create a table in either of the following ways:
                      • Click a database name. In the Tables area, click on the right to create a table in the current database.
                      • Click on the right of the database and choose Create Table from the shortcut menu to create a table in the current database.

                • In the displayed Create Table dialog box, set parameters as required.

                  • If you set Data Location to DLI, set related parameters by referring to Table 2.
                  • If you set Data Location to OBS, set related parameters by referring to Table 2 and Table 3. -
            Table 1 Description

            Parameter

            Description

            +

            Description

            Database Name

            +

            Database Name

            • The database name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
            • The database name is case insensitive and cannot be left blank.
            • The length of the database name cannot exceed 128 characters.
            +
            • Only numbers, letters, and underscores (_) are allowed. The value cannot contain only numbers or start with an underscore (_).
            • The database name is case insensitive and cannot be left blank.
            • The value can contain a maximum of 128 characters.
            NOTE:

            The default database is a built-in database. You cannot create the default. database.

            Description

            +

            Description

            Description of a database.

            +

            Description of a database.

            Tag

            +

            Tags

            Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

            +

            Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

            NOTE:
            • A maximum of 20 tags can be added.
            • Only one tag value can be added to a tag key.
            • The key name in each resource must be unique.
            -
            • Tag key: Enter a tag key name in the text box.
              NOTE:

              A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

              +
              • Tag key: Enter a tag key name in the text box.
                NOTE:

                A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

              -
              • Tag value: Enter a tag value in the text box.
                NOTE:

                A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                +
                • Tag value: Enter a tag value in the text box.
                  NOTE:

                  A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
            Table 2 Common parameters

            Parameter

            +
            - - - - - - - - - - - - - - - - - - - - - - - - - -
            Table 2 Common parameters

            Parameter

            Description

            +

            Description

            Example

            +

            Example

            Table Name

            +

            Table Name

            • The table name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
            • The table name is case insensitive and cannot be left unspecified.
            • The table name can contain the dollar sign ($). An example value is $test.
            • The length of the table name cannot exceed 128 characters.
            +
            • Only numbers, letters, and underscores (_) are allowed. The value cannot contain only numbers or start with an underscore (_).
            • The table name is case insensitive and cannot be left unspecified.
            • The table name can contain the dollar sign ($). An example value is $test.
            • The value can contain a maximum of 128 characters.

            table01

            +

            table01

            Data Location

            +

            Data Location

            Data storage location. Currently, DLI and OBS are supported.

            +

            Data storage location. Currently, DLI and OBS are supported.

            DLI

            +

            DLI

            Description

            +

            Description

            Description of the table.

            +

            Description of the table.

            -

            +

            -

            Column Type

            +

            Column Type

            Available values: Normal or Partition

            +

            Available values: Normal or Partition

            Normal

            +

            Normal

            Column

            +

            Column

            Name of a column in a table. The column name must contain at least one letter and can contain underscores (_). It cannot contain only digits.

            +

            Name of a column in a table. The column name must contain at least one letter and can contain underscores (_). It cannot contain only numbers.

            You can select Normal or Partition. Partition columns are dedicated to partition tables. User data is partitioned to improve query efficiency.

            NOTE:

            The column name is case-insensitive and must be unique.

            name

            +

            name

            Type

            +

            Type

            Data type of a column. This parameter corresponds to Column Name.

            -
            • string: The data is of the string type.
            • int: Each integer is stored on four bytes.
            • date: The value ranges from 0000-01-01 to 9999-12-31.
            • double: Each number is stored on eight bytes.
            • boolean: Each value is stored on one byte.
            • decimal: The valid bits are positive integers between 1 to 38, including 1 and 38. The decimal digits are integers less than 10.
            • smallint/short: The number is stored on two bytes.
            • bigint/long: The number is stored on eight bytes.
            • timestamp: The data indicates a date and time. The value can be accurate to six decimal points.
            • float: Each number is stored on four bytes.
            • tinyint: Each number is stored on one byte. Only OBS tables support this data type.
            +

            Data type of a column. This parameter corresponds to Column Name.

            +
            • string: The data is of the string type.
            • int: Each integer is stored on four bytes.
            • date: The value ranges from 0000-01-01 to 9999-12-31.
            • double: Each number is stored on eight bytes.
            • boolean: Each value is stored on one byte.
            • decimal: The valid bits are positive integers between 1 to 38, including 1 and 38. The decimal digits are integers less than 10.
            • smallint/short: The number is stored on two bytes.
            • bigint/long: The number is stored on eight bytes.
            • timestamp: The data indicates a date and time. The value can be accurate to six decimal points.
            • float: Each number is stored on four bytes.
            • tinyint: Each number is stored on one byte. Only OBS tables support this data type.

            string

            +

            string

            Column Description

            +

            Column Description

            Description of a column.

            +

            Description of a column.

            -

            +

            -

            Operation

            +

            Operation

            • Add Column
            • Delete
              NOTE:

              If the table to be created includes a great number of columns, you are advised to use SQL statements to create the table or import column information from the local EXCEL file.

              +
            • Add Column
            • Delete
              NOTE:

              If the table to be created includes a great number of columns, you are advised to use SQL statements to create the table or import column information from the local EXCEL file.

            -

            +

            -
            -
            Table 3 Parameter description when Data Location is set to OBS

            Parameter

            +
            - - - - - - - - - - - - - - - - - - - - - - - - - -
            Table 3 Parameter description when Data Location is set to OBS

            Parameter

            Description

            +

            Description

            Example

            +

            Example

            Data Format

            +

            Data Format

            DLI supports the following data formats:

            -
            • Parquet: DLI can read non-compressed data or data that is compressed using Snappy and gzip.
            • CSV: DLI can read non-compressed data or data that is compressed using gzip.
            • ORC: DLI can read non-compressed data or data that is compressed using Snappy.
            • JSON: DLI can read non-compressed data or data that is compressed using gzip.
            • Avro: DLI can read uncompressed Avro data.
            +

            DLI supports the following data formats:

            +
            • Parquet: DLI can read non-compressed data or data that is compressed using Snappy and gzip.
            • CSV: DLI can read non-compressed data or data that is compressed using gzip.
            • ORC: DLI can read non-compressed data or data that is compressed using Snappy.
            • JSON: DLI can read non-compressed data or data that is compressed using gzip.
            • Avro: DLI can read uncompressed Avro data.

            CSV

            +

            CSV

            Storage Path

            +

            Storage Path

            Enter or select an OBS path. The path can be a folder or a path.

            +

            Enter or select an OBS path. The path can be a folder or a path.

            NOTE:

            If you need to import data stored in OBS to the OBS table, set this parameter to the path of a folder. If the table creation path is a file, data fails to be imported.

            obs://obs1/sampledata.csv

            +

            obs://obs1/sampledata.csv

            Table Header: No/Yes

            +

            Table Header: No/Yes

            This parameter is valid only when Data Format is set to CSV. Whether the data source to be imported contains the table header.

            -

            Click Advanced Settings and select the check box next to Table Header: No. If the check box is selected, the table header is displayed. If the check box is deselected, no table header is displayed.

            +

            This parameter is valid only when Data Format is set to CSV. Whether the data source to be imported contains the table header.

            +

            Click Advanced Settings and select the checkbox next to Table Header: No. If the checkbox is selected, the table header is displayed. If the checkbox is deselected, no table header is displayed.

            -

            +

            -

            User-defined Delimiter

            +

            User-defined Delimiter

            This parameter is valid only when Data Format is set to CSV and you select User-defined Delimiter.

            +

            This parameter is valid only when Data Format is set to CSV and you select User-defined Delimiter.

            The following delimiters are supported:

            • Comma (,)
            • Vertical bar (|)
            • Tab character (\t)
            • Others: Enter a user-defined delimiter.

            Comma (,)

            +

            Comma (,)

            User-defined Quotation Character

            +

            User-defined Quotation Character

            This parameter is valid only when Data Format is set to CSV and you select User-defined Quotation Character.

            +

            This parameter is valid only when Data Format is set to CSV and you select User-defined Quotation Character.

            The following quotation characters are supported:

            • Single quotation mark (')
            • Double quotation marks (")
            • Others: Enter a user-defined quotation character.

            Single quotation mark (')

            +

            Single quotation mark (')

            User-defined Escape Character

            +

            User-defined Escape Character

            This parameter is valid only when Data Format is set to CSV and you select User-defined Escape Character.

            +

            This parameter is valid only when Data Format is set to CSV and you select User-defined Escape Character.

            The following escape characters are supported:

            • Backslash (\)
            • Others: Enter a user-defined escape character.

            Backslash (\)

            +

            Backslash (\)

            Date Format

            +

            Date Format

            This parameter is valid only when Data Format is set to CSV or JSON.

            +

            This parameter is valid only when Data Format is set to CSV or JSON.

            This parameter specifies the format of the date in the table and is valid only Advanced Settings is selected. The default value is yyyy-MM-dd. For definition of characters involved in the date pattern, see Table 3 in .

            2000-01-01

            +

            2000-01-01

            Timestamp Format

            +

            Timestamp Format

            This parameter is valid only when Data Format is set to CSV or JSON.

            +

            This parameter is valid only when Data Format is set to CSV or JSON.

            This parameter specifies the format of the timestamp in the table and is valid only Advanced Settings is selected. The default value is yyyy-MM-dd HH:mm:ss. For definition of characters involved in the time pattern, see Table 3 in .

            2000-01-01 09:00:00

            +

            2000-01-01 09:00:00
            -

          3. Click OK.

            After a table is created, you can view and select the table for use on the Data Management page or SQL Editor page.

            +

          4. Click OK.

            After a table is created, you can view and select the table for use on the Data Management page or SQL Editor page.

          5. (Optional) After a DLI table is created, you can decide whether to directly import data to the table.
            6. -
            Parent topic: Databases and Tables
            +
            Parent topic: Creating Databases and Tables
            diff --git a/docs/dli/umn/dli_01_0006.html b/docs/dli/umn/dli_01_0006.html index 0973a4bb7..9fe5f682f 100644 --- a/docs/dli/umn/dli_01_0006.html +++ b/docs/dli/umn/dli_01_0006.html @@ -5,19 +5,48 @@

            For example, to connect DLI to the MRS, RDS, CSS, Kafka, or GaussDB(DWS) data source, you need to enable the network between DLI and the VPC of the data source.

            Create an enhanced datasource connection on the console.

            -

            Constraints

            • Datasource connections cannot be created for the default queue.
            • Flink jobs can directly access DIS, OBS, and SMN data sources without using datasource connections.
            • VPC Administrator permissions are required for enhanced connections to use VPCs, subnets, routes, VPC peering connections.
            • If you use an enhanced datasource connection, the CIDR block of the elastic resource pool or queue cannot overlap with that of the data source.
            • Only queues bound with datasource connections can access datasource tables.
            • Datasource tables do not support the preview function.
            • When checking the connectivity of datasource connections, the notes and constraints on IP addresses are:
              • The IP address must be valid, which consists of four decimal numbers separated by periods (.). The value ranges from 0 to 255.
              • During the test, you can add a port after the IP address and separate them with colons (:). The port can contain a maximum of five digits. The value ranges from 0 to 65535.

                For example, 192.168.xx.xx or 192.168.xx.xx:8181.

                +

                Notes and Constraints

                +
                + + + + + + + + + + + + + + + + +
                Table 1 Notes and constraints on enhanced datasource connections

                Item

                +

                Description

                +

                Use case

                +
                • Datasource connections cannot be created for the default queue.
                • Flink jobs can directly access DIS, OBS, and SMN data sources without using datasource connections.
                +

                Permission

                +
                • VPC Administrator permissions are required for enhanced connections to use VPCs, subnets, routes, VPC peering connections.
                +

                Usage

                +
                • If you use an enhanced datasource connection, the CIDR block of the elastic resource pool or queue cannot overlap with that of the data source.
                • Only queues bound with datasource connections can access datasource tables.
                • Datasource tables do not support the preview function.
                +

                Connectivity check

                +
                • When checking the connectivity of datasource connections, the notes and constraints on IP addresses are:
                  • The IP address must be valid, which consists of four decimal numbers separated by periods (.). The value ranges from 0 to 255.
                  • During the test, you can add a port after the IP address and separate them with colons (:). The port can contain a maximum of five digits. The value ranges from 0 to 65535.

                    For example, 192.168.xx.xx or 192.168.xx.xx:8181.

                  -
                • When checking the connectivity of datasource connections, the notes and constraints on domain names are:
                  • The domain name can contain 1 to 255 characters. Only letters, digits, underscores (_), and hyphens (-) are allowed.
                  • The top-level domain name must contain at least two letters, for example, .com, .net, and .cn.
                  • During the test, you can add a port after the domain name and separate them with colons (:). The port can contain a maximum of five digits. The value ranges from 0 to 65535.

                    For example, example.com:8080.

                    +
                  • When checking the connectivity of datasource connections, the notes and constraints on domain names are:
                    • The domain name can contain 1 to 255 characters. Only letters, numbers, underscores (_), and hyphens (-) are allowed.
                    • The top-level domain name must contain at least two letters, for example, .com, .net, and .cn.
                    • During the test, you can add a port after the domain name and separate them with colons (:). The port can contain a maximum of five digits. The value ranges from 0 to 65535.

                      Example: example.com:8080

                  +
                +

                Process

                Figure 1 Enhanced datasource connection creation flowchart

                Prerequisites

                • An elastic resource pool or queue has been created.
                • You have obtained the VPC, subnet, private IP address, port, and security group information of the external data source.
                • The security group of the external data source has allowed access from the CIDR block of the elastic resource pool or queue.
                -

                Procedure

                1. Create an Enhanced Datasource Connection

                  1. Log in to the DLI management console.
                  2. In the left navigation pane, choose Datasource Connections.
                  3. On the Enhanced tab page displayed, click Create.

                    Configure parameters according to Table 1.

                    +

                    Procedure

                    1. Create an Enhanced Datasource Connection

                      1. Log in to the DLI management console.
                      2. In the navigation pane on the left, choose Datasource Connections.
                      3. On the displayed Enhanced tab, click Create.

                        Set parameters based on Table 2.

                        -
                        Table 1 Parameters

                        Parameter

                        +
                        @@ -26,7 +55,7 @@ @@ -76,9 +105,9 @@
                      4. Click OK.

                        After the creation is complete, the enhanced datasource connection is in the Active state, indicating that the connection is successfully created.

                      5. Security Group Where the Data Source Belongs Allows Access from the CIDR Block of the Elastic Resource Pool

                        1. On the DLI management console, obtain the network segment of the elastic resource pool or queue.

                          Choose Resources > Queue Management from the left navigation pane. On the page displayed, locate the queue on which jobs are running, and click the button next to the queue name to obtain the CIDR block of the queue.

                          -
                        2. Log in to the VPC console and find the VPC the data source belongs to.
                        3. On the network console, choose Virtual Private Cloud > Network Interfaces. On the Network Interfaces tab page displayed, search for the security group name, click More in the Operation column, and select Change Security Group.
                        4. In the navigation pane on the left, choose Access Control > Security Groups.
                        5. Click the name of the security group to which the external data source belongs.
                        6. Click the Inbound Rules tab and add a rule to allow access from the CIDR block of the queue.

                          Configure the inbound rule parameters according to Table 2.

                          +
                        7. Log in to the VPC console and find the VPC the data source belongs to.
                        8. On the network console, choose Virtual Private Cloud > Network Interfaces. On the Network Interfaces tab page displayed, search for the security group name, click More in the Operation column, and select Change Security Group.
                        9. In the navigation pane on the left, choose Access Control > Security Groups.
                        10. Click the name of the security group to which the external data source belongs.
                        11. Click the Inbound Rules tab and add a rule to allow access from the CIDR block of the queue.

                          Configure the inbound rule parameters according to Table 3.

                          -
                        6. Table 2 Parameters

                        Parameter

                        Description

                        Connection Name

                        Name of the created datasource connection.

                        -
                        • The name can contain only letters, digits, and underscores (_). The parameter must be specified.
                        • A maximum of 64 characters are allowed.
                        +
                        • Only letters, numbers, and underscores (_) are allowed. The parameter must be specified.
                        • A maximum of 64 characters are allowed.

                        Resource Pool

                        @@ -62,10 +91,10 @@

                        Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                        NOTE:
                        • A maximum of 20 tags can be added.
                        • Only one tag value can be added to a tag key.
                        • The key name in each resource must be unique.
                        -
                        • Tag key: Enter a tag key name in the text box.
                          NOTE:

                          A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                          +
                          • Tag key: Enter a tag key name in the text box.
                            NOTE:

                            A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                          -
                          • Tag value: Enter a tag value in the text box.
                            NOTE:

                            A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                            +
                            • Tag value: Enter a tag value in the text box.
                              NOTE:

                              A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                        Table 2 Inbound rule parameters

                        Parameter

                        +
                        @@ -143,7 +172,7 @@ diff --git a/docs/dli/umn/dli_01_0007.html b/docs/dli/umn/dli_01_0007.html index 263204d85..bbf784e8b 100644 --- a/docs/dli/umn/dli_01_0007.html +++ b/docs/dli/umn/dli_01_0007.html @@ -1,6 +1,6 @@ -

                        Previewing Data

                        +

                        Previewing Table Data on the DLI Console

                        The Preview page displays the first 10 records in the table.

                        Procedure

                        You can preview data on either the Data Management page or the SQL Editor page.
                        • To preview data on the Data Management page:
                          1. On the left of the management console, choose Data Management > Databases and Tables.
                          2. On the displayed Data Management page, click the name of the database where the target table whose data you want to export resides to switch to the Manage Tables page.
                          3. Click More in the Operation column of the target table and select View Properties.
                          4. Click the Preview tab to preview the table data.
                        • To preview data on the SQL Editor page:
                          1. On the left of the management console, click SQL Editor.
                          2. In the navigation pane of the displayed SQL Editor page, click Databases.
                          3. Click the corresponding database name to view the tables in the database.
                          4. Click on the right of the corresponding table, choose View Properties from the list menu, and click the Preview tab to preview the data of the table.
                          @@ -10,7 +10,7 @@
                        diff --git a/docs/dli/umn/dli_01_0008.html b/docs/dli/umn/dli_01_0008.html index 21f5410a6..127a8e145 100644 --- a/docs/dli/umn/dli_01_0008.html +++ b/docs/dli/umn/dli_01_0008.html @@ -1,6 +1,6 @@ -

                        Viewing Metadata

                        +

                        Viewing Table Metadata

                        Metadata Description

                        • Metadata is used to define data types. It describes information about the data, including the source, size, format, and other data features. In database fields, metadata interprets data content in the data warehouse.
                        • When you create a table, metadata is defined, consisting of the column name, type, and description.
                        • The Metadata page displays information about the target table, including Column Name, Column Type, Data Type, and Description.

                        Procedure

                        You can view metadata on either the Data Management page or the SQL Editor page.
                        • To view metadata on the Data Management page:
                          1. On the left of the management console, choose Data Management > Databases and Tables.
                          2. On the displayed Data Management page, click the name of the database where the target table whose data you want to export resides to switch to the Manage Tables page.
                          3. Click More in the Operation column of the target table and select View Properties. In the Metadata tab, view the metadata of the table.
                          @@ -11,7 +11,7 @@
                        -
                        Parent topic: Databases and Tables
                        +
                        Parent topic: Creating Databases and Tables
                        diff --git a/docs/dli/umn/dli_01_0010.html b/docs/dli/umn/dli_01_0010.html index 9a3e62870..b64a19cf7 100644 --- a/docs/dli/umn/dli_01_0010.html +++ b/docs/dli/umn/dli_01_0010.html @@ -1,6 +1,6 @@ -

                        Exporting Data from DLI to OBS

                        +

                        Exporting DLI Table Data to OBS

                        You can export data from a DLI table to OBS. During the export, a folder is created in OBS or the content in the existing folder is overwritten.

                        Precautions

                        • The exported file can be in JSON format, and the text format can only be UTF-8.
                        • Only the data in the DLI table (the table type is Managed) can be exported to the OBS bucket, and the export path must be specified to the folder level.
                        • Data can be exported across accounts. That is, after account B authorizes account A, account A can export data to the OBS path of account B if account A has the permission to read the metadata and permission information about the OBS bucket of account B and read and write the path.
                        @@ -66,7 +66,7 @@
                        diff --git a/docs/dli/umn/dli_01_0011.html b/docs/dli/umn/dli_01_0011.html index 87400259d..9edb98166 100644 --- a/docs/dli/umn/dli_01_0011.html +++ b/docs/dli/umn/dli_01_0011.html @@ -1,16 +1,11 @@ -

                        Deleting a Database or a Table

                        -

                        You can delete unnecessary databases and tables based on actual conditions.

                        -

                        Precautions

                        • You are not allowed to delete databases or tables that are being used for running jobs.
                        • The administrator, database owner, and users with the database deletion permission can delete the database. The administrator, database owner, and users with the table deletion permission can delete the table.

                          If a database or table is deleted, it cannot be recovered. Exercise caution when performing this operation.

                          +

                          Deleting a Database on the DLI Console

                          +

                          You can delete databases as needed.

                          +

                          Precautions

                          • You are not allowed to delete databases or tables that are being used for running jobs.
                          • The administrator, database owner, and users with the database deletion permission can delete the database.

                            If a database or table is deleted, it cannot be recovered. Exercise caution when performing this operation.

                          -

                          Deleting a Table

                          You can delete a table on either the Data Management page or the SQL Editor page.
                          • Delete the table on the Data Management page.
                            1. On the left of the management console, choose Data Management > Databases and Tables.
                            2. Locate the row where the database whose tables you want to delete, click the database name to switch to the Table Management page.
                            3. Locate the row where the target table locates and click More > Delete in the Operation column.
                            4. In the displayed dialog box, click Yes.
                            -
                          • Delete a table on the SQL Editor page.
                            1. On the top menu bar of the DLI management console, click SQL Editor.
                            2. In the navigation tree on the left, click Databases. Click the name of a database where the table you want belongs. The tables of the selected database are displayed.
                            3. Click on the right of the table and choose Delete from the shortcut menu.
                            4. In the dialog box that is displayed, click OK.
                            -
                          -
                          -

                          Deleting a Database

                          1. On the left of the management console, choose Data Management > Databases and Tables.
                          2. Locate the row where the target database locates and click More > Drop Database in the Operation column.

                            You cannot delete databases that contain tables. To delete a database containing tables, delete the tables first.

                          3. In the displayed dialog box, click Yes.
                          @@ -18,7 +13,7 @@
                          diff --git a/docs/dli/umn/dli_01_0012.html b/docs/dli/umn/dli_01_0012.html deleted file mode 100644 index 88f5d69f0..000000000 --- a/docs/dli/umn/dli_01_0012.html +++ /dev/null @@ -1,33 +0,0 @@ - - -

                          Queue Management

                          -
                          - - diff --git a/docs/dli/umn/dli_01_0013.html b/docs/dli/umn/dli_01_0013.html index c2d3396bc..dad68476b 100644 --- a/docs/dli/umn/dli_01_0013.html +++ b/docs/dli/umn/dli_01_0013.html @@ -25,7 +25,7 @@
                          diff --git a/docs/dli/umn/dli_01_0014.html b/docs/dli/umn/dli_01_0014.html index 649c1fc17..c3b0062ed 100644 --- a/docs/dli/umn/dli_01_0014.html +++ b/docs/dli/umn/dli_01_0014.html @@ -1,6 +1,6 @@ -

                          Adding a Route

                          +

                          Adding a Route for an Enhanced Datasource Connection

                          Scenario

                          A route is configured with the destination, next hop type, and next hop to determine where the network traffic is directed. Routes are classified into system routes and custom routes.

                          After an enhanced connection is created, the subnet is automatically associated with the default route. You can add custom routes as needed to forward traffic destined for the destination to the specified next hop.

                          • When an enhanced connection is created, the associated route table is the one associated with the subnet of the data source.
                          • The route to be added in the Add Route dialog box must be one in the route table associated with the subnet of the resource pool.
                          • The subnet of the data source must be different from that used by the resource pool. Otherwise, a network segment conflict occurs.
                          @@ -35,7 +35,7 @@
                          diff --git a/docs/dli/umn/dli_01_0015.html b/docs/dli/umn/dli_01_0015.html index 7c343f561..9c512a3e9 100644 --- a/docs/dli/umn/dli_01_0015.html +++ b/docs/dli/umn/dli_01_0015.html @@ -1,27 +1,31 @@

                          Queue Permission Management

                          -

                          Scenario

                          • You can isolate queues allocated to different users by setting permissions to ensure data query performance.
                          • The administrator and queue owner have all permissions, which cannot be set or modified by other users.
                          +

                          Administrators and queue owners have full operation permissions on queues. They can grant operation permissions to other users based on service needs. This ensures that users can execute their jobs independently without any impact on the performance of other users' job execution. This section describes how to manage queue permissions.

                          +

                          Operation Precautions

                          • The administrator and queue owner have all permissions, which cannot be set or modified by other users.
                          • When setting queue permissions for a new user, ensure that the region of the user group to which the user belongs has the Tenant Guest permission.

                            +

                            +

                            +
                          -

                          Operations

                          1. On the top menu bar of the DLI management console, click Resources > Queue Management.
                          2. Select the queue to be configured and choose Manage Permissions in the Operation column. The User Permission Info area displays the list of users who have permissions on the queue.

                            You can assign queue permissions to new users, modify permissions for users who have some permissions of a queue, and revoke all permissions of a user on a queue.

                            -
                            • Assign permissions to a new user.
                              A new user does not have permissions on the queue.
                              1. Click Set Permission on the right of User Permissions page. The Set Permission dialog box is displayed.
                              2. Specify Username and select corresponding permissions.
                              3. Click OK.

                                Table 1 describes the related parameters.

                                +

                                Operations

                                1. On the top menu bar of the DLI management console, click Resources > Queue Management.
                                2. Select the queue to be configured and choose Manage Permissions in the Operation column. The User Permission Info area displays the list of users who have permissions on the queue.

                                  You can grant queue permissions to new users, modify permissions for users who have some permissions on a queue, and revoke all permissions for a user on a queue.

                                  +
                                  • Grant permissions to a new user.
                                    A new user does not have permissions on the queue.
                                    1. Click Set Permission on the right of User Permissions page. The Set Permission dialog box is displayed.
                                    2. Specify Username and select corresponding permissions.
                                    3. Click OK.

                                      Table 1 describes the related parameters.

                                      -
                        Table 3 Inbound rule parameters

                        Parameter

                        Description
                        Table 1 Parameter description

                        Parameter

                        +
                        - - - - - @@ -29,16 +33,16 @@ -
                      6. To assign or revoke permissions of a user who has some permissions on the queue, perform the following steps:
                        1. In the list under User Permission Info for a queue, select the user whose permissions need to be modified and click Set Permission in the Operation column.
                        2. In the displayed Set Permission dialog box, modify the permissions of the current user. Table 1 lists the detailed permission descriptions.

                          If all options under Set Permission are gray, you are not allowed to change permissions on this queue. You can apply to the administrator, queue owner, or other authorized users for queue permission granting and revoking.

                          +
                        3. To grant or revoke permissions for a user who already has certain permissions on a queue, perform the following steps:
                          1. In the list under User Permission Info for a queue, select the user whose permissions need to be modified and click Set Permission in the Operation column.
                          2. In the displayed Set Permission dialog box, modify the permissions of the current user. Table 1 lists the detailed permission descriptions.

                            If all options under Set Permission are gray, you are not allowed to change permissions on this queue. You can apply to the administrator, queue owner, or other authorized users for queue permission granting and revoking.

                          3. Click OK.
                          -
                        4. To revoke all permission of a user on the queue, perform the following steps:

                          In the user list under Permission Info, select the user whose permission needs to be revoked and click Revoke Permission under Operation. In the Revoke Permission dialog box, click OK. All permissions on this queue are revoked.

                          +
                        5. To revoke all permissions for a user on a queue, perform the following steps:

                          In the user list under Permission Info, select the user whose permission needs to be revoked and click Revoke Permission under Operation. In the Revoke Permission dialog box, click OK. All permissions on this queue are revoked.

                        -
                        Parent topic: Queue Management
                        +
                        Parent topic: Managing Queues
                        diff --git a/docs/dli/umn/dli_01_0016.html b/docs/dli/umn/dli_01_0016.html index f99b73a52..fadc18c45 100644 --- a/docs/dli/umn/dli_01_0016.html +++ b/docs/dli/umn/dli_01_0016.html @@ -4,14 +4,14 @@

                        You can delete a queue based on actual conditions.

                        • This operation will fail if there are jobs in the Submitting or Running state on this queue.
                        • Deleting a queue does not cause table data loss in your database.
                        -

                        Procedure

                        1. On the left of the DLI management console, click Resources >Queue Management.
                        2. Locate the row where the target queue locates and click Delete in the Operation column.

                          If Delete in the Operation column is gray, the current user does not have the permission of deleting the queue. You can apply to the administrator for the permission.

                          +

                          Procedure

                          1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                          2. Locate the row where the target queue locates and click Delete in the Operation column.

                            If Delete in the Operation column is gray, the current user does not have the permission of deleting the queue. You can apply to the administrator for the permission.

                          3. In the displayed dialog box, click OK.
                          -
                          Parent topic: Queue Management
                          +
                          Parent topic: Managing Queues
                          diff --git a/docs/dli/umn/dli_01_0017.html b/docs/dli/umn/dli_01_0017.html index fe1a9b054..f609b07a3 100644 --- a/docs/dli/umn/dli_01_0017.html +++ b/docs/dli/umn/dli_01_0017.html @@ -1,59 +1,56 @@ -

                          SQL Job Management

                          -

                          SQL jobs allow you to execute SQL statements in the SQL job editing window, import data, and export data.

                          -

                          SQL job management provides the following functions:

                          - -

                          SQL Jobs Page

                          On the Overview page of the DLI console, click SQL Jobs to go to the SQL job management page. Alternatively, you can click Job Management > SQL Jobs. The job list displays all SQL jobs. If there are a large number of jobs, they will be displayed on multiple pages. You can switch to the specified page as needed. DLI allows you to view jobs in all statuses. By default, jobs in the job list are displayed in descending order of the job creation time.

                          +

                          Managing SQL Jobs

                          +

                          Viewing Basic Job Information on the SQL Jobs Page

                          The SQL Jobs page displays all SQL jobs, which may span multiple pages if there are many jobs. You can navigate to a specific page as needed. DLI allows you to view jobs in all statuses. Jobs in the job list are sorted by creation time in descending order by default.

                          -
                        7. Table 1 Parameter description

                        Parameter

                        Description

                        +

                        Description

                        Username

                        +

                        Username

                        Name of the authorized user.

                        +

                        Name of the authorized user.

                        NOTE:

                        The username is an existing IAM user name and has logged in to the DLI management console.

                        Permission Settings

                        +

                        Permission Settings

                        • Delete Queues: This permission allows you to delete the queue.
                        • Submit Jobs: This permission allows you to submit jobs using this queue.
                        • Terminate Jobs: This permission allows you to terminate jobs submitted using this queue.
                        • Grant Permission: This permission allows you to grant queue permissions to other users.
                        • Revoke Permission: This permission allows you to revoke the queue permissions that other users have but cannot revoke the queue owner's permissions.
                        • View Other User's Permissions: This permission allows you to view the queue permissions of other users.
                        • Restart Queues: This permission allows you to restart queues.
                        • Modify Queue Specifications: This permission allows you to modify queue specifications.
                        +
                        • Delete Queues: This permission allows you to delete the queue.
                        • Submit Jobs: This permission allows you to submit jobs using this queue.
                        • Terminate Jobs: This permission allows you to terminate jobs submitted using this queue.
                        • Grant Permission: This permission allows you to grant queue permissions to other users.
                        • Revoke Permission: This permission allows you to revoke the queue permissions that other users have but cannot revoke the queue owner's permissions.
                        • View Other User's Permissions: This permission allows you to view the queue permissions of other users.
                        • Restart Queues: This permission allows you to restart queues.
                        • Modify Queue Specifications: This permission allows you to modify queue specifications.
                        @@ -43,7 +43,7 @@ diff --git a/docs/dli/umn/dli_01_0020.html b/docs/dli/umn/dli_01_0020.html new file mode 100644 index 000000000..03c857a7f --- /dev/null +++ b/docs/dli/umn/dli_01_0020.html @@ -0,0 +1,441 @@ + + +

                        Using DLI to Submit a SQL Job to Query RDS for MySQL Data

                        +

                        Scenario

                        DLI can query data stored in RDS. This section describes how to use DLI to submit a SQL job to query RDS for MySQL data.

                        +

                        In this example, we will create an RDS for MySQL DB instance, create a database and table, create a DLI elastic resource pool and add a queue to it, create an enhanced datasource connection to connect the DLI elastic resource pool and the RDS for MySQL DB instance, and submit a SQL job to access the data in the RDS table.

                        +
                        +

                        Procedure

                        Table 1 shows the process for submitting a SQL job to query RDS for MySQL data.

                        + +
                        Table 1 SQL Job management parameters

                        Parameter

                        +
                        - - - - - - - - - - - - - - - - -
                        Table 1 SQL Job management parameters

                        Parameter

                        Description

                        +

                        Description

                        Queues

                        +

                        Queues

                        Name of the queue to which a job belongs

                        +

                        Name of the queue to which a job belongs

                        Username

                        +

                        Username

                        Name of the user who executed the job.

                        +

                        Name of the user who executed the job.

                        Type

                        +

                        Type

                        Job type. The following types are supported:

                        +

                        Job type. The following types are supported:

                        • IMPORT: A job that imports data to DLI
                        • EXPORT: A job that exports data from DLI
                        • DCL: Conventional DCLs and operations related to queue permissions
                        • DDL:Conventional DDLs, including creating and deleting databases and tables
                        • QUERY: A job that queries data by running SQL statements
                        • INSERT: A job that inserts data by running SQL statements
                        • UPDATE: A job that updates data.
                        • DELETE: A job that deletes a SQL job.
                        • DATA_MIGRATION: A job that migrates data.
                        • RESTART_QUEUE: A job that restarts a queue.
                        • SCALE_QUEUE: A job that changes queue specifications, including sale-out and scale-in.

                        Status

                        +

                        Status

                        Job status. Possible values are as follows:

                        +

                        Job status. Possible values are as follows:

                        • Submitting
                        • Running
                        • Finished
                        • Canceled
                        • Failed
                        • Scaling

                        Query

                        +

                        Query

                        SQL statements for operations such as exporting and creating tables

                        +

                        SQL statements for operations such as exporting and creating tables

                        You can click to copy the query statement.

                        Duration

                        +

                        Duration

                        Running duration of a job

                        +

                        Running duration of a job

                        Created

                        +

                        Created

                        Time when a job is created. Jobs can be displayed in ascending or descending order of the job creation time.

                        +

                        Time when a job is created. Jobs can be displayed in ascending or descending order of the job creation time.

                        Operation

                        +

                        Operation

                        • Edit: Edit the job.
                        • Cancel
                          • You can terminate a job only when the job is in Submitting or Running status.
                          • A job whose status is Finished, Failed, or Canceled cannot be terminated.
                          • If the Cancel button is gray, you are not allowed to perform this operation.
                          -
                        • Re-execute: Execute the job again.
                        • SparkUI: Display the Spark job execution page.
                          NOTE:
                          • When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click SparkUI before the cluster is created, an empty projectID will be cached. The SparkUI page cannot be displayed. You are advised to use a dedicated queue so that the cluster will not be released. Alternatively, wait for a while after the job is submitted (the cluster is created), and then check SparkUI.
                          • Currently, only the latest 100 job information records are displayed on the SparkUI of DLI.
                          +
                        • Edit: Edit the job.
                        • Cancel
                          • You can terminate a job only when the job is in Submitting or Running status.
                          • A job whose status is Finished, Failed, or Canceled cannot be terminated.
                          • If the Cancel button is gray, you are not allowed to perform this operation.
                          +
                        • Re-execute: Execute the job again.
                        • SparkUI: Display the Spark job execution page.
                          NOTE:
                          • When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click SparkUI before the cluster is created, an empty projectID will be cached. The SparkUI page cannot be displayed. You are advised to use a dedicated queue so that the cluster will not be released. Alternatively, wait for a while after the job is submitted (the cluster is created), and then check SparkUI.
                          • SparkUI can currently only display the latest 100 jobs.
                        • In addition to the preceding operations, the following operations are available for QUERY jobs and asynchronous DDL jobs.
                          • View Result: View the job running result.
                          • Export Result: Export the job running result to the created OBS bucket.
                          @@ -67,77 +64,21 @@
                        -

                        Searching for a Job

                        On the SQL Jobs page, you can search jobs with any of the following operations.

                        -
                        • Select a queue name.
                        • Set the date range.
                        • Enter a username, statement, or job ID.
                        • Select the creation time in ascending or descending order.
                        • Select a job type.
                        • Select a job status.
                        • Select the job execution duration in ascending or descending order.
                        -
                        -

                        Viewing Job Details

                        On the SQL Jobs page, you can click in front of a job record to view details about the job.

                        +

                        Viewing Job Details

                        On the SQL Jobs page, you can click in front of a job record to view details about the job.

                        Job details vary with job types. The job details vary depending on the job types, status, and configuration options. The following describes how to load data, create a table, and select a job. For details about other job types, see the information on the management console.

                        • Load data (job type: IMPORT) include the following information: queue, job ID, username, type, status, execution statement, running duration, creation time, end time, parameter settings, label, number of results, scanned data, number of scanned data, number of error records, storage path, data format, database, table, table header, separator, reference character, escape character, date format, timestamp format, total CPU used, and output bytes.
                        • Create table (job type: DDL) include the following information: queue, job ID, username, type, status, execution statement, running duration, creation time, end time, parameter settings, tags, number of results, scanned data, and database.
                        • Select (job type: QUERY) include the following information: queue, job ID, username, type, status, execution statement, running duration, creation time, end time, parameter setting, label, number of results (results of successful executions can be exported), and scanned data, username, result status (results of successful tasks can be viewed. Failure causes of failed tasks are displayed), database, total CPU used, and output bytes.
                          • Total CPU Used (Core x ms): total CPU used during job execution.
                          • Output Bytes: number of output bytes after the job is executed.
                        -

                        Terminating a Job

                        On the SQL Jobs page, you can click Terminate in the Operation column to stop a submitting or running job.

                        -
                        -

                        Exporting Query Results

                        A maximum of 1000 records can be displayed in the query result on the console. To view more or all data, you can export the data to OBS. The procedure is as follows:

                        -

                        You can export results on the SQL Jobs page or the SQL Editor page.

                        -
                        • On the Job Management > SQL Jobs page, you can click More > Export Result in the Operation column to export the query result.
                        • After the query statements are successfully executed on the SQL Editor page, click next to the View Result tab page to export query results.
                        -

                        If there are no numerical columns in the query results, job results cannot be exported.

                        -
                        -

                        - -
                        - - - - - - - - - - - - - - - - - - - - - - - - - -
                        Table 2 Parameters

                        Parameter

                        -

                        Description

                        -

                        Data Format

                        -

                        Format of the exported query result file. This parameter can be set to json or csv.

                        -

                        Queues

                        -

                        The queue where the jobs are executed. SQL jobs can only be executed on SQL queues. For details about how to create a queue, see Creating a Queue.

                        -

                        Compression Format

                        -

                        Compression format of the data to be exported. The options are:

                        -
                        • none
                        • bzip2
                        • deflate
                        • gzip
                        -

                        Storage Path

                        -

                        OBS path to store the result.

                        -
                        NOTE:
                        • After selecting an OBS bucket, enter a name for the folder. If the folder does not exist, it will be created in OBS.
                        • The folder name cannot contain the special characters of \ / : * ? "< > |, and cannot start or end with a dot (.).
                        -
                        -

                        Export Mode

                        -

                        Mode for saving the exported query result data.

                        -
                        • New OBS directory: If the specified export directory exists, an error is reported and the export operation cannot be performed.
                        • Existing OBS directory (Overwritten): If you create a file in the specified directory, the existing file will be overwritten.
                        -

                        Number of Results

                        -

                        Number of exported query results. If no value is entered or the value is 0, all results are exported.

                        -

                        Table Header

                        -

                        Whether the data to be exported contains table headers.

                        -
                        +

                        Searching for a Job

                        On the SQL Jobs page, you can search jobs with any of the following operations.

                        +
                        • Select a queue name.
                        • Set the date range.
                        • Enter a username, statement, or job ID.
                        • Select the creation time in ascending or descending order.
                        • Select a job type.
                        • Select a job status.
                        • Select the job execution duration in ascending or descending order.
                        +

                        Terminating a SQL Job

                        On the SQL Jobs page, you can click Terminate in the Operation column to stop a submitting or running job.

                        -
                        Parent topic: Job Management
                        +
                        Parent topic: Submitting a SQL Job Using DLI
                        diff --git a/docs/dli/umn/dli_01_0018.html b/docs/dli/umn/dli_01_0018.html index 9ca6ac814..853fd3bfa 100644 --- a/docs/dli/umn/dli_01_0018.html +++ b/docs/dli/umn/dli_01_0018.html @@ -20,7 +20,7 @@
                        diff --git a/docs/dli/umn/dli_01_0019.html b/docs/dli/umn/dli_01_0019.html index cf8a514c2..a0b962d90 100644 --- a/docs/dli/umn/dli_01_0019.html +++ b/docs/dli/umn/dli_01_0019.html @@ -22,7 +22,7 @@
                        • Click the text box and select a predefined tag key from the drop-down list.

                          To add a predefined tag, you need to create one on TMS and then select it from the Tag key drop-down list. You can click View predefined tags to go to the Predefined Tags page of the TMS console. Then, click Create Tag in the upper corner of the page to create a predefined tag.

                        -
                        • Enter a tag key in the text box.
                          NOTE:

                          A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                          +
                          • Enter a tag key in the text box.
                            NOTE:

                            A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                          @@ -30,7 +30,7 @@

                        Tag value

                        You can perform the following operations:

                        -
                        • Click the text box and select a predefined tag value from the drop-down list.
                        • Enter a tag value in the text box.
                          NOTE:

                          A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                          +
                          • Click the text box and select a predefined tag value from the drop-down list.
                          • Enter a tag value in the text box.
                            NOTE:

                            A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                        + + + + + + + + + + + + + + + + + + + + + + +
                        Table 1 Procedure for using DLI to submit a SQL job to query RDS for MySQL data

                        Procedure

                        +

                        Description

                        +

                        Step 1: Create an RDS MySQL Instance

                        +

                        Create an RDS for MySQL instance for the example scenario.

                        +

                        Step 2: Create an RDS Database Table

                        +

                        Log in to the RDS for MySQL DB instance and create a database and a table.

                        +

                        Step 3: Create an Elastic Resource Pool and Add Queues to the Pool

                        +

                        Create compute resources required for submitting jobs.

                        +

                        Step 4: Create an Enhanced Datasource Connection

                        +

                        Create an enhanced datasource connection to connect the DLI elastic resource pool and the RDS for MySQL DB instance.

                        +

                        Step 5: Create a Datasource Authentication

                        +

                        Create a datasource authentication to store the access credentials required by DLI to read from and write to RDS for MySQL data.

                        +

                        Step 6: Submit a SQL Job

                        +

                        Use standard SQL statements to query and analyze data.

                        +
                        +
                        + +

                        Step 1: Create an RDS MySQL Instance

                        In this example, assuming the job name is JobSample, RDS is used as the data source to create an RDS for MySQL DB instance.

                        +

                        For details, see "RDS for MySQL Getting Started" in the Relational Database Service Getting Started.

                        +
                        1. Log in to the RDS management console.
                        2. Select a region and a project in the upper left corner.
                        3. In the navigation pane on the left, choose Instances. On the displayed page, click Buy DB Instance in the upper right corner.
                        4. On the Buy DB Instance page, set instance parameters, and click Buy.

                          Set the parameters listed below based on your service planning.

                          + +
                          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                          Table 2 RDS for MySQL instance parameters

                          Parameter

                          +

                          Description

                          +

                          Example Value

                          +

                          Region

                          +

                          Region where you want to create the instance

                          +

                          _

                          +

                          DB Instance Name

                          +

                          Instance name

                          +

                          rds-demo

                          +

                          DB Engine

                          +

                          MySQL

                          +

                          MySQL

                          +

                          DB Engine Version

                          +

                          If you select MySQL for DB Engine, select an engine version that best suits your service needs. You are advised to select the latest available version for more stable performance, higher security, and greater reliability.

                          +

                          8.0

                          +

                          DB Instance Type

                          +

                          Primary/standby mode of the instance

                          +

                          Single

                          +

                          Storage Type

                          +

                          Determines the instance read/write speed. The higher the maximum throughput, the faster the read and write speeds.

                          +

                          Cloud SSD

                          +

                          AZ

                          +

                          For a single DB instance, you only need to select a single AZ.

                          +

                          -

                          +

                          Time Zone

                          +

                          Select a time zone based on the region you selected. You can change it after the DB instance is created.

                          +

                          Retain the default value.

                          +

                          Instance Class

                          +

                          vCPUs and memory. These instance classes support varying number of connections and maximum IOPS.

                          +

                          2 vCPUs | 4 GB

                          +

                          Storage Space

                          +

                          If the storage type is cloud SSD or extreme SSD, you can enable storage autoscaling. If the available storage drops to a specified threshold, autoscaling is triggered.

                          +

                          40 GB

                          +

                          Disk Encryption

                          +

                          Determine whether to enable disk encryption.

                          +

                          Disable

                          +

                          VPC

                          +

                          Select an existing VPC.

                          +

                          For how to recreate a VPC and subnet, refer to "Creating a VPC and Subnet" in the Virtual Private Cloud User Guide.

                          +
                          NOTE:

                          In datasource scenarios, the CIDR block of the data source cannot overlap that of the elastic resource pool.

                          +
                          +

                          -

                          +

                          Database Port

                          +

                          Port 3306 is used by default.

                          +

                          3306

                          +

                          Security Group

                          +

                          Enhances security by providing rules that control access to RDS from other services.

                          +

                          The security group where the data source is must allow access from the CIDR block of the DLI elastic resource pool.

                          +

                          -

                          +

                          Password

                          +

                          Set a password for logging in to the DB instance.

                          +

                          -

                          +

                          Administrator

                          +

                          root

                          +

                          root

                          +

                          Administrator Password

                          +

                          Administrator password

                          +

                          -

                          +

                          Parameter Template

                          +

                          A template of parameters for creating an instance. The template contains engine configuration values that are applied to one or more instances.

                          +

                          Default-MySQL-5.7

                          +

                          Table Name

                          +

                          Determines whether the table name is case-insensitive.

                          +

                          Case insensitive

                          +

                          Enterprise Project

                          +

                          If the instance has been associated with an enterprise project, select the target project from the Enterprise Project drop-down list.

                          +

                          default

                          +

                          Quantity

                          +

                          Number of instances to buy

                          +

                          1

                          +
                          +
                          +
                        5. Click Next. The confirmation page is displayed.
                        6. Click Submit.
                        7. To view and manage the DB instance, go to the Instance Management page.

                          During the creation process, the DB instance status is Creating. When the creation process is complete, the instance status will change to Available. You can view the detailed progress and result of the task on the Task Center page.

                          +
                        +
                        +

                        Step 2: Create an RDS Database Table

                        1. Log in to the RDS management console.
                        2. In the upper left corner of the management console, select the target region and project.
                        3. On the Instances page, locate the DB instance you just created and record its floating IP address.
                        4. Locate the RDS for MySQL DB instance you created, click More in the Operation column, and select Log In. On the displayed page, enter the username and password for logging in to the instance and click Test Connection. After Connection is successful is displayed, click Log In.
                        5. Click Create Database. In the displayed dialog box, enter database name dli_demo. Then, click OK.
                        6. Click SQL Query and run the following SQL statement to create a table:
                          CREATE TABLE `dli_demo`.`tabletest` (
+	`id` VARCHAR(32) NOT NULL,
+	`name` VARCHAR(32) NOT NULL,
+	PRIMARY KEY (`id`)
+)	ENGINE = InnoDB
+	DEFAULT CHARACTER SET = utf8mb4;
                          +
                        +
                        +

                        Step 3: Create an Elastic Resource Pool and Add Queues to the Pool

                        To execute SQL jobs in datasource scenarios, you must use your own SQL queue as the existing default queue cannot be used. In this example, create an elastic resource pool named dli_resource_pool and a queue named dli_queue_01.

                        +
                        1. Log in to the DLI management console.
                        2. In the navigation pane on the left, choose Resources > Resource Pool.
                        3. On the displayed page, click Buy Resource Pool in the upper right corner.
                        4. On the displayed page, set the parameters.
                          Table 3 describes the parameters. +
                          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                          Table 3 Parameters

                          Parameter

                          +

                          Description

                          +

                          Example Value

                          +

                          Region

                          +

                          Select a region where you want to buy the elastic resource pool.

                          +

                          _

                          +

                          Project

                          +

                          Project uniquely preset by the system for each region

                          +

                          Default

                          +

                          Name

                          +

                          Name of the elastic resource pool

                          +

                          dli_resource_pool

                          +

                          Specifications

                          +

                          Specifications of the elastic resource pool

                          +

                          Standard

                          +

                          CU Range

                          +

                          The maximum and minimum CUs allowed for the elastic resource pool

                          +

                          64-64

                          +

                          CIDR Block

                          +

                          CIDR block the elastic resource pool belongs to. If you use an enhanced datasource connection, this CIDR block cannot overlap that of the data source. Once set, this CIDR block cannot be changed.

                          +

                          172.16.0.0/19

                          +

                          Enterprise Project

                          +

                          Select an enterprise project for the elastic resource pool.

                          +

                          default

                          +
                          +
                          +
                          +
                        5. Click Buy.
                        6. Click Submit.
                        7. In the elastic resource pool list, locate the pool you just created and click Add Queue in the Operation column.
                        8. Set the basic parameters listed below. +
                          + + + + + + + + + + + + + + + + + +
                          Table 4 Basic parameters for adding a queue

                          Parameter

                          +

                          Description

                          +

                          Example Value

                          +

                          Name

                          +

                          Name of the queue to add

                          +

                          dli_queue_01

                          +

                          Type

                          +

                          Type of the queue

                          +
                          • To execute SQL jobs, select For SQL.
                          • To execute Flink or Spark jobs, select For general purpose.
                          +

                          _

                          +

                          Enterprise Project

                          +

                          Select an enterprise project.

                          +

                          default

                          +
                          +
                          +
                        9. Click Next and configure scaling policies for the queue.

                          Click Create to add a scaling policy with varying priority, period, minimum CUs, and maximum CUs.

                          +

                          + +
                          + + + + + + + + + + + + + + + + + + + + + +
                          Table 5 Scaling policy parameters

                          Parameter

                          +

                          Description

                          +

                          Example Value

                          +

                          Priority

                          +

                          Priority of the scaling policy in the current elastic resource pool. A larger value indicates a higher priority. In this example, only one scaling policy is configured, so its priority is set to 1 by default.

                          +

                          1

                          +

                          Period

                          +

                          The first scaling policy is the default policy, and its Period parameter configuration cannot be deleted or modified.

                          +

                          The period for the scaling policy is from 00 to 24.

                          +

                          00–24

                          +

                          Min CU

                          +

                          Minimum number of CUs allowed by the scaling policy

                          +

                          16

                          +

                          Max CU

                          +

                          Maximum number of CUs allowed by the scaling policy

                          +

                          64

                          +
                          +
                          +
                        10. Click OK.
                        +
                        +

                        Step 4: Create an Enhanced Datasource Connection

                        1. Create a rule on the security group of the RDS DB instance to allow access from the CIDR block of the DLI queue.

                          1. Go to the RDS management console and choose Instances in the navigation pane on the left. In the instance list, click the name of the RDS for MySQL DB instance you created to access its basic information page.
                          2. In the navigation pane on the left, choose Connectivity & Security. In the Security Group Rules area, find the Inbound Rules tab and click Add Inbound Rule.

                            For example, if the CIDR block of the queue is 172.16.0.0/19, add the rule as follows:

                            +
                            • Type: Select IPv4.
                            • Protocol & Port: Select Protocols/TCP (Custom) as the protocol and leave the port number blank.
                            • Source: Select IP Address and enter 172.16.0.0/19.
                            +

                            Click OK. The security group rule is added.

                            +
                          +

                        2. Create an enhanced datasource connection between RDS for MySQL and DLI.

                          The CIDR block of the elastic resource pool bound to a datasource connection cannot overlap that of the data source.

                          +
                          +
                          1. Go to the DLI management console and choose Datasource Connections in the navigation pane on the left.
                          2. On the displayed Enhanced tab, click Create. Set the following parameters: +
                          3. Click OK.
                          4. In the Enhanced tab, click the created connection dlirds to view its VPC Peering ID and Connection Status. If the connection status is Active, the connection is successful.
                          5. Test if the queue can connect to the RDS for MySQL DB instance.
                            1. In the navigation pane on the left, choose Resources > Queue Management. On the displayed page, locate the queue added in Step 3: Create an Elastic Resource Pool and Add Queues to the Pool, click More in the Operation column, and select Test Address Connectivity.
                            2. Enter the floating IP address of the RDS for MySQL DB instance recorded in Step 2: Create an RDS Database Table.

                              On the Instance Management page, click the target DB instance. On the displayed page, choose Connection Information > Floating IP Address to obtain the floating IP address.

                              +
                              +
                              • If the IP address is reachable, the DLI queue can connect to the RDS for MySQL DB instance through the created enhanced datasource connection.
                              • If the IP address is unreachable, check if the security group where the data source is allows access from the IP address of the elastic resource pool. Once the fault is rectified, retest the network connectivity.
                              +
                            +
                          +

                        +
                        +

                        Step 5: Create a Datasource Authentication

                        When analyzing across multiple sources, you are advised not to configure authentication information directly in a job as it can lead to password leakage. Instead, you are advised to use datasource authentication provided by DLI to securely store data source authentication information.

                        +

                        To connect a Spark SQL job to an RDS for MySQL data source, you can create a password-type datasource authentication.

                        +
                        1. Log in to the DLI management console.
                        2. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Datasource Authentication.
                        3. Click Create.

                          Set authentication parameters based on Table 6.

                          + +
                          + + + + + + + + + + + + + + + + +
                          Table 6 Datasource authentication parameters

                          Parameter

                          +

                          Description

                          +

                          Type

                          +

                          Select Password.

                          +

                          Authentication Certificate

                          +

                          Name of the datasource authentication to create

                          +
                          • Only numbers, letters, and underscores (_) are allowed. The name cannot contain only numbers or start with an underscore (_).
                          • The value can contain a maximum of 128 characters.
                          +

                          Username

                          +

                          Username for logging in to the RDS for MySQL DB instance

                          +

                          Password

                          +

                          Password for logging in to the RDS for MySQL DB instance

                          +
                          +
                          +
                        +
                        +

                        Step 6: Submit a SQL Job

                        In this example, a SQL job accesses an RDS table using a datasource connection.

                        +
                        1. On the DLI management console, choose SQL Editor in the navigation pane on the left.
                        2. In the editing window on the right of the SQL Editor page, enter the following SQL statement to create database db1 and click Execute.
                          create database db1;
                          +
                        3. On the top of the editing window, choose the dli_queue_01 queue and the db1 database. Enter the following SQL statements to create a table, insert data to the table, and query the data. Click Execute.
                          View the query result to verify that the query is successful and the datasource connection works.
                          CREATE TABLE IF NOT EXISTS rds_test USING JDBC OPTIONS (
+'url' = 'jdbc:mysql://{{ip}}:{{port}}',  // Private IP address and port of RDS
+  'driver' = 'com.mysql.jdbc.Driver',
+'dbtable' = 'dli_demo.tabletest', // Name of the created DB instance and table name
+    'passwdauth'="xxxxx" // Name of the datasource authentication of the password type created on DLI. If datasource authentication is used, you do not need to set the username and password for the job.
+)
+
+insert into rds_test VALUES ('123','abc');
+
+
+SELECT * from rds_test;
                          +
                          +
                        +
                        + +
                        +
                        +
                        Parent topic: Getting Started
                        +
                        +
                        + diff --git a/docs/dli/umn/dli_01_0021.html b/docs/dli/umn/dli_01_0021.html index 17ca1a768..7309d5c9e 100644 --- a/docs/dli/umn/dli_01_0021.html +++ b/docs/dli/umn/dli_01_0021.html @@ -1,139 +1,68 @@ -

                        Managing SQL Templates

                        +

                        Creating a SQL Job Template

                        To facilitate SQL operation execution, DLI allows you to customize query templates or save the SQL statements in use as templates. After templates are saved, you do not need to compile SQL statements. Instead, you can directly perform the SQL operations using the templates.

                        -

                        SQL templates include sample templates and custom templates. The default sample template contains 22 standard TPC-H query statements, which can meet most TPC-H test requirements. For details, see TPC-H Sample Data in the SQL Template.

                        -

                        SQL template management provides the following functions:

                        - -

                        Table Settings

                        In the upper right corner of the SQL Template page, click Set Property to determine whether to display templates by group.

                        -

                        If you select Display by Group, the following display modes are available:

                        -
                        • Expand the first group
                        • Expand all
                        • Collapse All
                        -
                        -

                        Sample Templates

                        The current sample template contains 22 standard TPC-H query statements. You can view the template name, description, and statements. For details about TPC-H examples, see TPC-H Sample Data in the SQL Template.

                        +

                        SQL templates include sample templates and custom templates. The default sample template contains 22 standard TPC-H query statements, which can meet most TPC-H test requirements. For details, see TPC-H Sample Data in the SQL Templates Preset on DLI.

                        +

                        In the navigation pane on the left, choose Job Templates > SQL Templates. In the upper right corner of the displayed page, click Settings. In the displayed Settings dialog box, choose whether to display templates by group.

                        +

                        If you enable Display by Group, the display options are Expand the first group, Expand all, and Collapse all.

                        +
                        +

                        Creating a SQL Job Template

                        You can create a template on either the Job Templates or the SQL Editor page.

                        +
                        • To create a template on the Job Templates page:
                          1. On the left of the management console, choose Job Templates > SQL Templates.
                          2. On the SQL Templates page, click Create Template to create a template.

                            Enter the template name, SQL statement, and description information. For details, see Table 1.

                            -
                            @@ -44,7 +44,7 @@
                            -
                            Parent topic: Queue Management
                            +
                            Parent topic: Managing Queues
                            diff --git a/docs/dli/umn/dli_01_0220.html b/docs/dli/umn/dli_01_0220.html index 3ffaab0dc..b092753a7 100644 --- a/docs/dli/umn/dli_01_0220.html +++ b/docs/dli/umn/dli_01_0220.html @@ -4,13 +4,15 @@
                            diff --git a/docs/dli/umn/dli_01_0228.html b/docs/dli/umn/dli_01_0228.html index f75ecdbe8..dc6c0ddda 100644 --- a/docs/dli/umn/dli_01_0228.html +++ b/docs/dli/umn/dli_01_0228.html @@ -1,126 +1,78 @@ -

                            Overview

                            -

                            DLI database and table management provide the following functions:

                            - -

                            Difference Between DLI Tables and OBS Tables

                            • Data stored in DLI tables is applicable to delay-sensitive services, such as interactive queries.
                            • Data stored in OBS tables is applicable to delay-insensitive services, such as historical data statistics and analysis.
                            -
                            -

                            Notes and Constraints

                            • Database
                              • default is the database built in DLI. You cannot create a database named default.
                              • DLI supports a maximum of 50 databases.
                              -
                            • Table
                              • DLI supports a maximum of 5,000 tables.
                              • DLI supports the following table types:
                                • MANAGED: Data is stored in a DLI table.
                                • EXTERNAL: Data is stored in an OBS table.
                                • View: A view can only be created using SQL statements.
                                • Datasource table: The table type is also EXTERNAL.
                                -
                              • You cannot specify a storage path when creating a DLI table.
                              -
                            • Data import
                              • Only OBS data can be imported to DLI or OBS.
                              • You can import data in CSV, Parquet, ORC, JSON, or Avro format from OBS to tables created on DLI.
                              • To import data in CSV format to a partitioned table, place the partition column in the last column of the data source.
                              • The encoding format of imported data can only be UTF-8.
                              -
                            • Data export
                              • Data in DLI tables (whose table type is MANAGED) can only be exported to OBS buckets, and the export path must contain a folder.
                              • The exported file is in JSON format, and the text format can only be UTF-8.
                              • Data can be exported across accounts. That is, after account B authorizes account A, account A has the permission to read the metadata and permission information of account B's OBS bucket as well as the read and write permissions on the path. Account A can export data to the OBS path of account B.
                              -
                            -
                            -

                            Databases and Tables Page

                            The Databases and Tables page displays all created databases. You can view the database information, such as the owner and the number of tables.

                            - -
                            Table 1 Template management parameters

                            Parameter

                            +
                            - - - - - - - - - - - - - - -
                            Table 1 Parameter description

                            Parameter

                            Description

                            +

                            Description

                            Name

                            +

                            Name

                            Indicates the template name.

                            -
                            • A template name can contain only digits, letters, and underscores (_), but cannot start with an underscore (_) or contain only digits. It cannot be left empty.
                            • The template name can contain a maximum of 50 characters.
                            -

                            Description

                            -

                            Description of the template you create.

                            -

                            Statement

                            -

                            SQL statement created as the template.

                            -

                            Operation

                            -

                            Execute: After you click this button, the system switches to the SQL Editor page, where you can modify or directly perform the statement as required. For details, see Executing the Template.

                            -
                            -
                            -

                            The existing sample templates apply to the following scenarios:

                            -
                            • Price summary report query
                            • Minimum cost supplier analysis
                            • Shipping priority analysis
                            • Analysis of order priority check
                            • Analysis of the number of local suppliers
                            • Analysis of forecasted income changes
                            • Freight volume analysis
                            • National market share analysis
                            • Profit estimation analysis by product type
                            • Analysis of returned parts
                            • Analysis of key inventory indicators
                            • Freight mode and command priority analysis
                            • Consumer allocation analysis
                            • Promotion effect analysis
                            • Analysis of the supplier with the largest contribution
                            • Analysis of the relationship between parts and suppliers
                            • Revenue analysis of small-lot orders
                            • Customer analysis for large orders
                            • Discounted revenue analysis
                            • Analysis of potential component improvements
                            • Analysis of suppliers who fail to deliver goods on time
                            • Global sales opportunity analysis
                            - -

                            Custom Templates

                            The custom template list displays all templates you have created. You can view the template name, description, statements, and more.

                            - -
                            - - - - - - - - - - - - - - - - -
                            Table 2 Template management parameters

                            Parameter

                            -

                            Description

                            -

                            Name

                            -

                            Indicates the template name.

                            -
                            • A template name can contain only digits, letters, and underscores (_), but cannot start with an underscore (_) or contain only digits. It cannot be left empty.
                            • The template name can contain a maximum of 50 characters.
                            -

                            Description

                            -

                            Description of the template you create.

                            -

                            Statement

                            -

                            SQL statement created as the template.

                            -

                            Operation

                            -
                            • Execute: After you click this button, the system switches to the SQL Editor page, where you can modify or directly perform the statement as required. For details, see Executing the Template.
                            • Modify: Click Modify. In the displayed Modify Template dialog box, modify the template information as required. For details, see Modifying a Template.
                            -
                            -
                            -
                            -

                            Creating a Template

                            You can create a template on either the Job Templates or the SQL Editor page.

                            -
                            • To create a template on the Job Templates page:
                              1. On the left of the management console, choose Job Templates > SQL Templates.
                              2. On the SQL Templates page, click Create Template to create a template.

                                Enter the template name, SQL statement, and description information. For details, see Table 3.

                                - -
                                - - - - - - - - - - - - -
                                Table 3 Parameter description

                                Parameter

                                -

                                Description

                                -

                                Name

                                -

                                Indicates the template name.

                                +

                                Indicates the template name.

                                • A template name can contain only digits, letters, and underscores (_), but cannot start with an underscore (_) or contain only digits. It cannot be left empty.
                                • The template name can contain a maximum of 50 characters.

                                Statement

                                +

                                Statement

                                SQL statement to be saved as a template.

                                +

                                SQL statement to be saved as a template.

                                Description

                                +

                                Description

                                Description of the template you create.

                                +

                                Description of the template you create.

                                Group

                                +

                                Group

                                • Use existing
                                • Use new
                                • Do not use
                                +
                                • Use existing
                                • Use new
                                • Do not use

                                Group Name

                                +

                                Group Name

                                If you select Use existing or Use new, you need to enter the group name.

                                +

                                If you select Use existing or Use new, you need to enter the group name.
                              3. Click OK.
                              -
                            • To create a template on the SQL Editor page:
                              1. On the left of the management console, click SQL Editor.
                              2. In the SQL job editing area of the displayed SQL Editor page, click More in the upper right corner and choose Save as Template.

                                Enter the template name, SQL statement, and description information. For details, see Table 3.

                                +
                              3. To create a template on the SQL Editor page:
                                1. On the left of the management console, click SQL Editor.
                                2. In the SQL job editing area of the displayed SQL Editor page, click More in the upper right corner and choose Save as Template.

                                  Enter the template name, SQL statement, and description information. For details, see Table 1.

                                3. Click OK.
                            -

                            Executing the Template

                            Perform the template operation as follows:
                            1. On the left of the management console, choose Job Templates > SQL Templates.
                            2. On the SQL Templates page, select a template and click Execute in the Operation column. The SQL Editor page is displayed, and the corresponding SQL statement is automatically entered in the SQL job editing window.
                            3. In the upper right corner of the SQL job editing window, Click Execute to run the SQL statement. After the execution is complete, you can view the execution result below the current SQL job editing window.
                            +

                            Submitting a SQL Job Using a Template

                            Perform the template operation as follows:
                            1. On the left of the management console, choose Job Templates > SQL Templates.
                            2. On the SQL Templates page, select a template and click Execute in the Operation column. The SQL Editor page is displayed, and the corresponding SQL statement is automatically entered in the SQL job editing window.
                            3. In the upper right corner of the SQL job editing window, Click Execute to run the SQL statement. After the execution is complete, you can view the execution result below the current SQL job editing window.
                            -

                            Searching for a Template

                            On the SQL Templates page, you can enter the template name keyword in the search box on the upper right corner to search for the desired template.

                            +

                            Searching for a SQL Job Template

                            On the SQL Templates page, you can enter the template name keyword in the search box on the upper right corner to search for the desired template.

                            -

                            Modifying a Template

                            Only custom templates can be modified. To modify a template, perform the following steps:

                            +

                            Modifying a SQL Job Template

                            Only custom templates can be modified. To modify a template, perform the following steps:

                            1. On the SQL Templates page, locate the target template and click Modify in the Operation column.
                            2. In the displayed Modify Template dialog box, modify the template name, statement, and description as required.
                            3. Click OK.
                            -

                            Deleting a Template

                            On the SQL Templates page, select one or more templates to be deleted and click Delete to delete the selected templates.

                            +

                            Deleting a Template

                            On the SQL Templates page, select one or more templates to be deleted and click Delete to delete the selected templates.

                            diff --git a/docs/dli/umn/dli_01_0022.html b/docs/dli/umn/dli_01_0022.html index c4ab96483..b3889e91c 100644 --- a/docs/dli/umn/dli_01_0022.html +++ b/docs/dli/umn/dli_01_0022.html @@ -21,7 +21,7 @@
                            • Click the text box and select a predefined tag key from the drop-down list.

                              To add a predefined tag, you need to create one on TMS and then select it from the Tag key drop-down list. You can click View predefined tags to go to the Predefined Tags page of the TMS console. Then, click Create Tag in the upper corner of the page to create a predefined tag.

                            -
                            • Enter a tag key in the text box.
                              NOTE:

                              A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                              +
                              • Enter a tag key in the text box.
                                NOTE:

                                A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                              @@ -29,7 +29,7 @@

                            Tag value

                            You can specify the tag value in either of the following ways:

                            -
                            • Click the text box and select a predefined tag value from the drop-down list.
                            • Enter a tag value in the text box.
                              NOTE:

                              A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                              +
                              • Click the text box and select a predefined tag value from the drop-down list.
                              • Enter a tag value in the text box.
                                NOTE:

                                A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                            - - - - - - - - - - - - - - - - - - - - - - -
                            Table 1 Database and table management parameters

                            Parameter

                            -

                            Description

                            -

                            Database Name

                            -
                            • The database name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
                            • The database name is case insensitive and cannot be left unspecified.
                            • It cannot exceed 128 characters.
                            -

                            Username

                            -

                            Database owner.

                            -

                            Tables

                            -

                            Number of tables in the database.

                            -

                            Description

                            -

                            Description of the database specified during database creation. If no description is provided, -- is displayed.

                            -

                            Enterprise Project

                            -

                            Enterprise project to which the database belongs. An enterprise project facilitates project-level management and grouping of cloud resources and users.

                            -

                            Operation

                            -
                            • Permissions: View the permission information and perform user authorization, permission settings, and user permission revocation.
                            • Tables: View the tables in the corresponding database. For details, see Table Management Page.
                            • Create Table: This permission allows you to create a table in the corresponding database.
                            • Modify Database. This permission allows you to change the owner of the database. The username must exist under the same account.
                            • Drop Database: This permission allows you to delete the selected database.
                            -
                            -
                            -
                        -

                        Table Management Page

                        From the Data Management page, click the database name or Tables in the Operation column to switch to the table management page.

                        -

                        The displayed page lists all tables created in the current database. You can view the table type, data storage location, and other information. Tables are listed in chronological order by default, with the most recently created tables displayed at the top.

                        - -
                        - - - - - - - - @@ -127,7 +127,7 @@ diff --git a/docs/dli/umn/dli_01_0318.html b/docs/dli/umn/dli_01_0318.html index 2d5289e35..43c46d4e3 100644 --- a/docs/dli/umn/dli_01_0318.html +++ b/docs/dli/umn/dli_01_0318.html @@ -1,548 +1,403 @@ -

                        DLI Operations That Can Be Recorded by CTS

                        -

                        With CTS, you can record operations associated with DLI for later query, audit, and backtrack operations.

                        +

                        Using CTS to Audit DLI

                        +

                        With CTS, you can log operations related to DLI, making it easier to search, audit, and trace in the future.

                        -
                        Table 2 Table management parameters

                        Parameter

                        -

                        Description

                        -

                        Table Name

                        -
                        • The table name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_).
                        • The table name is case insensitive and cannot be left unspecified.
                        • The table name can contain the dollar sign ($). An example value is $test.
                        • It cannot exceed 128 characters.
                        -

                        Table Type

                        -

                        Table type. Available options are as follows:

                        -
                        • Managed: Indicates that data is stored in a DLI table.
                        • External: Indicates that data is stored in an OBS table.
                        • View: Indicates the view type. You can only create views using SQL statements.
                          NOTE:

                          The table or view information contained in the view cannot be modified. If the table or view information is modified, the query may fail.

                          +

                          Understanding Data Catalogs, Databases, and Tables

                          +
                          Databases and tables are the basis for developing SQL and Spark jobs. Before running a job, you need to define databases and tables based on your service scenarios.

                          Flink allows for dynamic data types, enabling the definition of data structures at runtime without the need for predefined metadata.

                          +
                          +

                          Data Catalog

                          A data catalog is a metadata management object that can contain multiple databases.

                          +

                          For how to create a database and a table in a DLI data catalog, see Creating a Database and Table on the DLI Console.

                          +
                          +

                          Database

                          A database is a repository of data organized, stored, and managed on computer storage devices according to data structures. Databases are typically used to store, retrieve, and manage structured data, consisting of multiple data tables that are interrelated through keys and indexes.

                          +
                          +

                          Table

                          Tables are one of the most important components of a database, consisting of rows and columns. Each row represents a data item, while each column represents a property or feature of the data. Tables are used to organize and store specific types of data, making it possible to query and analyze the data effectively.

                          +

                          A database is a framework, and tables are its essential content. A database contains one or more tables.

                          +

                          You can create databases and tables on the management console or using SQL statements. This section describes how to create a database and a table on the management console.

                          +

                          When creating a database or table, you need to grant permissions to other users so that they can view the new database or table.

                          +
                          +
                          +

                          Table Metadata

                          Metadata is data used to define the type of data. It primarily describes information about the data itself, including its source, size, format, or other characteristics. In database fields, metadata is used to interpret the content of a data warehouse.

                          +

                          When creating a table, metadata is defined by three columns: column name, type, and column description.

                          +
                          +

                          Table Types Supported by DLI

                          • DLI table

                            DLI tables are data tables stored in a DLI data lake. They can store structured, semi-structured, and unstructured data.

                            +

                            Data in DLI tables is stored internally within the DLI service, resulting in better query performance. This makes it suitable for time-sensitive services, such as interactive queries.

                            +

                            In the navigation pane on the left of the DLI console, choose Data Management > Databases and Tables. On the displayed page, click the name of a database. In the displayed table list, tables whose Type is MANAGED are DLI tables.

                            +
                          • OBS table

                            Data in OBS tables is stored in the OBS service, which is suitable for latency-insensitive services, such as historical data statistics and analysis.

                            +

                            An OBS table stores data in the form of objects. Each object contains data and related metadata.

                            +

                            In the navigation pane on the left of the DLI console, choose Data Management > Databases and Tables. On the displayed page, click the name of a database. In the displayed table list, tables whose Type is EXTERNAL and Storage Location is OBS are OBS tables.

                            +
                          • View table

                            A view table is a virtual table that does not store actual data. Instead, it dynamically generates data based on the defined query logic. Views are typically used to simplify complex queries or provide customized data views for different users or applications.

                            +

                            A view table can be created based on one or multiple tables, providing a flexible way to display data without affecting the storage and organization of the underlying data.

                            +

                            In the navigation pane on the left of the DLI console, choose Data Management > Databases and Tables. On the displayed page, click the name of a database. In the displayed table list, tables whose Type is VIEW are view tables.

                            +

                            A view can only be created using SQL statements. You cannot create a view on the Create Table page. Table or view information in a view cannot be modified. Otherwise, the query may fail.

                            +
                            +
                          • Datasource table

                            A datasource table is a data table that can be queried and analyzed across multiple data sources. This type of tables can integrate data from varying data sources and provide a unified data view.

                            +

                            Datasource tables are typically used in data warehouse and data lake architectures, allowing users to perform complex queries across multiple data sources.

                            +

                            In the navigation pane on the left of the DLI console, choose Data Management > Databases and Tables. On the displayed page, click the name of a database. In the displayed table list, tables whose Type is EXTERNAL and Storage Location is not OBS are datasource tables.

                          +
                          +

                          Notes and Constraints on Databases and Tables

                          +
                          + + + + + - - - - - - - - - - - - - - - - - -
                          Table 1 Notes and constraints on DLI resources

                          Item

                          +

                          Description

                          +

                          Database

                          +
                          • default is the database built in DLI. You cannot create a database named default.
                          • DLI supports a maximum of 50 databases.

                          Owner

                          +

                          Data table

                          User who creates the table.

                          +
                          • DLI supports a maximum of 5,000 tables.
                          • DLI supports the following table types:
                            • MANAGED: Data is stored in a DLI table.
                            • EXTERNAL: Data is stored in an OBS table.
                            • View: A view can only be created using SQL statements.
                            • Datasource table: The table type is also EXTERNAL.
                            +
                          • You cannot specify a storage path when creating a DLI table.

                          Storage Location

                          +

                          Data import

                          DLI, OBS, View, CloudTable, and CSS data location

                          +
                          • Only OBS data can be imported to DLI or OBS.
                          • You can import data in CSV, Parquet, ORC, JSON, or Avro format from OBS to tables created on DLI.
                          • To import data in CSV format to a partitioned table, place the partition column in the last column of the data source.
                          • The encoding format of imported data can only be UTF-8.

                          Size

                          +

                          Data export

                          Size of the data in the table. The value is displayed only for tables of the Managed type. For tables of other types, -- is displayed.

                          -

                          Data Source Path

                          -
                          • If Data Location is OBS, the corresponding OBS path is displayed.
                          • If Data Location is DLI and View, -- is displayed.
                          • When the data storage location is a datasource connection service such as CloudTable and CSS, the corresponding URL is displayed.
                          -

                          Created

                          -

                          Time when the table is created.

                          -

                          Last Accessed

                          -

                          Last time when an operation was performed on the table.

                          -

                          The last access time of a table refers only to the last time it was updated, not the time it was read (SELECT operation).

                          -

                          Operation

                          -
                          • Manage Permissions: This operation allows you to view the permission information and perform user authorization, permission settings, and user permission revocation.
                          • More:
                            • Delete: Delete a table from the corresponding database.
                            • Modify Owner: Change the owner of a table The username must exist under the same account.
                            • Import: Import data stored in an OBS bucket to a DLI or OBS table.
                            • Properties: View data in Metadata and Preview tabs.
                            -
                          +
                          • Data in DLI tables (whose table type is MANAGED) can only be exported to OBS buckets, and the export path must contain a folder.
                          • The exported file is in JSON format, and the text format can only be UTF-8.
                          • Data can be exported across accounts. That is, after account B authorizes account A, account A has the permission to read the metadata and permission information of account B's OBS bucket as well as the read and write permissions on the path. Account A can export data to the OBS path of account B.
                          +

                          Table Management Page

                          From the Data Management page, click the database name or Tables in the Operation column to switch to the table management page.

                          +

                          The displayed page lists all tables created in the current database. You can view the table type, data storage location, and other information. Tables are listed in chronological order by default, with the most recently created tables displayed at the top.

                          +
                          -
                          Parent topic: Databases and Tables
                          +
                          Parent topic: Creating Databases and Tables
                          diff --git a/docs/dli/umn/dli_01_0253.html b/docs/dli/umn/dli_01_0253.html index 5efbeac4d..d87ef8470 100644 --- a/docs/dli/umn/dli_01_0253.html +++ b/docs/dli/umn/dli_01_0253.html @@ -1,7 +1,7 @@ -

                          Importing Data to the Table

                          -

                          You can import data from OBS to a table created in DLI.

                          +

                          Importing OBS Data to DLI

                          +

                          This section describes how to import data stored in OBS to a table on the DLI console.

                          Precautions

                          • Only one path can be specified during data import. The path cannot contain commas (,).
                          • To import data in CSV format to a partitioned table, place the column to be partitioned in the last column of the data source.
                          • You are advised not to concurrently import data in to a table. If you concurrently import data into a table, there is a possibility that conflicts occur, leading to failed data import.
                          • The imported file can be in CSV, Parquet, ORC, JSON, and Avro format. The encoding format must be UTF-8.

                          Prerequisites

                          The data to be imported has been stored on OBS.

                          @@ -60,7 +60,7 @@

                        Table Header: No/Yes

                        This parameter is valid only when File Format is set to CSV. Whether the data source to be imported contains the table header.

                        -

                        Click Advanced Settings and select the check box next to Table Header: No. If the check box is selected, the table header is displayed. If the check box is deselected, no table header is displayed.

                        +

                        Click Advanced Settings and select the checkbox next to Table Header: No. If the checkbox is selected, the table header is displayed. If the checkbox is deselected, no table header is displayed.

                        -
                        Table 1 DLI operations that can be recorded by CTS

                        Operation

                        +
                        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                        Table 1 DLI operations that can be recorded by CTS

                        Operation

                        Resource Type

                        +

                        Resource Type

                        Trace Name

                        +

                        Trace Name

                        Creating a database

                        +

                        Creating a database

                        database

                        +

                        database

                        createDatabase

                        +

                        createDatabase

                        Deleting a database

                        +

                        Deleting a database

                        database

                        +

                        database

                        deleteDatabase

                        +

                        deleteDatabase

                        Changing the database owner

                        +

                        Changing the database owner

                        database

                        +

                        database

                        alterDatabaseOwner

                        +

                        alterDatabaseOwner

                        Creating a table

                        +

                        Creating a table

                        table

                        +

                        table

                        createTable

                        +

                        createTable

                        Deleting tables

                        +

                        Deleting tables

                        table

                        +

                        table

                        deleteTable

                        +

                        deleteTable

                        Exporting table data

                        +

                        Exporting table data

                        table

                        +

                        table

                        exportData

                        +

                        exportData

                        Importing table data

                        +

                        Importing table data

                        table

                        +

                        table

                        importData

                        +

                        importData

                        Modifying the owner of a table

                        +

                        Changing the owner of a table

                        table

                        +

                        table

                        alterTableOwner

                        +

                        alterTableOwner

                        Creating a queue

                        +

                        Creating a queue

                        queue

                        +

                        queue

                        createQueue

                        +

                        createQueue

                        Deleting a queue

                        +

                        Deleting a queue

                        queue

                        +

                        queue

                        dropQueue

                        +

                        deleteQueue

                        Granting permissions to a queue

                        +

                        Granting permissions to a queue

                        queue

                        +

                        queue

                        shareQueue

                        +

                        queueAuthorize

                        Modifying the CIDR block of a queue

                        +

                        Modifying the CIDR block of a queue

                        queue

                        +

                        queue

                        replaceQueue

                        +

                        replaceQueue

                        Restarting a queue

                        +

                        Restarting a queue

                        queue

                        +

                        queue

                        queueActions

                        +

                        queueActions

                        Scaling out/in a queue

                        +

                        Scaling out/in a queue

                        queue

                        +

                        queue

                        queueActions

                        +

                        queueActions

                        Submitting a job (SQL)

                        +

                        Submitting a job (SQL)

                        queue

                        +

                        queue

                        submitJob

                        +

                        submitJob

                        Canceling a job (SQL)

                        +

                        Canceling a job (SQL)

                        queue

                        +

                        jobs

                        cancelJob

                        +

                        cancelJob

                        Granting DLI the permission to access OBS buckets

                        +

                        Granting DLI the permission to access OBS buckets

                        obs

                        +

                        obs

                        obsAuthorize

                        +

                        authorizeObsBucketsForStream

                        Checking the SQL syntax

                        +

                        Checking SQL syntax

                        job

                        +

                        jobs

                        checkSQL

                        +

                        checkSQL

                        Creating a job

                        +

                        Deleting a job

                        job

                        +

                        jobs

                        createJob

                        +

                        deleteStreamJob

                        Updating a job

                        +

                        Creating a Flink OpenSource SQL job

                        job

                        +

                        jobs

                        updateJob

                        +

                        createStreamSqlJob

                        Deleting a job

                        +

                        Updating a Flink OpenSource SQL job

                        job

                        +

                        jobs

                        deleteJob

                        +

                        updateStreamSqlJob

                        Creating a Flink OpenSource SQL job

                        +

                        Deleting Flink jobs in batches

                        jobs

                        +

                        jobs

                        createStreamSqlJob

                        +

                        deleteStreamJobs

                        Updating a Flink OpenSource SQL job

                        +

                        Stopping a Flink job

                        jobs

                        +

                        jobs

                        updateStreamSqlJob

                        +

                        stopStreamJobs

                        Deleting a Flink job

                        +

                        Submitting a Flink job

                        jobs

                        +

                        jobs

                        deleteStreamJobs

                        +

                        submitStreamJobs

                        Stopping a Flink job

                        +

                        Creating a Flink Jar job

                        jobs

                        +

                        jobs

                        stopStreamJobs

                        +

                        createStreamJarJob

                        Submitting a Flink job

                        +

                        Updating a Flink Jar job

                        jobs

                        +

                        jobs

                        submitStreamJobs

                        +

                        updateStreamJarJob

                        Creating a Flink Jar job

                        +

                        Viewing Flink jobs

                        jobs

                        +

                        jobs

                        createStreamJarJob

                        +

                        checkStreamJob

                        Updating a Flink Jar job

                        +

                        Importing a savepoint

                        jobs

                        +

                        jobs

                        updateStreamJarJob

                        +

                        dealSavepoint

                        Checking Flink jobs

                        +

                        Granting data permissions

                        jobs

                        +

                        authorization

                        checkStreamJob

                        +

                        dataAuthorize

                        Importing a savepoint

                        +

                        Granting permissions on other projects

                        jobs

                        +

                        authorization

                        dealSavepoint

                        +

                        projectDataAuthorize

                        Purchasing CUH packages

                        +

                        Exporting query results

                        order

                        +

                        jobs

                        orderPackage

                        +

                        storeJobResult

                        Freezing resources

                        +

                        Saving a SQL template

                        resource

                        +

                        template

                        freezeResource

                        +

                        createTemplate

                        Unfreezing resources

                        +

                        Updating a SQL template

                        resource

                        +

                        template

                        unfreezeResource

                        +

                        updateTemplate

                        Terminating resources

                        +

                        Deleting a SQL template

                        resource

                        +

                        template

                        deleteResource

                        +

                        deleteTemplates

                        Clearing resources

                        +

                        Creating a Flink template

                        resource

                        +

                        template

                        cleanResource

                        +

                        createStreamTemplate

                        Granting data permissions

                        +

                        Updating a Flink template

                        data

                        +

                        template

                        dataAuthorize

                        +

                        updateStreamTemplate

                        Granting permissions on other projects

                        +

                        Viewing Flink templates

                        data

                        +

                        template

                        authorizeProjectData

                        +

                        checkStreamTemplate

                        Exporting query results

                        +

                        Deleting a Flink template

                        data

                        +

                        template

                        storeJobResult

                        +

                        deleteStreamTemplate

                        Saving a SQL template

                        +

                        Creating a datasource authentication and uploading a certificate

                        sqlTemplate

                        +

                        datasourceauth

                        saveSQLTemplate

                        +

                        uploadAuthInfo

                        Updating a SQL template

                        +

                        Updating datasource authentication information

                        sqlTemplate

                        +

                        datasourceauth

                        updateSQLTemplate

                        +

                        updateAuthInfo

                        Deleting a SQL template

                        +

                        Deleting datasource authentication information

                        sqlTemplate

                        +

                        datasourceauth

                        deleteSQLTemplate

                        +

                        deleteAuthInfo

                        Creating a Flink template

                        +

                        Uploading a resource package

                        template

                        +

                        resource

                        createStreamTemplate

                        +

                        uploadResources

                        Updating a Flink template

                        +

                        Deleting a resource package

                        template

                        +

                        resource

                        updateStreamTemplate

                        +

                        deleteResource

                        Checking Flink templates

                        +

                        Creating an enhanced datasource connection

                        template

                        +

                        edsconnection

                        checkStreamTemplate

                        +

                        createConnection

                        Deleting a Flink template

                        +

                        Deleting an enhanced datasource connection

                        flinkTemplate

                        +

                        edsconnection

                        deleteStreamTemplate

                        +

                        deleteConnection

                        Creating a data upload task

                        +

                        Binding a queue

                        uploader

                        +

                        edsconnection

                        createUploadJob

                        +

                        associateQueueToConnection

                        Obtaining the authentication to perform a data upload task

                        +

                        Unbinding a queue

                        uploader

                        +

                        edsconnection

                        getUploadAuthInfo

                        +

                        disassociateQueueToConnection

                        Submitting a data upload task

                        +

                        Modifying host information

                        uploader

                        +

                        edsconnection

                        commitUploadJob

                        +

                        updateHostInfo

                        Creating a datasource authentication and uploading a certificate

                        +

                        Adding a route

                        authInfo

                        +

                        edsconnection

                        uploadAuthInfo

                        +

                        addRoute

                        Updating a datasource authentication

                        +

                        Deleting a route

                        authInfo

                        +

                        edsconnection

                        updateAuthInfop

                        +

                        deleteRoute

                        Deleting a datasource authentication

                        +

                        Creating a batch processing job

                        authInfo

                        +

                        jobs

                        deleteAuthInfo

                        +

                        createBatch

                        Updating the quota

                        +

                        Canceling a batch processing job

                        quota

                        +

                        jobs

                        updateQuota

                        +

                        cancelBatch

                        Uploading a resource package

                        +

                        Creating a global variable

                        pkgResource

                        +

                        variable

                        uploadResources

                        +

                        createGlobalVariable

                        Deleting a resource package

                        +

                        Deleting a global variable

                        pkgResource

                        +

                        variable

                        deleteResource

                        +

                        deleteGlobalVariable

                        Creating a basic datasource connection

                        +

                        Modifying a global variable

                        datasource

                        +

                        variable

                        createDatasourceConn

                        -

                        Deleting a basic datasource connection

                        -

                        datasource

                        -

                        deleteDatasourceConn

                        -

                        Reactivating a basic datasource connection

                        -

                        datasource

                        -

                        reactivateDSConnection

                        -

                        Creating an enhanced datasource connection

                        -

                        datasource

                        -

                        createConnection

                        -

                        Deleting an enhanced datasource connection

                        -

                        datasource

                        -

                        getConnection

                        -

                        Binding a queue

                        -

                        datasource

                        -

                        associateQueueToDatasourceConn

                        -

                        Unbinding a queue

                        -

                        datasource

                        -

                        disassociateQueueToDatasourceConn

                        -

                        Modifying the host information

                        -

                        datasource

                        -

                        updateHostInfo

                        -

                        Adding a route

                        -

                        datasource

                        -

                        addRoute

                        -

                        Deleting a route

                        -

                        datasource

                        -

                        deleteRoute

                        -

                        Creating a topic

                        -

                        smn

                        -

                        createTopic

                        -

                        Creating an agency

                        -

                        agency

                        -

                        createAgencyV2

                        -

                        Creating a batch processing job

                        -

                        batch

                        -

                        createBatch

                        -

                        Canceling a batch processing job

                        -

                        batch

                        -

                        cancelBatch

                        -

                        Creating a session

                        -

                        session

                        -

                        createSession

                        -

                        Deleting a session

                        -

                        session

                        -

                        deleteSession

                        -

                        Creating a statement

                        -

                        statement

                        -

                        createStatement

                        -

                        Canceling execution of a statement

                        -

                        statement

                        -

                        cancelStatement

                        -

                        Creating a global variable

                        -

                        globalVar

                        -

                        createGlobalVariable

                        -

                        Deleting a global variable

                        -

                        globalVar

                        -

                        deleteGlobalVariable

                        -

                        Modifying a global variable

                        -

                        globalVar

                        -

                        updateGlobalVariable

                        +

                        updateGlobalVariable
                        -
                        -
                        -
                        Parent topic: Other Common Operations
                        -
                        -
                        diff --git a/docs/dli/umn/dli_01_0320.html b/docs/dli/umn/dli_01_0320.html index 066b8c93d..003ce509b 100644 --- a/docs/dli/umn/dli_01_0320.html +++ b/docs/dli/umn/dli_01_0320.html @@ -1,10 +1,10 @@ -

                        SQL Editor

                        -

                        Introduction

                        You can edit and run SQL statements in the SQL job editor to execute data query.

                        -

                        The editor supports SQL:2003 and is compatible with Spark SQL. For details about the syntax, see .

                        +

                        Creating and Submitting a SQL Job

                        +

                        Introduction

                        The SQL Editor allows you to execute data query operations using SQL statements.

                        +

                        It supports SQL:2003 and is compatible with Spark SQL.

                        To access the SQL editor, choose SQL Editor in the left navigation pane of the DLI console, or click Create Job in the upper right corner of the Job Management > SQL Jobs page.

                        -

                        This topic describes the main functions of the SQL editor.

                        +

                        This section describes how to create and submit a SQL job using the DLI SQL editor.

                        Notes

                        • If you access the SQL editor for the first time, the system prompts you to set a bucket for DLI jobs. The created bucket is used to store temporary data generated by DLI, such as job logs.

                          You cannot view job logs if you choose not to create the bucket. The bucket name will be set by the system.

                          On the OBS console, you can configure lifecycle rules for a bucket to periodically delete objects in it or change object storage classes.

                          @@ -12,222 +12,175 @@
                          • SQL statements can be executed in batches on the SQL editor page.
                          • Commonly used syntax in the job editing window is highlighted in different colors.
                          • Both single-line comment and multi-line comment are allowed. Use two consecutive hyphens (--) in each line to comment your statements.
                        -

                        Navigation pane

                        The navigation pane on the left consists of Databases, Queues, and Templates tabs.

                        -
                        -
                        Table 1 Navigation pane buttons

                        No.

                        +

                        Creating and Submitting a SQL Job Using the SQL Editor

                        1. Log in to the DLI management console. In the navigation pane on the left, choose SQL Editor.

                          On the SQL editor page, the system prompts you to create an OBS bucket to store temporary data generated by DLI jobs. In the Set Job Bucket dialog box, click Setting. On the page displayed, click the edit button in the upper right corner of the job bucket card. In the displayed Set Job Bucket dialog box, enter the job bucket path and click OK.

                          +
                          +
                        2. Above the SQL job editing window, set the parameters required for running a SQL job, such as the queue and database. For how to set the parameters, refer to Table 1. +
                          - - - - - - - - - - - - - - -
                          Table 1 Setting SQL job parameters

                          Button & Drop-Down List

                          Name

                          -

                          Description

                          +

                          Description

                          1

                          +

                          Queues

                          Databases

                          -

                          Displays all the existing databases and tables in these databases.

                          -
                          • Click a database name to view the tables in the database.
                          • Click a table name to view the metadata in the table. A maximum of 20 metadata records can be displayed.
                          • After you double-click a table name, a SQL query statement is automatically entered in the editing window.
                          +

                          Select a queue from the drop-down list box. If there is no queue available, the default queue is displayed. However, this queue is shared among all users for experience only. This may lead to resource contention, preventing you from obtaining the required resources for your jobs. So, you are advised to create your own queue for executing your jobs. For how to create a queue, see Creating an Elastic Resource Pool and Creating Queues Within It.

                          +

                          SQL jobs can only be executed on SQL queues.

                          2

                          +

                          Databases

                          Queues

                          -

                          Displays existing queues.

                          -

                          3

                          -

                          Templates

                          -

                          Click the drop-down button to view 22 built-in standard TPC-H query templates and custom templates.

                          -
                          -
                          -
                        - -

                        SQL Editing Window

                        SQL job editing window is displayed in the upper right part of the page. For details about the parameters, see Table 2.

                        -

                        The SQL statement editing area is below the operation bar. For details about keyboard shortcuts, see Table 3.

                        -
                        -
                        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                        Table 2 Components of the SQL job editing window

                        No.

                        -

                        Button & Drop-Down List

                        -

                        Description

                        -

                        2

                        -

                        Queues

                        -

                        Select a queue from the drop-down list box. If no queue is available, the default queue is displayed. Refer to Creating a Queue and create a queue.

                        -

                        SQL jobs can only be executed on SQL queues.

                        -

                        3

                        -

                        Database

                        -

                        Select a database from the drop-down list box. If no database is available, the default database is displayed. For details about how to create a database, see Creating a Database or a Table.

                        -
                        NOTE:

                        If you specify the database in the SQL statements, the database you choose from the drop-down list will not be used.

                        +

                        Select a database from the drop-down list box. If no database is available, the default database is displayed. For how to create a database, see Creating a Database and Table on the DLI Console.

                        +
                        NOTE:

                        If you have specified a database where tables are located in SQL statements, the database you choose here does not apply.

                        4

                        +

                        Settings

                        Execute

                        -

                        Click this button to run the SQL statements in the job editing window.

                        -

                        5

                        -

                        Format

                        -

                        Click this button to format the SQL statements.

                        -

                        6

                        -

                        Syntax Reference

                        -

                        Click this button to view the Data Lake Insight SQL Syntax Reference.

                        -

                        7

                        -

                        Settings

                        -

                        Add parameters and tags.

                        -

                        Parameter Settings: Set parameters in key/value format for SQL jobs.

                        -

                        Tags: Set tags in key/value format for SQL jobs.

                        -

                        8

                        -

                        More

                        -

                        The drop-down list includes the following options:

                        -
                        • Click Verify Syntax to check whether the SQL statements are correct.
                        • Click Set as Template to set SQL statements as a template. For details, see Managing SQL Templates.
                        • Click Change Theme to switch between dark and light modes.
                        +

                        Add parameters and tags.

                        +

                        Parameter Settings: Set parameters in key/value format for SQL jobs.

                        +

                        Tags: Assign key-value pairs as tags to a SQL job.
                        - -
                        - - - - - - - - - - - - - - - - - - - -
                        Table 3 Keyboard shortcut description

                        Shortcut

                        -

                        Description

                        -

                        Ctrl+Enter

                        -

                        Execute SQL statements. You can run SQL statements by pressing Ctrl+R or Ctrl + Enter on the keyboard.

                        -

                        Ctrl+F

                        -

                        Search for SQL statements. You can press Ctrl+F to search for a required SQL statement.

                        -

                        Shift+Alt+F

                        -

                        Format SQL statements. You can press Shift + Alt + F to format a SQL statement.

                        -

                        Ctrl+Q

                        -

                        Syntax verification. You can press Ctrl + Q to verify the syntax of SQL statements.

                        -

                        F11

                        -

                        Full screen. You can press F11 to display the SQL Job Editor window in full screen. Press F11 again to leave the full screen.

                        -
                        -
                        -
                        -
                        -

                        Executed Queries (Last Day) and View Result

                        After the SQL job is executed, you can view the execution history and result in the lower part of the editing area.

                        -
                        • Executed Queries (Last Day)

                          You can filter the execution history in the following ways:

                          -
                          • In the search box in the upper right corner of the Executed Queries (Last Day) pane, select a queue name or enter an execution statement in the search box.
                          • In the list, click the icon next to Created and choose Ascending or Descending.
                          • Select a job status from the Status list.
                          - -
                          - - - - - - - -
                          Table 4 Area description

                          Area

                          -

                          Description

                          -

                          Executed Queries (Last Day)

                          -
                          The latest daily information about the submitted jobs, including the following items:
                          • Queues: Queue name
                          • Username: User who executes the SQL statements
                          • Type: Type of the SQL job
                          • Status: Execution status of the SQL job
                          • Query
                          • Created
                          • Operation
                            • Edit: Edit the SQL statement.
                            • SparkUI: Switch to the SparkUI page to view the SQL statement execution process.
                              NOTE:

                              When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click SparkUI before the cluster is created, an empty projectID will be cached. The SparkUI page cannot be displayed. You are advised to use a dedicated queue so that the cluster will not be released. Alternatively, wait for a while after the job is submitted (the cluster is created), and then check SparkUI.

                              -

                              Currently, only the latest 100 job information records are displayed on the SparkUI of DLI.

                              -

                              This function is not supported for synchronization jobs and jobs running on the default queue.

                              -
                              -
                            • More: The following operations vary depending on the SQL job types and running status.

                              Cancel: Cancel a SQL job that is running or being submitted.

                              -

                              Re-execute: Execute the SQL statement again.

                              -

                              View Result: View the execution result of a QUERY job.

                              -

                              Export Result: Export the execution results of a QUERY job to a specified OBS path.

                              -

                              View Log: View the OBS path for storing SQL statement execution logs.

                              -
                              NOTE:

                              The View Log button is not available for synchronization jobs and jobs running on the default queue.

                              -
                              -
                            -
                          -
                          -
                          -
                          -
                        -
                        -
                        • View Result -
                          - - - - - - - - - - - - - -
                          Table 5 Operations on the result tab

                          Operation

                          -

                          Description

                          -

                          Clear the result

                          -

                          Clear the displayed SQL statement query results.

                          -

                          View chart/table

                          -

                          Click to view the query result in a chart or table.

                          -

                          Export job results

                          -

                          Export the job execution results to the created OBS bucket.

                          -
                          -
                          -
                        -

                        SQL Query Procedure

                        1. Log in to the DLI management console. On the page displayed, choose Job Management > SQL Jobs. On the page displayed, click Create Job.

                          On the SQL editor page, the system prompts you to create an OBS bucket to store temporary data generated by DLI jobs. In the Set Job Bucket dialog box, click Setting. On the page displayed, click the edit button in the upper right corner of the job bucket card. In the Set Job Bucket dialog box displayed, enter the job bucket path and click OK.

                          -
                          -
                        2. Select a queue from the queue list in the upper left corner of the SQL job editing window. For details about how to create a queue, see Creating a Queue.
                        3. In the upper right corner of the SQL job editing window, select a database, for example, qw, from the Databases drop-down list.
                        4. Create a table, for example, qw. For details about how to create a database and table, see Creating a Database or a Table.
                        5. In the SQL job editing window, enter the following SQL statement:
                          1
                          SELECT * FROM qw.qw LIMIT 10;
+
                        6. Create a database and a table.
                          Create them in advance by referring to Creating a Database and Table on the DLI Console. For example, create a table named qw.
                          
+
                        7. In the SQL job editing window, enter the following SQL statement:
                          1
                          SELECT * FROM qw.qw LIMIT 10;
 
                          
 
                          
 
                          Alternatively, you can double-click the table name qw. The query statement is automatically entered in the SQL job editing window.
                          
-
                        8. On top of the editing window, click More > Verify Syntax to check whether the SQL statement is correct.
                          1. If the verification fails, check the SQL statement syntax by referring to Data Lake Insight SQL Syntax Reference.
                          2. If the syntax verification is successful, click Execute. Read and agree to the privacy agreement. Click OK to execute the SQL statement.
                          3. After the execution is complete, you can view the execution result in the area under the SQL job editing window.
                          
-
                        9. (Optional) A maximum of 1000 records can be displayed in the query result on the current console. To view more or all data, click  to export the data to OBS.
                        10. (Optional) In the View Result tab, click  to display the query result in a chart. Click  to switch back to the table view.
                           
                          • If no column of the numeric type is displayed in the execution result, the result cannot be represented in charts.
                          • You can view the data in a bar chart, line chart, or fan chart.
                          • In the bar chart and line chart, the X axis can be any column, while the Y axis can only be columns of the numeric type. The fan chart displays the corresponding legends and indicators.
                          
+
                          DLI offers a range of SQL templates that come with use cases, code examples, and usage guides. You can also use these templates to quickly implement your service logic. For more information about templates, see Creating a SQL Job Template.
                          
+
                        11. On top of the editing window, click More > Verify Syntax to check whether the SQL statement is correct.
                          1. If the verification fails, check the SQL statement syntax by referring to Data Lake Insight SQL Syntax Reference.
                          2. If the syntax verification is successful, click Execute. Read and agree to the privacy agreement. Click OK to execute the SQL statement.
                          
+
                          Once successfully executed, you can check the execution result on the View Result tab below the SQL job editing window.
                          
+
                        12. View job execution results.
                          On the View Result tab, click  to display execution results in a chart. Click  to switch back to the table view.
                          
+
                          You can view a maximum of 1,000 data records on this View Result tab. To view more or full data, click  to export the data to OBS.
                           
                          • If no column of the numeric type is displayed in the execution result, the result cannot be represented in charts.
                          • You can view the data in a bar chart, line chart, or fan chart.
                          • In the bar chart and line chart, the X axis can be any column, while the Y axis can only be columns of the numeric type. The fan chart displays the corresponding legends and indicators.
                          
 
                          
+
                          
 
                          13.
                          -

                          Quickly Importing SQL Statements

                          • Double-click a table name in the navigation pane on the left to import the query statement of the selected table into the SQL statement editing window, and then click Execute to query.
                          • You can click More and choose Save as Template to save the SQL statement as a template for future use.

                            To use the SQL statement template, click Templates from the left pane of the SQL editor page. Double-click the required template in the template list, and modify it as required before executing the SQL statements.

                            +

                            Functions of SQL Editor

                            • Parameter settings for SQL jobs

                              Click Settings in the upper right corner of the SQL Editor page. You can set parameters and tags for the SQL job.

                              +
                              • Parameter Settings: Assign key-value pairs as parameter settings.

                                +

                                +
                              • Tags: Assign key-value pairs as tags to a SQL job.
                              + +
                              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                              Table 2 Parameters for SQL job running

                              Parameter

                              +

                              Default Value

                              +

                              Description

                              +

                              spark.sql.files.maxRecordsPerFile

                              +

                              0

                              +

                              Maximum number of records to be written into a single file. If the value is zero or negative, there is no limit.

                              +

                              spark.sql.autoBroadcastJoinThreshold

                              +

                              209715200

                              +

                              Maximum size, in bytes, of the table that displays all working nodes when a connection is executed. You can set this parameter to -1 to disable the display.

                              +
                              NOTE:

                              Currently, only configuration units that store tables analyzed using the ANALYZE TABLE COMPUTE statistics noscan command and file-based data source tables that calculate statistics directly from data files are supported.

                              +
                              +

                              spark.sql.shuffle.partitions

                              +

                              200

                              +

                              Default number of partitions used to filter data for join or aggregation.

                              +

                              spark.sql.dynamicPartitionOverwrite.enabled

                              +

                              false

                              +

                              When set to false, DLI will delete all partitions that meet the conditions before overwriting them. For example, if there is a partition named 2021-01 in a partitioned table and you use the INSERT OVERWRITE statement to write data to the 2021-02 partition, the data in the 2021-01 partition will also be overwritten.

                              +

                              When set to true, DLI will not delete partitions in advance, but will overwrite partitions with data written during runtime.

                              +

                              spark.sql.files.maxPartitionBytes

                              +

                              134217728

                              +

                              Maximum number of bytes to be packed into a single partition when a file is read.

                              +

                              spark.sql.badRecordsPath

                              +

                              -

                              +

                              Path of bad records.

                              +

                              dli.sql.sqlasync.enabled

                              +

                              true

                              +

                              Whether DDL and DCL statements are executed asynchronously. The value true indicates that asynchronous execution is enabled.

                              +

                              dli.sql.job.timeout

                              +

                              -

                              +

                              Job running timeout interval, in seconds. If the job times out, it will be canceled.

                              +
                              +
                              +
                            • Switching to the SparkUI page to view the SQL statement execution process

                              The SQL editor allows you to switch to the SparkUI to view the SQL statement execution process.

                              +
                              • You can view only the latest 100 job records on DLI's SparkUI.
                              • If a job is running on the default queue or is a synchronization one, you cannot switch to the SparkUI to view the SQL statement execution process.
                              +

                              When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click SparkUI before the cluster is created, an empty projectID will be cached. The SparkUI page cannot be displayed. You are advised to use a dedicated queue so that the cluster will not be released. Alternatively, wait for a while after the job is submitted (the cluster is created), and then check SparkUI.

                              +
                              +
                            • Archiving SQL run logs

                              On the Executed Queries (Last Day) tab of the SQL Editor page, click More and select View Log in the Operation column of the SQL job. The system automatically switches to the OBS path where logs are stored. You can download logs as needed.

                              +

                              The View Log button is not available for synchronization jobs and jobs running on the default queue.

                              +
                              +
                            • SQL Editor shortcuts +
                              + + + + + + + + + + + + + + + + + + + +
                              Table 3 Keyboard shortcuts

                              Shortcut

                              +

                              Description

                              +

                              Ctrl+Enter

                              +

                              Execute SQL statements. You can run SQL statements by pressing Ctrl+R or Ctrl + Enter on the keyboard.

                              +

                              Ctrl+F

                              +

                              Search for SQL statements. You can press Ctrl+F to search for a required SQL statement.

                              +

                              Shift+Alt+F

                              +

                              Format SQL statements. You can press Shift + Alt + F to format a SQL statement.

                              +

                              Ctrl+Q

                              +

                              Syntax verification. You can press Ctrl + Q to verify the syntax of SQL statements.

                              +

                              F11

                              +

                              Full screen. You can press F11 to display the SQL Job Editor window in full screen. Press F11 again to leave the full screen.

                              +
                              +
                          -
                          +
                          +
                          +
                          Parent topic: Submitting a SQL Job Using DLI
                          +
                          +
                          diff --git a/docs/dli/umn/dli_01_0363.html b/docs/dli/umn/dli_01_0363.html index 4100a31b4..c66e75103 100644 --- a/docs/dli/umn/dli_01_0363.html +++ b/docs/dli/umn/dli_01_0363.html @@ -1,60 +1,89 @@ -

                          Creating a Queue

                          -

                          Before executing a job, you need to create a queue.

                          +

                          Creating a Non-Elastic Resource Pool Queue (Discarded and Not Recommended)

                          +

                          Queues in the non-elastic resource pool mode are the previous-gen of resource management for DLI. It involved purchasing and releasing resources based on usage demands, requiring estimation of resource needs before making purchases.

                          +

                          Queues in an elastic resource pool are recommended, as they offer the flexibility to use resources with high utilization as needed. For how to buy an elastic resource pool and create queues within it, see Creating an Elastic Resource Pool and Creating Queues Within It.

                          • If you use a sub-account to create a queue for the first time, log in to the DLI management console using the main account and keep records in the DLI database before creating a queue.
                          • It takes 6 to 10 minutes for a job running on a new queue for the first time.
                          • After a queue is created, if no job is run within one hour, the system releases the queue.
                          • Queues with 16 CUs do not support scale-out or scale-in.
                          • Queues with 64 CUs do not support scale-in.
                          -

                          Procedure

                          1. You can create a queue on the Overview, SQL Editor, or Queue Management page.
                            • In the upper right corner of the Overview page, click Create Queue.
                            • To create a queue on the Queue Management page:
                              1. In the navigation pane of the DLI management console, choose Resources >Queue Management.
                              2. In the upper right corner of the Queue Management page, click Create Queue to create a queue.
                              -
                            • To create a queue on the SQL Editor page:
                              1. In the navigation pane of the DLI management console, click SQL Editor.
                              2. Click Queues. On the tab page displayed, click on the right to create a queue.
                              -
                            -
                          2. On the Create Queue page displayed, set the parameters according to Table 1. -
                            @@ -56,12 +56,13 @@
                            Table 1 Parameters

                            Parameter

                            +

                            Notes and Constraints

                            +
                            - - - + + + + + + + + +
                            Table 1 Notes and constraints on queues

                            Item

                            Description

                            +

                            Description

                            Name

                            +

                            Resource type

                            Name of a queue.

                            -
                            • The queue name can contain only digits, letters, and underscores (_), but cannot contain only digits, start with an underscore (_), or be left unspecified.
                            • The length of the name cannot exceed 128 characters.
                            +
                            • Queue types:
                              • default queue: A queue named default is preset in DLI, where you can use resources on demand.
                              • For SQL: Spark SQL jobs can be submitted to SQL queues.
                              • For general purpose: The queue is used to run Spark programs, Flink SQL jobs, and Flink Jar jobs.
                              +
                            • You cannot change the queue type once a queue is purchased. To use another queue type, purchase a new queue.
                            +

                            Managing queues

                            +
                            • The region of a queue cannot be changed.
                            • DLI queues cannot access the Internet.
                            +

                            Queue scaling

                            +
                            • Queues with 16 CUs do not support scale-out or scale-in.
                            • Queues with 64 CUs do not support scale-in.
                            • Newly created queues need to run jobs before they can be scaled in or out.
                            +
                            +
                            +
                            +

                            Procedure

                            1. You can create a queue on the Overview, SQL Editor, or Queue Management page.
                              • In the upper right corner of the Overview page, click Create Queue.
                              • To create a queue on the Queue Management page:
                                1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                                2. In the upper right corner of the Queue Management page, click Create Queue to create a queue.
                                +
                              • To create a queue on the SQL Editor page:
                                1. In the navigation pane of the DLI management console, click SQL Editor.
                                2. Click Queues. On the tab page displayed, click on the right to create a queue.
                                +
                              +
                            2. On the Create Queue page displayed, set the parameters according to Table 2. +
                              + + + + + - - - - - - - - - - @@ -62,7 +91,7 @@
                              Table 2 Parameters

                              Parameter

                              +

                              Description

                              +

                              Name

                              +

                              Queue name

                              +
                              • Only numbers, letters, and underscores (_) are allowed. The value cannot contain only numbers, start with an underscore (_), or be left unspecified.
                              • The value can contain a maximum of 128 characters.
                              NOTE:

                              The queue name is case-insensitive. Uppercase letters will be automatically converted to lowercase letters.

                              Type

                              +

                              Type

                              • For SQL: compute resources used for SQL jobs.
                              • For general purpose: compute resources used for Spark and Flink jobs.
                                NOTE:

                                Selecting Dedicated Resource Mode enables you to create a dedicated queue. Enhanced datasource connections can only be created for dedicated queues.

                                +
                              • For SQL: compute resources used for SQL jobs.
                              • For general purpose: compute resources used for Spark and Flink jobs.
                                NOTE:

                                When a dedicated queue is not in use, its resources are still reserved and not released. This means that resources are constantly being held, regardless of whether the queue is being utilized. By using a dedicated queue, resources can be guaranteed to be available whenever jobs are submitted. You can create enhanced datasource connections for a dedicated queue.

                                +

                              Specifications

                              +

                              Specifications

                              The compute nodes' total number of CUs. One CU equals one vCPU and 4 GB of memory. DLI automatically assigns CPU and memory resources to each compute node, and the client does not need to know how many compute nodes are being used.

                              +

                              The compute nodes' total number of CUs. One CU equals one vCPU and 4 GB of memory. DLI automatically assigns CPU and memory resources to each compute node, and the client does not need to know how many compute nodes are being used.

                              Description

                              +

                              Description

                              Description of the queue to be created. The description can contain a maximum of 128 characters.

                              +

                              Description of the queue to be created. The value can contain a maximum of 128 characters.

                              Advanced Settings

                              +

                              Advanced Settings

                              In the Queue Type area, select Dedicated Resource Mode and then click Advanced Settings.
                              • Default: The system automatically configures the parameter.
                              • Custom

                                CIDR Block: You can specify the CIDR block. For details, see Modifying the CIDR Block. If DLI enhanced datasource connection is used, the CIDR block of the DLI queue cannot overlap with that of the data source.

                                +
                              In the Queue Type area, select Dedicated Resource Mode and then click Advanced Settings.
                              • Default: The system automatically configures the parameter.
                              • Custom

                                CIDR Block: Enter a CIDR block range. If DLI enhanced datasource connection is used, the CIDR block of the DLI queue cannot overlap with that of the data source.

                                Queue Type: When running an AI-related SQL job, select AI-enhanced. When running other jobs, select Basic.

                              Tags

                              +

                              Tags

                              Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                              +

                              Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                              NOTE:
                              • A maximum of 20 tags can be added.
                              • Only one tag value can be added to a tag key.
                              • The key name in each resource must be unique.
                              -
                              • Tag key: Enter a tag key name in the text box.
                                NOTE:

                                A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                +
                                • Tag key: Enter a tag key name in the text box.
                                  NOTE:

                                  A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                  -
                                • Tag value: Enter a tag value in the text box.
                                  NOTE:

                                  A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                  +
                                • Tag value: Enter a tag value in the text box.
                                  NOTE:

                                  A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                              -
                            3. Click Create Now to create a queue.

                              After a queue is created, you can view and select the queue for use on the Queue Management page.

                              +
                            4. Click Create Now to create a queue.

                              Once created, you can check and use it on the Queue Management page.

                              It takes 6 to 10 minutes for a job running on a new queue for the first time.

                            @@ -70,7 +99,7 @@
                            diff --git a/docs/dli/umn/dli_01_0366.html b/docs/dli/umn/dli_01_0366.html index 1352042b4..ca1a92347 100644 --- a/docs/dli/umn/dli_01_0366.html +++ b/docs/dli/umn/dli_01_0366.html @@ -1,25 +1,25 @@ -

                            Package Management

                            +

                            Managing Program Packages of Jar Jobs

                            diff --git a/docs/dli/umn/dli_01_0367.html b/docs/dli/umn/dli_01_0367.html index e4bb90f6c..be1da1f56 100644 --- a/docs/dli/umn/dli_01_0367.html +++ b/docs/dli/umn/dli_01_0367.html @@ -1,6 +1,6 @@ -

                            Creating a Package

                            +

                            Creating a DLI Package

                            DLI allows you to submit program packages in batches to the general-use queue for running.

                            If you need to update a package, you can use the same package or file to upload it to the same location (in the same group) on DLI to overwrite the original package or file.

                            @@ -44,10 +44,10 @@

                            Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                            NOTE:
                            • A maximum of 20 tags can be added.
                            • Only one tag value can be added to a tag key.
                            • The key name in each resource must be unique.
                            -
                            • Tag key: Enter a tag key name in the text box.
                              NOTE:

                              A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                              +
                              • Tag key: Enter a tag key name in the text box.
                                NOTE:

                                A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                              -
                              • Tag value: Enter a tag value in the text box.
                                NOTE:

                                A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                +
                                • Tag value: Enter a tag value in the text box.
                                  NOTE:

                                  A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.

                          3. Click OK.

                            After a package is created, you can view and select the package for use on the Package Management page.

                            +

                            Once a job is executed, you can free up DLI storage space by promptly deleting the job package from the package management page if it is no longer needed.

                          diff --git a/docs/dli/umn/dli_01_0369.html b/docs/dli/umn/dli_01_0369.html index 3c5dbe60c..00cf39928 100644 --- a/docs/dli/umn/dli_01_0369.html +++ b/docs/dli/umn/dli_01_0369.html @@ -1,13 +1,44 @@ -

                          Deleting a Package

                          -

                          You can delete a package based on actual conditions.

                          -

                          Procedure

                          1. On the left of the management console, choose Data Management > Package Management.
                          2. Click Delete in the Operation column of the package to be deleted.
                          3. In the dialog box that is displayed, click Yes.
                          +

                          Managing DLI Package Tags

                          +

                          Tags are key-value pairs that you can define to identify cloud resources. They assist you in categorizing and searching for cloud resources. A tag consists of a key and a value.

                          +

                          DLI allows you to add tags to package groups or packages.

                          +
                          1. Log in to the DLI management console and choose Data Management > Package Management.
                          2. On the Package Management page, locate the desired package, click More in the Operation column, and select Tags.
                          3. On the page that appears, click Add/Edit Tag in the upper left corner.
                          4. In the Add/Edit Tag dialog box, set parameters. +
                            + + + + + + + + + + +
                            Table 1 Tag parameters

                            Parameter

                            +

                            Description

                            +

                            Tag key

                            +

                            You can specify the tag key in either of the following ways:

                            +
                            • Click the text box and select a predefined tag key from the drop-down list.

                              To add a predefined tag, you need to create one on TMS and then select it from the tag key drop-down list. You can click View predefined tags to go to the Predefined Tags page of the TMS console. Then, click Create Tag in the upper corner of the page to create a predefined tag.

                              +

                              +
                            +
                            • Enter a tag key in the text box.
                              NOTE:

                              A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                              +
                              +
                            +

                            Tag value

                            +

                            You can specify the tag value in either of the following ways:

                            +
                            • Click the text box and select a predefined tag value from the drop-down list.
                            • Enter a tag value in the text box.
                              NOTE:

                              A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.

                              +
                              +
                            +
                            +
                            • A maximum of 20 tags can be added.
                            • Only one tag value can be added to a tag key.
                            • The key name in each resource must be unique.
                            +
                            +
                          5. Click OK.
                          6. (Optional) To delete a tag, locate the tag in the tag list and click Delete in its Operation column.
                          diff --git a/docs/dli/umn/dli_01_0375.html b/docs/dli/umn/dli_01_0375.html index ed769d0a3..3a63422e1 100644 --- a/docs/dli/umn/dli_01_0375.html +++ b/docs/dli/umn/dli_01_0375.html @@ -1,40 +1,191 @@ -

                          Creating and Submitting a Spark Jar Job

                          -

                          Scenario

                          DLI can query data stored in OBS. This section describes how to use a Spark Jar job on DLI to query OBS data in real time.

                          +

                          Using DLI to Submit a Spark Jar Job

                          +

                          Scenario

                          DLI allows you to submit Spark jobs compiled as JAR files, which contain the necessary code and dependency information for executing the job. These files are used for specific data processing tasks such as data query, analysis, and machine learning. Before submitting a Spark Jar job, upload the package to OBS and submit it along with the data and job parameters to run the job.

                          +

                          This example introduces the basic process of submitting a Spark Jar job package through the DLI console. Due to different service requirements, the specific writing of the Jar package may vary. It is recommended that you refer to the sample code provided by DLI and edit and customize it according to your actual business scenario.

                          -

                          Procedure

                          You can use DLI to submit Spark jobs for real-time computing. The general procedure is as follows:

                          -

                          Step 1: Upload Data to OBS

                          -

                          Step 2: Create a Queue

                          -

                          Step 3: Create a Package

                          -

                          Step 4: Submit a Spark Job

                          +

                          Procedure

                          Table 1 describes the procedure for submitting a Spark Jar job using DLI.

                          + +
                          + + + + + + + + + + + + + +
                          Table 1 Procedure for submitting a Spark Jar job using DLI

                          Step

                          +

                          Description

                          +

                          Step 1: Upload Data to OBS

                          +

                          Prepare a Spark Jar job package and upload it to OBS.

                          +

                          Step 2: Create an Elastic Resource Pool and Add Queues to the Pool

                          +

                          Create compute resources required for submitting the Spark Jar job.

                          +

                          Step 3: Submit a Spark Job

                          +

                          Create a Spark Jar job to analyze data.

                          +
                          -

                          Step 1: Upload Data to OBS

                          Write a Spark Jar job program, and compile and pack it as spark-examples.jar. Perform the following steps to submit the job:

                          +
                          +

                          Step 1: Upload Data to OBS

                          Develop a Spark Jar job program, compile it, and pack it into spark-examples.jar. Perform the following steps to upload the program:

                          Before submitting Spark Jar jobs, upload data files to OBS.

                          -
                          1. Log in to the DLI console.
                          2. In the service list, click Object Storage Service under Storage. The OBS console page is displayed.
                          3. Create a bucket. In this example, name it dli-test-obs01.
                            1. Click Create Bucket.
                            2. On the displayed Create Bucket page, enter the Bucket Name. Retain the default values for other parameters or set them as required.

                              When creating an OBS bucket, you must select the same region as the DLI management console.

                              +
                              1. Log in to the DLI console.
                              2. In the service list, click Object Storage Service under Storage.
                              3. Create a bucket. In this example, name it dli-test-obs01.
                                1. On the displayed Buckets page, click Create Bucket in the upper right corner.
                                2. On the displayed Create Bucket page, enter the Bucket Name. Retain the default values for other parameters or set them as required.

                                  Select a region that matches the location of the DLI console.

                                3. Click Create Now.
                                -
                              4. Click dli-test-obs01 to switch to the Objects tab page.
                              5. Click Upload Object. In the dialog box displayed, drag or add files or folders, for example, spark-examples.jar, to the upload area. Then, click Upload.

                                After the file is uploaded successfully, the file path is obs://dli-test-obs01/spark-examples.jar.

                                -
                                • For more information about OBS operations, see the Object Storage Service Console Operation Guide.
                                • For more information about the tool, see the OBS Tool Guide.
                                • You are advised to use an OBS tool, such as OBS Browser+, to upload large files because OBS Console has restrictions on the file size and quantity.
                                  • OBS Browser+ is a graphical tool that provides complete functions for managing your buckets and objects in OBS. You are advised to use this tool to create buckets or upload objects.
                                  -
                                -
                                +
                              6. In the bucket list, click the name of the dli-test-obs01 bucket you just created to access its Objects tab.
                              7. Click Upload Object. In the dialog box displayed, drag or add files or folders, for example, spark-examples.jar, to the upload area. Then, click Upload.

                                In this example, the path after upload is obs://dli-test-obs01/spark-examples.jar.

                                +

                                For more operations on the OBS console, see the Object Storage Service User Guide.

                              -

                              Step 2: Create a Queue

                              If you submit a Spark job for the first time, you need to create a queue first. For example, create a queue named sparktest and set Queue Type to General Queue.

                              -
                              1. Log in to the DLI management console.
                              2. In the navigation pane of the DLI management console, choose Resources > Queue Management.
                              3. In the upper right corner of the Queue Management page, click Create Queue to create a queue.
                              4. Create a queue, name it sparktest, and set the queue usage to for general purpose. For details, see Creating a Queue.

                                -
                              5. Click Create Now to create a queue.
                              +

                              Step 2: Create an Elastic Resource Pool and Add Queues to the Pool

                              In this example, the elastic resource pool dli_resource_pool and queue dli_queue_01 are created.
                              1. Log in to the DLI management console.
                              2. In the navigation pane on the left, choose Resources > Resource Pool.
                              3. On the displayed page, click Buy Resource Pool in the upper right corner.
                              4. On the displayed page, set the parameters.
                                Table 2 describes the parameters. +
                                + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                Table 2 Parameters

                                Parameter

                                +

                                Description

                                +

                                Example Value

                                +

                                Region

                                +

                                Select a region where you want to buy the elastic resource pool.

                                +

                                _

                                +

                                Project

                                +

                                Project uniquely preset by the system for each region

                                +

                                Default

                                +

                                Name

                                +

                                Name of the elastic resource pool

                                +

                                dli_resource_pool

                                +

                                Specifications

                                +

                                Specifications of the elastic resource pool

                                +

                                Standard

                                +

                                CU Range

                                +

                                The maximum and minimum CUs allowed for the elastic resource pool

                                +

                                64-64

                                +

                                CIDR Block

                                +

                                CIDR block the elastic resource pool belongs to. If you use an enhanced datasource connection, this CIDR block cannot overlap that of the data source. Once set, this CIDR block cannot be changed.

                                +

                                172.16.0.0/19

                                +

                                Enterprise Project

                                +

                                Select an enterprise project for the elastic resource pool.

                                +

                                default

                                +
                                -

                                Step 3: Create a Package

                                Before submitting a Spark job, you need to create a package, for example, spark-examples.jar.

                                -
                                1. In the navigation pane on the left of the DLI console, choose Data Management > Package Management.
                                2. On the Package Management page, click Create in the upper right corner to create a package.
                                3. In the Create Package dialog box, set Type to JAR, OBS Path to the path of the spark-examples.jar package in Step 1: Upload Data to OBS, and Group to Do not use.

                                  -
                                4. Click OK.

                                  You can view and select the package on the Package Management page.

                                  -
                                -

                                For details about how to create a package, see "Creating a Package".

                                -

                                Step 4: Submit a Spark Job

                                1. On the DLI management console, choose Job Management > Spark Jobs in the navigation pane on the left. On the displayed page, click Create Job.
                                2. On the Spark job editing page, set Queues to the queue created in Step 2: Create a Queue and Application to the package created in Step 3: Create a Package.

                                  For details about other parameters, see the description of the Spark job editing page in "Creating a Spark Job".

                                  +
                                3. Click Buy.
                                4. Click Submit.
                                5. In the elastic resource pool list, locate the pool you just created and click Add Queue in the Operation column.
                                6. Set the basic parameters listed below. +
                                  + + + + + + + + + + + + + + + + + +
                                  Table 3 Basic parameters for adding a queue

                                  Parameter

                                  +

                                  Description

                                  +

                                  Example Value

                                  +

                                  Name

                                  +

                                  Name of the queue to add

                                  +

                                  dli_queue_01

                                  +

                                  Type

                                  +

                                  Type of the queue

                                  +
                                  • To execute SQL jobs, select For SQL.
                                  • To execute Flink or Spark jobs, select For general purpose.
                                  +

                                  _

                                  +

                                  Enterprise Project

                                  +

                                  Select an enterprise project.

                                  +

                                  default

                                  +
                                  +
                                  +
                                7. Click Next and configure scaling policies for the queue.

                                  Click Create to add a scaling policy with varying priority, period, minimum CUs, and maximum CUs.

                                  +

                                  + +
                                  + + + + + + + + + + + + + + + + + + + + + +
                                  Table 4 Scaling policy parameters

                                  Parameter

                                  +

                                  Description

                                  +

                                  Example Value

                                  +

                                  Priority

                                  +

                                  Priority of the scaling policy in the current elastic resource pool. A larger value indicates a higher priority. In this example, only one scaling policy is configured, so its priority is set to 1 by default.

                                  +

                                  1

                                  +

                                  Period

                                  +

                                  The first scaling policy is the default policy, and its Period parameter configuration cannot be deleted or modified.

                                  +

                                  The period for the scaling policy is from 00 to 24.

                                  +

                                  00–24

                                  +

                                  Min CU

                                  +

                                  Minimum number of CUs allowed by the scaling policy

                                  +

                                  16

                                  +

                                  Max CU

                                  +

                                  Maximum number of CUs allowed by the scaling policy

                                  +

                                  64

                                  +
                                  +
                                  +
                                8. Click OK.
                                +
                                +
                                +

                                Step 3: Submit a Spark Job

                                1. On the DLI management console, choose Job Management > Spark Jobs in the navigation pane on the left. On the displayed page, click Create Job in the upper right corner.
                                2. Set the following Spark job parameters: +

                                  For other parameters, refer to the description about the Spark job editing page in "Creating a Spark Job" in the Data Lake Insight User Guide.

                                3. Click Execute in the upper right corner of the Spark job editing window, read and agree to the privacy agreement, and click OK. Submit the job. A message is displayed, indicating that the job is submitted.

                                  -
                                4. (Optional) Switch to the Job Management > Spark Jobs page to view the status and logs of the submitted Spark job.

                                  When you click Execute on the DLI management console for the first time, you need to read the privacy agreement. Once agreed to the agreement, you will not receive any privacy agreement messages for subsequent operations.

                                  -
                                  -
                                +
                              5. (Optional) Switch to the Job Management > Spark Jobs page to view the status and logs of the submitted Spark job.
                              diff --git a/docs/dli/umn/dli_01_0376.html b/docs/dli/umn/dli_01_0376.html index b48216389..51151ca9e 100644 --- a/docs/dli/umn/dli_01_0376.html +++ b/docs/dli/umn/dli_01_0376.html @@ -1,18 +1,16 @@ -

                              Modifying the Owners of Databases and Tables

                              -

                              During actual use, developers create databases and tables and submit them to test personnel for testing. After the test is complete, the databases and tables are transferred to O&M personnel for user experience. In this case, you can change the owner of the databases and tables to transfer data to other owners.

                              -

                              Modifying the Database Owner

                              You can change the owner of a database on either the Data Management page or the SQL Editor page.
                              • On the Data Management page, change the database owner.
                                1. On the left of the management console, choose Data Management > Databases and Tables.
                                2. On the Databases and Tables page, locate the database you want and click More > Modify Database in the Operation column.
                                3. In the displayed dialog box, enter a new owner name (an existing username) and click OK.
                                +

                                Changing the Database Owner on the DLI Console

                                +

                                In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed over to O&M personnel for experience. In this scenario, ownership of the data can be transferred to another owner by changing the database owner.

                                +

                                Changing the Database Owner

                                You can change the owner of a database on either the Data Management page or the SQL Editor page.
                                • On the Data Management page, change the database owner.
                                  1. On the left of the management console, choose Data Management > Databases and Tables.
                                  2. On the Databases and Tables page, locate the database you want and click More > Modify Database in the Operation column.
                                  3. In the displayed dialog box, enter a new owner name (an existing username) and click OK.
                                • Change the database owner on the SQL Editor page.
                                  1. On the left of the management console, click SQL Editor.
                                  2. In the navigation tree on the left, click Databases, click on the right of the database you want to modify, and choose Modify Database from the shortcut menu.
                                  3. In the displayed dialog box, enter a new owner name (an existing username) and click OK.
                                -

                                Modifying the Table Owner

                                1. On the left of the management console, choose Data Management > Databases and Tables.
                                2. Click the name of the database corresponding to the table to be modified. The Manage Tables page of the database is displayed.
                                3. In the Operation column of the target table, choose More > Modify Owner.
                                4. In the displayed dialog box, enter a new owner name (an existing username) and click OK.
                                -
                                diff --git a/docs/dli/umn/dli_01_0377.html b/docs/dli/umn/dli_01_0377.html index 1d060c238..c15319069 100644 --- a/docs/dli/umn/dli_01_0377.html +++ b/docs/dli/umn/dli_01_0377.html @@ -1,17 +1,27 @@ -

                                DLI Console Overview

                                -

                                The Overview page of the DLI console provides you with the DLI workflow and resource usage.

                                -

                                How to Use DLI

                                The process of using DLI is as follows:

                                -
                                1. Create a queue.

                                  Queues are DLI's compute resources. There are SQL queues and general-purpose queues. For a SQL queue, you can only submit Spark SQL jobs. For a general-purpose queue, you can submit Spark programs, Flink OpenSource SQL jobs, and Flink Jar jobs.

                                  -
                                2. Prepare data.

                                  Create databases and tables before you run a Spark SQL job. Upload a program package before you run a Spark job or a Flink Jar job.

                                  -
                                3. Edit and submit a job.

                                  After you set the job parameters, you can submit the job.

                                  -
                                4. View job status.

                                  Go to the Job Management page to view the job execution status.

                                  -

                                  -
                                + +

                                Managing Flink Jobs

                                + +

                                -

                                Queue Usage (hours)

                                The overview page provides usage durations of all queues or a specific queue.

                                -
                                • Usage Usage (hours): an overview of the usage of all resources.
                                • Selected Queue Usage (hours): recent usage of a single queue.
                                + + diff --git a/docs/dli/umn/dli_01_0378.html b/docs/dli/umn/dli_01_0378.html index c4ef4d49d..bfc8fba10 100644 --- a/docs/dli/umn/dli_01_0378.html +++ b/docs/dli/umn/dli_01_0378.html @@ -1,18 +1,19 @@ -

                                What Is Data Lake Insight?

                                +

                                What Is Data Lake Insight

                                DLI Introduction

                                Data Lake Insight (DLI) is a serverless data processing and analysis service fully compatible with Apache Spark and Apache Flink ecosystems. It frees you from managing any servers.

                                DLI supports standard SQL and is compatible with Spark SQL and Flink SQL. It also supports multiple access modes, and is compatible with mainstream data formats. DLI supports SQL statements and Spark applications for heterogeneous data sources, including CloudTable, RDS, GaussDB(DWS), CSS, OBS, custom databases on ECSs, and offline databases.

                                Functions

                                You can query and analyze heterogeneous data sources such as RDS, and GaussDB(DWS) on the cloud using access methods, such as visualized interface, RESTful API, JDBC, and Beeline. The data format is compatible with five mainstream data formats: CSV, JSON, Parquet, and ORC.

                                -
                                • Basic functions
                                  • You can use standard SQL statements to query in SQL jobs.
                                  • Flink jobs support Flink SQL online analysis. Aggregation functions such as Window and Join, geographic functions, and CEP functions are supported. SQL is used to express service logic, facilitating service implementation.
                                  • For spark jobs, fully-managed Spark computing can be performed. You can submit computing tasks through interactive sessions or in batch to analyze data in the fully managed Spark queues.
                                  -
                                • Federated analysis of heterogeneous data sources
                                  • Spark datasource connection: Data sources such as GaussDB(DWS), RDS, and CSS can be accessed through DLI.
                                  • Interconnection with multiple cloud services is supported in Flink jobs to form a rich stream ecosystem. The DLI stream ecosystem consists of cloud service ecosystems and open source ecosystems.
                                    • Cloud service ecosystem: DLI can interconnect with other services in Flink SQL. You can directly use SQL to read and write data from cloud services.
                                    • Open-source ecosystems: After connections to other VPCs are established through datasource connections, you can access all data sources and output targets (such as Kafka, HBase, and Elasticsearch) supported by Flink and Spark in your dedicated DLI queue.
                                    +
                                    • Basic functions
                                      • You can use standard SQL statements to query in SQL jobs.
                                      • Flink jobs support Flink SQL online analysis capabilities: supporting aggregation functions such as Window and Join, using SQL to express service logic, and achieving service implementation conveniently and quickly.
                                      • For spark jobs, fully-managed Spark computing can be performed. You can submit computing tasks through interactive sessions or in batch to analyze data in the fully managed Spark queues.
                                      +
                                    • Federated analysis of heterogeneous data sources
                                      • Spark datasource connection: Data sources such as GaussDB(DWS), RDS, and CSS can be accessed through DLI.
                                      • Interconnection with multiple cloud services is supported in Flink jobs to form a rich stream ecosystem. The DLI stream ecosystem consists of cloud service ecosystems and open source ecosystems.
                                        • Cloud service ecosystem: DLI can interconnect with other services in Flink SQL. You can directly use SQL to read and write data from cloud services.
                                        • Open-source ecosystem: By establishing network connections with other VPCs through enhanced datasource connections, you can access all Flink and Spark-supported data sources and output sources, such as Kafka, Hbase, Elasticsearch, in the tenant-authorized DLI queues.

                                    • Storage-compute decoupling

                                      DLI is interconnected with OBS for data analysis. In this architecture where storage and compute are decoupled, resources of these two types are charged separately, helping you reduce costs and improving resource utilization.

                                      You can choose single-AZ or multi-AZ storage when you create an OBS bucket for storing redundant data on the DLI console. The differences between the two storage policies are as follows:

                                      • Multi-AZ storage means data is stored in multiple AZs, improving data reliability. If the multi-AZ storage is enabled for a bucket, data is stored in multiple AZs in the same region. If one AZ becomes unavailable, data can still be properly accessed from the other AZs. The multi-AZ storage is ideal for scenarios that demand high reliability. You are advised to use this policy.
                                      • Single-AZ storage means that data is stored in a single AZ, with lower costs.
                                      -
                                    • Elastic resource pool

                                      Elastic resource pools support the CCE cluster architecture for heterogeneous resources so you can centrally manage and allocate them. For details, see Elastic Resource Pool.

                                      +
                                    • Elastic resource pool

                                      The backend of elastic resource pools adopts a CCE cluster architecture, supporting heterogeneous resources, so you can manage and schedule resources in a unified manner.

                                      +

                                      For details, see Creating an Elastic Resource Pool and Queues Within It.

                                      Elastic resource pools have the following advantages:

                                      • Unified management
                                        • You can manage multiple internal clusters and schedule jobs. You can manage millions of cores for compute resources.
                                        • Elastic resource pools can be deployed across multiple AZs to support high availability.
                                      • Tenant resource isolation

                                        Resources of different queues are isolated to reduce the impact on each other.

                                        @@ -22,40 +23,40 @@

                                      Using elastic resource pools has the following advantages.

                                      -

                                      Advantage

                                      +
                                      - - - - - - - - - - - - - @@ -63,7 +64,7 @@ -

                                      DLI Core Engine: Spark+Flink

                                      • Spark is a unified analysis engine that is ideal for large-scale data processing. It focuses on query, compute, and analysis. DLI optimizes performance and reconstructs services based on open-source Spark. It is compatible with the Apache Spark ecosystem and interfaces, and improves performance by 2.5x when compared with open-source Spark. In this way, DLI enables you to perform query and analysis of EB's of data within hours.
                                      • Flink is a distributed compute engine that is ideal for batch processing, that is, for processing static data sets and historical data sets. You can also use it for stream processing, that is, processing real-time data streams and generating data results in real time. DLI enhances features and security based on the open-source Flink and provides the Stream SQL feature required for data processing.
                                      +

                                      DLI Core Engine: Spark+Flink+Trino

                                      • Spark is a unified analysis engine that is ideal for large-scale data processing. It focuses on query, compute, and analysis. DLI optimizes performance and reconstructs services based on open-source Spark. It is compatible with the Apache Spark ecosystem and interfaces, and improves performance by 2.5x when compared with open-source Spark. In this way, DLI enables you to perform query and analysis of EB's of data within hours.
                                      • Flink is a distributed compute engine that is ideal for batch processing, that is, for processing static data sets and historical data sets. You can also use it for stream processing, that is, processing real-time data streams and generating data results in real time. DLI enhances features and security based on the open-source Flink and provides the Stream SQL feature required for data processing.

                                      Serverless Architecture

                                      DLI is a serverless big data query and analysis service. It has the following advantages:

                                      • Auto scaling: DLI ensures you always have enough capacity on hand to deal with any traffic spikes.
                                      @@ -74,6 +75,8 @@
                                      • Using APIs

                                        If you need to integrate DLI into a third-party system for secondary development, you can call DLI APIs to use the service.

                                        For details, see Data Lake Insight API Reference.

                                        +
                                      • DataArts Studio

                                        DataArts Studio is a one-stop data operations platform that provides intelligent data lifecycle management. It supports intelligent construction of industrial knowledge libraries and incorporates data foundations such as big data storage, computing, and analysis engines. With DataArts Studio, your company can easily construct end-to-end intelligent data systems. These systems can help eliminate data silos, unify data standards, accelerate data monetization, and promote digital transformation.

                                        +

                                        Create a data connection on the DataArts Studio management console to access DLI for data analysis.

                                      diff --git a/docs/dli/umn/dli_01_0379.html b/docs/dli/umn/dli_01_0379.html index 0047decef..14ff1483b 100644 --- a/docs/dli/umn/dli_01_0379.html +++ b/docs/dli/umn/dli_01_0379.html @@ -1,17 +1,24 @@ -

                                      Job Templates

                                      -
                                      +

                                      Querying Logs for Spark Jobs

                                      +

                                      Scenario

                                      DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.

                                      +

                                      This section describes how to configure a bucket for DLI jobs on the DLI console and obtain Spark job logs.

                                      +
                                      +

                                      Notes

                                      • To avoid disordered job results, do not use the OBS bucket configured for DLI jobs for any other purposes.
                                      • DLI jobs must be set and modified by the main account as IAM users do not have required permissions.
                                      • You cannot view the logs for DLI jobs before configuring a bucket.
                                      • You can configure lifecycle rules to periodically delete objects from buckets or change storage classes of objects.
                                      • Exercise caution when modifying the job bucket, as it may result in the inability to retrieve historical data.
                                      • Spark log splitting rules:
                                        • Split by size: By default, each log file has a maximum size of 128 MB.
                                        • Split by time: A new log file is automatically created every hour.
                                        +
                                      +
                                      +

                                      Prerequisites

                                      Before the configuration, create an OBS bucket or parallel file system (PFS). In big data scenarios, you are advised to create a PFS. PFS is a high-performance file system provided by OBS, with access latency in milliseconds. PFS can achieve a bandwidth performance of up to TB/s and millions of IOPS, which makes it ideal for processing high-performance computing (HPC) workloads.

                                      +
                                      +

                                      Configuring a Bucket for DLI Jobs

                                      1. In the navigation pane of the DLI console, choose Global Configuration > Project.
                                      2. On the Project page, click next to Job Bucket to configure bucket information.
                                      3. Click to view available buckets.
                                      4. In the displayed OBS dialog box, click the name of a bucket or search for and click a bucket name and then click OK. In the Set Job Bucket dialog box, click OK.

                                        Temporary data generated during DLI job running will be stored in the OBS bucket.

                                        +
                                      +
                                      +

                                      Querying Logs for Spark Jobs

                                      1. Log in to the DLI console. In the navigation pane on the left, choose Job Management > Spark Jobs.
                                      2. Select the Spark job whose jobs you want to query, click More in the Operation column, and select View Log.

                                        The system automatically switches to the log path of the DLI job bucket.

                                        +
                                      3. On the Files tab, select the log file of the desired date and time and click Download in the Operation column to download the file to your local host.
                                      +
                                      +
                                      diff --git a/docs/dli/umn/dli_01_0384.html b/docs/dli/umn/dli_01_0384.html index 738016f57..5438fe108 100644 --- a/docs/dli/umn/dli_01_0384.html +++ b/docs/dli/umn/dli_01_0384.html @@ -3,89 +3,99 @@

                                      Creating a Spark Job

                                      DLI provides fully-managed Spark computing services by allowing you to execute Spark jobs.

                                      On the Overview page, click Create Job in the upper right corner of the Spark Jobs tab or click Create Job in the upper right corner of the Spark Jobs page. The Spark job editing page is displayed.

                                      -

                                      On the Spark job editing page, a message is displayed, indicating that a temporary DLI data bucket will be created. The created bucket is used to store temporary data generated by DLI, such as job logs and job results. You cannot view job logs if you choose not to create it. The bucket will be created and the default bucket name is used.

                                      +

                                      On the Spark job editing page, a message is displayed, indicating that a temporary DLI data bucket will be created. The created bucket is used to store temporary data generated by DLI, such as job logs and job results. You cannot view job logs if you choose not to create the bucket. The bucket will be created and the default bucket name is used.

                                      If you do not need to create a DLI temporary data bucket and do not want to receive this message, select Do not show again and click Cancel.

                                      -

                                      Prerequisites

                                      • You have uploaded the dependencies to the corresponding OBS bucket on the Data Management > Package Management page. For details, see Creating a Package.
                                      • Before creating a Spark job to access other external data sources, such as OpenTSDB, HBase, Kafka, GaussDB(DWS), RDS, CSS, CloudTable, DCS Redis, and DDS, you need to create a datasource connection to enable the network between the job running queue and external data sources.
                                        • For details about the external data sources that can be accessed by Spark jobs, see Cross-Source Analysis Development Methods.
                                        • For details about how to create a datasource connection, see Enhanced Datasource Connections.

                                          On the Resources > Queue Management page, locate the queue you have created, click More in the Operation column, and select Test Address Connectivity to check if the network connection between the queue and the data source is normal. For details, see Testing Address Connectivity.

                                          +

                                          Prerequisites

                                          Procedure

                                          1. In the left navigation pane of the DLI management console, choose Job Management > Spark Jobs. The Spark Jobs page is displayed.

                                            Click Create Job in the upper right corner. In the job editing window, you can set parameters in Fill Form mode or Write API mode.

                                            -

                                            The following uses the Fill Form as an example. In Write API mode, refer to the Data Lake Insight API Reference for parameter settings.

                                            +

                                            The following uses the Fill Form as an example. In Write API mode, refer to the Data Lake Insight API Reference for parameter settings.

                                          1. Select a queue.

                                            Select the queue you want to use from the drop-down list box.

                                            +
                                          2. Select a Spark version. Select a supported Spark version from the drop-down list. The latest version is recommended.

                                            You are advised not to use Spark/Flink engines of different versions for a long time.

                                            +
                                            • Doing so can lead to code incompatibility, which can negatively impact job execution efficiency.
                                            • Doing so may result in job execution failures due to conflicts in dependencies. Jobs rely on specific versions of libraries or components.
                                            +
                                          3. Configure the job.

                                            Configure job parameters by referring to Table 1.

                                            -

                                      Advantage

                                      No Elastic Resource Pool

                                      +

                                      No Elastic Resource Pool

                                      Use Elastic Resource Pool

                                      +

                                      Use Elastic Resource Pool

                                      Efficiency

                                      +

                                      Efficiency

                                      You need to set scaling tasks repeatedly to improve the resource utilization.

                                      +

                                      You need to set scaling tasks repeatedly to improve the resource utilization.

                                      Dynamic scaling can be done in seconds.

                                      +

                                      Dynamic scaling can be done in seconds.

                                      Resource utilization

                                      +

                                      Resource utilization

                                      Resources cannot be shared among different queues.

                                      +

                                      Resources cannot be shared among different queues.

                                      For example, a queue has idle CUs and another queue is heavily loaded. Resources cannot be shared. You can only scale up the second queue.

                                      Queues added to the same elastic resource pool can share compute resources.

                                      +

                                      Queues added to the same elastic resource pool can share compute resources.

                                      When you set a data source, you must allocate different network segments to each queue, which requires a large number of VPC network segments.

                                      +

                                      When you set a data source, you must allocate different network segments to each queue, which requires a large number of VPC network segments.

                                      You can add multiple general-purpose queues in the same elastic resource pool to one network segment, simplifying the data source configuration.

                                      +

                                      You can add multiple general-purpose queues in the same elastic resource pool to one network segment, simplifying the data source configuration.

                                      Resource allocation

                                      +

                                      Resource allocation

                                      If resources are insufficient for scale-out tasks of multiple queues, some queues will fail to be scaled out.

                                      +

                                      If resources are insufficient for scale-out tasks of multiple queues, some queues will fail to be scaled out.

                                      You can set the priority for each queue in the elastic resource pool based on the peak hours to ensure proper resource allocation.

                                      +

                                      You can set the priority for each queue in the elastic resource pool based on the peak hours to ensure proper resource allocation.
                                      Table 1 Job configuration parameters

                                      Parameter

                                      +
                                      - - - - - - - - - + + + - - - - - - - - - - - - - - - - @@ -94,45 +104,45 @@ -
                                      Table 1 Job configuration parameters

                                      Parameter

                                      Description

                                      +

                                      Description

                                      Job Name (--name)

                                      +

                                      Job Name (--name)

                                      Set a job name.

                                      +

                                      Set a job name.

                                      Application

                                      +

                                      Application

                                      Select the package to be executed. The value can be .jar or .py.

                                      -

                                      You can select the name of a JAR or pyFile package that has been uploaded to the DLI resource management system. You can also specify an OBS path, for example, obs://Bucket name/Package name.

                                      -

                                      Spark 3.3.x or later supports only packages in OBS paths.

                                      +

                                      Select the package to be executed. The value can be .jar or .py.

                                      +

                                      There are the following ways to manage JAR files:

                                      +
                                      • Upload packages to OBS: Upload Jar packages to an OBS bucket in advance and select the corresponding OBS path.
                                      • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.
                                      +

                                      For Spark 3.3.x or later, you can only select packages in OBS paths.

                                      Main Class (--class)

                                      +

                                      Main Class (--class)

                                      Enter the name of the main class. When the application type is .jar, the main class name cannot be empty.

                                      +

                                      Enter the name of the main class. When the application type is .jar, the main class name cannot be empty.

                                      Spark Arguments (--conf)

                                      +

                                      Application Parameters

                                      Enter a parameter in the format of key=value. Press Enter to separate multiple key-value pairs.

                                      +

                                      User-defined parameters. Separate multiple parameters by Enter.

                                      +

                                      These parameters can be replaced with global variables. For example, if you create a global variable batch_num on the Global Configuration > Global Variables page, you can use {{batch_num}} to replace a parameter with this variable after the job is submitted.

                                      +

                                      Spark Arguments (--conf)

                                      +

                                      Enter a parameter in the format of key=value. Press Enter to separate multiple key-value pairs.

                                      These parameters can be replaced with global variables. For example, if you create a global variable custom_class on the Global Configuration > Global Variables page, you can use "spark.sql.catalog"={{custom_class}} to replace a parameter with this variable after the job is submitted.

                                      NOTE:
                                      • The JVM garbage collection algorithm cannot be customized for Spark jobs.
                                      • If the Spark version is 3.1.1, configure Spark parameters (--conf) to select a dependent module. For details about the example configuration, see Table 2.

                                      Job Type

                                      +

                                      Job Type

                                      Set this parameter when you select a CCE queue. Type of the Spark image used by a job. The options are as follows:

                                      -
                                      • Basic: Basic images provided by DLI. Select this option for non-AI jobs.
                                      • AI-enhanced: AI images provided by DLI. Select this option for AI jobs.
                                      • Image: Custom Spark images. Select an existing image name and version on SWR.
                                      +

                                      Set this parameter when you select a CCE queue. Type of the Spark image used by a job. The options are as follows:

                                      +
                                      • Basic: Base images provided by DLI. Select this option for non-AI jobs.
                                      • Image: Custom Spark images. Select an existing image name and version on SWR.

                                      JAR Package Dependencies (--jars)

                                      +

                                      JAR Package Dependencies (--jars)

                                      JAR file on which the Spark job depends. You can enter the JAR package name or the corresponding OBS path. The format is as follows: obs://Bucket name/Folder name/Package name.

                                      +

                                      JAR file on which the Spark job depends. You can enter the JAR file name or the OBS path of the JAR file in the format of obs://Bucket name/Folder path/JAR file name.

                                      Python File Dependencies (--py-files)

                                      +

                                      Python File Dependencies (--py-files)

                                      py-files on which the Spark job depends. You can enter the Python file name or the corresponding OBS path of the Python file. The format is as follows: obs://Bucket name/Folder name/File name.

                                      +

                                      py-files on which the Spark job depends. You can enter the Python file name or the corresponding OBS path of the Python file. The format is as follows: obs://Bucket name/Folder name/File name.

                                      Other Dependencies (--files)

                                      +

                                      Other Dependencies (--files)

                                      Other files on which the Spark job depends. You can enter the name of the dependency file or the corresponding OBS path of the dependency file. The format is as follows: obs://Bucket name/Folder name/File name.

                                      +

                                      Other files on which the Spark job depends. You can enter the name of the dependency file or the corresponding OBS path of the dependency file. The format is as follows: obs://Bucket name/Folder name/File name.

                                      Group Name

                                      +

                                      Group Name

                                      If you select a group when creating a package, you can select all the packages and files in the group. For details about how to create a package, see Creating a Package.

                                      +

                                      If you select a group when creating a package, you can select all the packages and files in the group. For how to create a package, see Creating a DLI Package.

                                      Access Metadata

                                      +

                                      Access Metadata

                                      Whether to access metadata through Spark jobs. .

                                      +

                                      Whether to access metadata through Spark jobs.

                                      Retry upon Failure

                                      +

                                      Retry upon Failure

                                      Indicates whether to retry a failed job.

                                      +

                                      Indicates whether to retry a failed job.

                                      If you select Yes, you need to set the following parameters:

                                      Maximum Retries: Maximum number of retry times. The maximum value is 100.

                                      Advanced Settings

                                      +

                                      Advanced Settings

                                      • Skip
                                      • Configure
                                        • Select Dependency Resources: For details about the parameters, see Table 3.
                                        • Configure Resources: For details about the parameters, see Table 4.
                                        +
                                      • Skip
                                      • Configure
                                        • Select Dependency Resources: For details about the parameters, see Table 3.
                                        • Configure Resources: For details about the parameters, see Table 4.
                                      Table 2 Spark Parameter (--conf) configuration

                                      Datasource

                                      +
                                      - - - - - - - - - - - - - @@ -144,70 +154,70 @@

                                      You can properly plan compute resource specifications based on the compute CUs of the queue you have purchased.

                                      Note that Spark tasks need to be jointly executed by multiple roles, such as driver and executor. So, the number of executors multiplied by the number of executor CPU cores must be less than the number of compute CUs of the queue to prevent other roles from failing to start Spark tasks. For more information about roles for Spark tasks, see Apache Spark.

                                      Calculation formula for Spark job parameters:

                                      -
                                      • CUs = Driver Cores + Executors x Executor Cores
                                      • Memory = Driver Memory + (Executors x Executor Memory)
                                      +
                                      • Number of CUs = Actual number of CUs = Max{(Driver Cores + Executors x Executor Cores), [(Driver Memory + Executors x Executor Memory)/4]}
                                      • Memory = Driver Memory + (Executors x Executor Memory)
                                      -
                                      Table 2 Spark Parameter (--conf) configuration

                                      Datasource

                                      Example Value

                                      +

                                      Example Value

                                      CSS

                                      +

                                      CSS

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/css/*

                                      +

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/css/*

                                      spark.executor.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/css/*

                                      DWS

                                      +

                                      GaussDB(DWS)

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/dws/*

                                      +

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/dws/*

                                      spark.executor.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/dws/*

                                      HBase

                                      +

                                      HBase

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/hbase/*

                                      +

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/hbase/*

                                      spark.executor.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/hbase/*

                                      OpenTSDB

                                      +

                                      OpenTSDB

                                      park.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/opentsdb/*

                                      +

                                      park.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/opentsdb/*

                                      spark.executor.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/opentsdb/*

                                      RDS

                                      +

                                      RDS

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/rds/*

                                      +

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/rds/*

                                      spark.executor.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/rds/*

                                      Redis

                                      +

                                      Redis

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/redis/*

                                      +

                                      spark.driver.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/redis/*

                                      spark.executor.extraClassPath=/usr/share/extension/dli/spark-jar/datasource/redis/*
                                      Table 3 Parameters for selecting dependency resources

                                      Parameter

                                      +
                                      - - - - -
                                      Table 3 Parameters for selecting dependency resources

                                      Parameter

                                      Description

                                      +

                                      Description

                                      modules

                                      +

                                      modules

                                      If the Spark version is 3.1.1, you do not need to select a module. Configure Spark parameters (--conf).

                                      +

                                      If the Spark version is 3.1.1, you do not need to select a module. Configure Spark parameters (--conf).

                                      Dependency modules provided by DLI for executing datasource connection jobs. To access different services, you need to select different modules.
                                      • MRS HBase: sys.datasource.hbase
                                      • DDS: sys.datasource.mongo
                                      • MRS OpenTSDB: sys.datasource.opentsdb
                                      • DWS: sys.datasource.dws
                                      • RDS MySQL: sys.datasource.rds
                                      • RDS PostGre: sys.datasource.rds
                                      • DCS: sys.datasource.redis
                                      • CSS: sys.datasource.css

                                      Resource Package

                                      +

                                      Resource Package

                                      JAR package on which the Spark job depends.

                                      +

                                      JAR package on which the Spark job depends.
                                      -
                                      Table 4 Resource specification parameters

                                      Parameter

                                      +
                                      - - - - - - - - - - - - - @@ -219,7 +229,7 @@ diff --git a/docs/dli/umn/dli_01_0385.html b/docs/dli/umn/dli_01_0385.html index f99ca6c6a..e9aa7d89f 100644 --- a/docs/dli/umn/dli_01_0385.html +++ b/docs/dli/umn/dli_01_0385.html @@ -1,58 +1,53 @@ -

                                      Spark Job Management

                                      -

                                      Based on the open-source Spark, DLI optimizes performance and reconstructs services to be compatible with the Apache Spark ecosystem and interfaces, and executes batch processing tasks.

                                      -

                                      DLI also allows you to use Spark jobs to access DLI metadata.

                                      -

                                      Spark job management provides the following functions:

                                      - -

                                      In addition, you can click Quick Links to switch to the details on User Guide.

                                      -

                                      Spark Jobs Page

                                      On the Overview page, click Spark Jobs to go to the SQL job management page. Alternatively, you can click Job Management > Spark Jobs. The page displays all Spark jobs. If there are a large number of jobs, they will be displayed on multiple pages. DLI allows you to view jobs in all statuses.

                                      +

                                      Managing Spark Jobs

                                      +

                                      Viewing Basic Information

                                      On the Overview page, click Spark Jobs to go to the SQL job management page. Alternatively, you can click Job Management > Spark Jobs. The page displays all Spark jobs. If there are a large number of jobs, they will be displayed on multiple pages. DLI allows you to view jobs in all statuses.

                                      -
                                      Table 4 Resource specification parameters

                                      Parameter

                                      Description

                                      +

                                      Description

                                      Resource Specifications

                                      +

                                      Resource Specifications

                                      Select a resource specification from the drop-down list box. The system provides three resource specification options for you to choose from.

                                      +

                                      Select a resource specification from the drop-down list box. The system provides three resource specification options for you to choose from.

                                      Resource specifications involve the following parameters:

                                      • Executor Memory
                                      • Executor Cores
                                      • Executors
                                      • Driver Cores
                                      • Driver Memory

                                      If modified, your modified settings of the items are used.

                                      Executor Memory

                                      +

                                      Executor Memory

                                      Customize the configuration item based on the selected resource specifications.

                                      +

                                      Customize the configuration item based on the selected resource specifications.

                                      Memory of each Executor. It is recommended that the ratio of Executor CPU cores to Executor memory be 1:4.

                                      Executor Cores

                                      +

                                      Executor Cores

                                      Number of CPU cores of each Executor applied for by Spark jobs, which determines the capability of each Executor to execute tasks concurrently.

                                      +

                                      Number of CPU cores of each Executor applied for by Spark jobs, which determines the capability of each Executor to execute tasks concurrently.

                                      Executors

                                      +

                                      Executors

                                      Number of Executors applied for by a Spark job

                                      +

                                      Number of Executors applied for by a Spark job

                                      Driver Cores

                                      +

                                      Driver Cores

                                      Number of CPU cores of the driver

                                      +

                                      Number of CPU cores of the driver

                                      Driver Memory

                                      +

                                      Driver Memory

                                      Driver memory size. It is recommended that the ratio of the number of driver CPU cores to the driver memory be 1:4.

                                      +

                                      Driver memory size. It is recommended that the ratio of the number of driver CPU cores to the driver memory be 1:4.
                                      Table 1 Job management parameters

                                      Parameter

                                      +
                                      - - - - - - - - - - - - - - - - - @@ -61,16 +56,16 @@
                                      Table 1 Job management parameters

                                      Parameter

                                      Description

                                      +

                                      Description

                                      Job ID

                                      +

                                      Job ID

                                      ID of a submitted Spark job, which is generated by the system by default.

                                      +

                                      ID of a submitted Spark job, which is generated by the system by default.

                                      Name

                                      +

                                      Name

                                      Name of a submitted Spark job.

                                      +

                                      Name of a submitted Spark job.

                                      Queues

                                      +

                                      Queues

                                      Queue where the submitted Spark job runs

                                      +

                                      Queue where the submitted Spark job runs

                                      Username

                                      +

                                      Username

                                      Name of the user who executed the Spark job

                                      +

                                      Name of the user who executed the Spark job

                                      Status

                                      +

                                      Status

                                      Job status. The following values are available:

                                      +

                                      Job status. The following values are available:

                                      • Starting: The job is being started.
                                      • Running: The job is being executed.
                                      • Failed: The session has exited.
                                      • Finished: The session is successfully executed.
                                      • Restoring: The job is being restored.

                                      Created

                                      +

                                      Created

                                      Time when a job is created. Jobs can be displayed in ascending or descending order of the job creation time.

                                      +

                                      Time when a job is created. Jobs can be displayed in ascending or descending order of the job creation time.

                                      Last Modified

                                      +

                                      Last Modified

                                      Time when a job is completed.

                                      +

                                      Time when a job is completed.

                                      Operation

                                      +

                                      Operation

                                      • Edit: You can modify the current job configuration and re-execute the job.
                                      • SparkUI: After you click this button, the Spark job execution page is displayed.
                                        NOTE:
                                        • The SparkUI page cannot be viewed for jobs in the Starting state.
                                        • Currently, only the latest 100 job information records are displayed on the SparkUI of DLI.
                                        +
                                      • Edit: You can modify the current job configuration and re-execute the job.
                                      • SparkUI: After you click this button, the Spark job execution page is displayed.
                                        NOTE:
                                        • The SparkUI page cannot be viewed for jobs in the Starting state.
                                        • Currently, only the latest 100 job information records are displayed on the SparkUI of DLI.
                                      • Terminate Job: Cancel a job that is being started or running.
                                      • Re-execute: Run the job again.
                                      • Archive Log: Save job logs to the temporary bucket created by DLI.
                                      • Commit Log: View the logs of submitted jobs.
                                      • Driver Log: View the logs of running jobs.
                                      -

                                      Re-executing a Job

                                      On the Spark Jobs page, click Edit in the Operation column of the job. On the Spark job creation page that is displayed, modify parameters as required and execute the job.

                                      +

                                      Re-executing a Job

                                      On the Spark Jobs page, click Edit in the Operation column of the job. On the Spark job creation page that is displayed, modify parameters as required and execute the job.

                                      -

                                      Searching for a Job

                                      On the Spark Jobs page, select Status or Queues. The system displays the jobs that meet the filter condition in the job list.

                                      +

                                      Searching for a Job

                                      On the Spark Jobs page, select Status or Queues. The system displays the jobs that meet the filter condition in the job list.

                                      -

                                      Terminating a Job

                                      On the Spark Jobs page, choose More > Terminate Job in the Operation column of the job that you want to stop.

                                      +

                                      Terminating a Job

                                      On the Spark Jobs page, choose More > Terminate Job in the Operation column of the job that you want to stop.

                                      diff --git a/docs/dli/umn/dli_01_0389.html b/docs/dli/umn/dli_01_0389.html index 463ca9521..39c4e5575 100644 --- a/docs/dli/umn/dli_01_0389.html +++ b/docs/dli/umn/dli_01_0389.html @@ -1,33 +1,23 @@ -

                                      Flink Job Management

                                      +

                                      Submitting a Flink Job Using DLI

                                      diff --git a/docs/dli/umn/dli_01_0390.html b/docs/dli/umn/dli_01_0390.html index 5dd040794..3e4dbb75d 100644 --- a/docs/dli/umn/dli_01_0390.html +++ b/docs/dli/umn/dli_01_0390.html @@ -1,35 +1,19 @@ -

                                      Databases and Tables

                                      +

                                      Creating Databases and Tables

                                      diff --git a/docs/dli/umn/dli_01_0397.html b/docs/dli/umn/dli_01_0397.html index df3b613a0..0b7a613ad 100644 --- a/docs/dli/umn/dli_01_0397.html +++ b/docs/dli/umn/dli_01_0397.html @@ -1,6 +1,6 @@ -

                                      Built-in Dependencies

                                      +

                                      DLI Built-in Dependencies

                                      DLI built-in dependencies are provided by the platform by default. In case of conflicts, you do not need to upload them when packaging JAR packages of Spark or Flink Jar jobs.

                                      Spark 3.1.1 Dependencies

                                      Table 1 Spark 3.1.1 dependencies

                                      Dependency

                                      @@ -2963,7 +2963,7 @@ diff --git a/docs/dli/umn/dli_01_0402.html b/docs/dli/umn/dli_01_0402.html index 5368bf346..f70473ad4 100644 --- a/docs/dli/umn/dli_01_0402.html +++ b/docs/dli/umn/dli_01_0402.html @@ -1,128 +1,23 @@ -

                                      Overview

                                      -

                                      Queue

                                      Queues in DLI are computing resources, which are the basis for using DLI. All executed jobs require computing resources.

                                      -

                                      Currently, DLI provides two types of queues: For SQL and For general purpose.

                                      -
                                      • For SQL: The queue is used to run SQL jobs.
                                      • For general purpose: The queue is used to run Spark programs, Flink SQL jobs, and Flink Jar jobs.
                                      -
                                      -

                                      Constraints

                                      • A queue named default is preset in DLI for you to experience. Resources are allocated on demand.
                                      • Queue types:
                                        • For SQL: Spark SQL jobs can be submitted to SQL queues.
                                        • For general purpose: The queue is used to run Spark programs, Flink SQL jobs, and Flink Jar jobs.
                                        -

                                        The queue type cannot be changed. If you want to use another queue type, purchase a new queue.

                                        -
                                      • The region of a queue cannot be changed.
                                      • Queues with 16 CUs do not support scale-out or scale-in.
                                      • Queues with 64 CUs do not support scale-in.
                                      • A newly created queue can be scaled in or out only after a job is executed on the queue.
                                      • DLI queues cannot access the Internet.

                                        -

                                        -
                                      -
                                      -

                                      Difference Between Computing and Storage Resources

                                      -
                                      - - - - - - - - - - - - - -
                                      Table 1 Difference between computing and storage resources

                                      Resource

                                      -

                                      How to Obtain

                                      -

                                      Function

                                      -

                                      Compute resource

                                      -

                                      Create queue on the DLI management console.

                                      -

                                      Used for executing queries.

                                      -

                                      Storage resource

                                      -

                                      DLI has a 5 GB quota.

                                      -

                                      Used for storing data in the database and DLI tables.

                                      -
                                      -
                                      -
                                      • Storage resources are internal storage resources of DLI for storing database and DLI tables and represent the amount of data stored in DLI.
                                      • By default, DLI provides a 5 GB quota for storage resources.
                                      • A queue named default is preset in DLI. If you are uncertain about the required queue capacity or have no available queue capacity to run queries, you can execute jobs using this queue.
                                      -
                                      • The default queue is used only for user experience. It may be occupied by multiple users at a time. Therefore, it is possible that you fail to obtain the resource for related operations. You are advised to use a self-built queue to execute jobs.
                                      -
                                      -
                                      -

                                      Dedicated Queue

                                      Resources of a dedicated queue are not released when the queue is idle. That is, resources are reserved regardless of whether the queue is used. Dedicated queues ensure that resources exist when jobs are submitted.

                                      -
                                      -

                                      Elastic Scaling

                                      DLI allows you to flexibly scale in or out queues on demand. After a queue with specified specifications is created, you can scale it in and out as required.

                                      -

                                      To change the queue specifications, see Elastic Scaling.

                                      -

                                      Scaling can be performed for a newly created queue only when jobs are running on this queue.

                                      -
                                      -
                                      -

                                      Scheduled Elastic Scaling

                                      DLI allows you to schedule tasks for periodic queue scaling. After creating a queue, the scheduled scaling tasks can be executed.

                                      -
                                      You can schedule an automatic scale-out/scale-in based on service requirements. The system periodically triggers queue scaling. For details, see Scheduling CU Changes.

                                      Scaling can be performed for a newly created queue only when jobs are running on this queue.

                                      -
                                      -
                                      -
                                      -

                                      Automatic Queue Scaling

                                      Flink jobs use queues. DLI can automatically trigger scaling for jobs based on the job size.

                                      -
                                      -

                                      Scaling can be performed for a newly created queue only when there are jobs running on this queue.

                                      -
                                      -

                                      Queue Management Page

                                      Queue Management provides the following functions:

                                      - -

                                      To receive notifications when a DLI job fails, SMN Administrator permissions are required.

                                      -
                                      -

                                      The queue list displays all queues created by you and the default queue. Queues are listed in chronological order by default in the queue list, with the most recently created queues displayed at the top.

                                      - -
                                      - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                                      Table 2 Parameter description

                                      Parameter

                                      -

                                      Description

                                      -

                                      Name

                                      -

                                      Name of a queue.

                                      -

                                      Type

                                      -

                                      Queue type.

                                      -
                                      • For SQL
                                      • For general purpose
                                      • Spark queue (compatible with earlier versions)
                                      -

                                      Specifications

                                      -

                                      Queue size. Unit: CU

                                      -

                                      CU is the pricing unit of queues. A CU consists of 1 vCPU and 4-GB memory. The computing capabilities of queues vary with queue specifications. The higher the specifications, the stronger the computing capability.

                                      -

                                      Actual CUs

                                      -

                                      Actual size of the current queue.

                                      -

                                      Elastic Scaling

                                      -

                                      Target CU value for scheduled scaling, or the maximum and minimum CU values of the current specifications.

                                      -

                                      Username

                                      -

                                      Queue owner

                                      -

                                      Description

                                      -

                                      Description of a queue specified during queue creation. If no description is provided, -- is displayed.

                                      -

                                      Operation

                                      -
                                      • Delete: Allow you to delete the selected queue. You cannot delete a queue where there are running jobs or jobs are being submitted.
                                      • Manage Permissions: You can view the user permissions corresponding to the queue and grant permissions to other users.
                                      • More
                                        • Restart: Forcibly restart a queue.
                                          NOTE:

                                          Only the SQL queue has the Restart operation.

                                          -
                                          -
                                        • Elastic Scaling: You can select Scale-out or Scale-in as required. The number of CUs after modification must be an integer multiple of 16.
                                        • Schedule CU Changes: You can set different queue sizes at different time or in different periods based on the service period or usage. The system automatically performs scale-out or scale-in as scheduled. The After Modification value must be an integer multiple of 16.
                                        • Modifying CIDR Block: When DLI enhanced datasource connection is used, the CIDR block of the DLI queue cannot overlap with that of the data source. You can modify the CIDR block as required.

                                          -
                                        • Test Address Connectivity: Test whether the queue is reachable to the specified address. The domain name and IP address are supported. The port can be specified.
                                        -
                                      -
                                      -
                                      -
                                      -
                                      +

                                      Submitting a SQL Job Using DLI

                                      +
                                      diff --git a/docs/dli/umn/dli_01_0403.html b/docs/dli/umn/dli_01_0403.html index 5caf870f4..348053943 100644 --- a/docs/dli/umn/dli_01_0403.html +++ b/docs/dli/umn/dli_01_0403.html @@ -1,85 +1,16 @@ -

                                      Overview

                                      -

                                      On the Job Management page of Flink jobs, you can submit a Flink job. Currently, the following job types are supported:

                                      - -
                                      • Flink SQL uses SQL statements to define jobs and can be submitted to any general purpose queue.
                                      • Flink Jar customizes a JAR package job based on Flink APIs. It runs on dedicated queues.
                                      -

                                      Flink job management provides the following functions:

                                      - -

                                      Flink Jobs Page

                                      On the Overview page, click Flink Jobs to go to the Flink job management page. Alternatively, you can choose Job Management > Flink Jobs from the navigation pane on the left. The page displays all Flink jobs. If there are a large number of jobs, they will be displayed on multiple pages. DLI allows you to view jobs in all statuses.

                                      - -
                                      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                                      Table 1 Job management parameters

                                      Parameter

                                      -

                                      Description

                                      -

                                      ID

                                      -

                                      ID of a submitted Flink job, which is generated by the system by default.

                                      -

                                      Name

                                      -

                                      Name of the submitted Flink job.

                                      -

                                      Type

                                      -

                                      Type of the submitted Flink job. Including:

                                      -
                                      • Flink SQL: Flink SQL jobs
                                      • Flink Jar: Flink Jar jobs
                                      -

                                      Status

                                      -

                                      Job statuses, including:

                                      -
                                      • Draft
                                      • Submitting
                                      • Submission failed
                                      • Running: After the job is submitted, a normal result is returned.
                                      • Running exception: The job stops running due to an exception.
                                      • Downloading
                                      • Idle
                                      • Stopping
                                      • Stopped
                                      • Stopping failed
                                      • Creating the savepoint
                                      • Completed
                                      -

                                      Description

                                      -

                                      Description of the submitted Flink job.

                                      -

                                      Username

                                      -

                                      Name of the user who submits a job.

                                      -

                                      Created

                                      -

                                      Time when a job is created.

                                      -

                                      Started

                                      -

                                      Time when a Flink job starts to run.

                                      -

                                      Duration

                                      -

                                      Time consumed by job running.

                                      -

                                      Operation

                                      -
                                      • Edit: Edit a created job. For details, see Editing a Job.
                                      • Start: Start and run a job. For details, see Starting a Job.
                                      • More
                                        • FlinkUI: After you click this button, the Flink job execution page is displayed.
                                          NOTE:

                                          When you execute a job on a created queue, the cluster is restarted. It takes about 10 minutes. If you click FlinkUI before the cluster is created, an empty projectID will be cached. The FlinkUI page cannot be displayed.

                                          -

                                          You are advised to use a dedicated queue so that the cluster will not be released. Alternatively, wait for a while after the job is submitted (the cluster is created), and then check FlinkUI.

                                          -
                                          -
                                        • Stop: Stop a Flink job. If this function is unavailable, jobs in the current status cannot be stopped.
                                        • Delete: Delete a job.
                                          NOTE:

                                          A deleted job cannot be restored.

                                          -
                                          -
                                        • Modify Name and Description: You can modify the name and description of a job. For details, see Modifying Name and Description.
                                        • Import Savepoint: Import the data exported from the original CS job. For details, see Importing to a Savepoint.
                                        • Trigger Savepoint: You can click this button for jobs in the Running status to save the job status. For details, see Triggering a Savepoint.
                                        • Permissions: You can view the user permissions corresponding to the job and grant permissions to other users. For details, see Managing Flink Job Permissions.
                                        • Runtime Configuration: You can enable Alarm Generation upon Job Exception and Auto Restart upon Exception. For details, see Runtime Configuration.
                                        +

                                        Flink Job Overview

                                        +

                                        DLI supports two types of Flink jobs:

                                        +
                                        • Flink OpenSource SQL job:
                                          • It is fully compatible with Flink of the community edition, ensuring that jobs can run smoothly on these Flink versions.
                                          • DLI Flink has expanded the support for connectors based on Flink of the community edition, supporting Redis and GaussDB(DWS) as new data source types. With this expansion, you can now utilize a wider range of data source types, providing greater flexibility and convenience when working with datasets.
                                          • Flink OpenSource SQL jobs are ideal for scenarios where stream processing logic can be defined and executed through SQL statements. This simplifies stream processing, allowing developers to focus more on implementing service logic.
                                          +

                                          For how to create a Flink OpenSource SQL job, see Creating a Flink OpenSource SQL Job.

                                          +
                                        • Flink Jar job:
                                          • DLI allows you to submit Flink jobs compiled into JAR files, providing higher flexibility and customization capabilities. It is applicable to scenarios where complex data processing is required.
                                          • If the connectors provided by Flink of the community edition cannot meet specific needs, you can use Jar jobs to implement custom connectors or data processing logic.
                                          • It is ideal for scenarios where user-defined functions (UDFs) or specific library integration are required. You can use the Flink ecosystem to implement advanced stream processing logic and status management.
                                          +

                                          For how to create a Flink Jar job, see Creating a Flink Jar Job.

                                        -
                                      -
                                      -
                                      diff --git a/docs/dli/umn/dli_01_0407.html b/docs/dli/umn/dli_01_0407.html index 57925345d..4df8604a1 100644 --- a/docs/dli/umn/dli_01_0407.html +++ b/docs/dli/umn/dli_01_0407.html @@ -1,60 +1,76 @@ -

                                      Overview

                                      -

                                      Package management provides the following functions:

                                      -
                                      • Managing Package Permissions
                                      • Creating a Package
                                      • Deleting a Package

                                        You can delete program packages in batches.

                                        +

                                        Package Management Overview

                                        +

                                        Before running DLI jobs, UDF JAR files or Jar job packages need to be uploaded to the cloud platform for unified management and maintenance.

                                        +

                                        There are two ways to manage packages:

                                        +
                                        • (Recommended) Upload packages to OBS: Upload Jar packages to an OBS bucket in advance and select the OBS path when configuring a job.
                                        • Upload packages to DLI: Upload Jar packages to an OBS bucket in advance, create a package on the Data Management > Package Management page of the DLI management console, and select the DLI package when configuring a job.
                                        +

                                        This section describes how to upload and manage packages on the DLI management console.

                                        +
                                        • When using Spark 3.3.1 or later or Flink1.15 or later to run jobs, you are advised to select packages stored in OBS.
                                        • When packaging Spark or Flink Jar jobs, do not upload the dependency packages that the platform already has to avoid conflicts with the built-in dependency packages of the platform. Refer to DLI Built-in Dependencies for built-in dependency packages.
                                        -
                                      • Modifying the Owner
                                      -

                                      Notes and Constraints

                                      • A package can be deleted, but a package group cannot be deleted.
                                      • The following types of packages can be uploaded:
                                        • JAR: JAR file
                                        • PyFile: User Python file
                                        • File: User file
                                        • ModelFile: User AI model file
                                        -
                                      -
                                      -

                                      Package Management Page

                                      -
                                      Table 1 Package management parameters

                                      Parameter

                                      +

                                      Notes and Constraints

                                      +
                                      - - - - +
                                      Table 1 Notes and constraints on package usage

                                      Item

                                      Description

                                      +

                                      Description

                                      Group Name

                                      +

                                      Package

                                      Name of the group to which the package belongs. If the package is not grouped, -- is displayed.

                                      +
                                      • A package can be deleted, but a package group cannot be deleted.
                                      • The following types of packages can be uploaded:
                                        • JAR: JAR file
                                        • PyFile: User Python file
                                        • File: User file
                                        • ModelFile: User AI model file
                                        +

                                      Package Name

                                      +
                                      +
                                      +
                                      +

                                      Package Management Page

                                      +
                                      + + + + - - - - - + + + - - - - - - - -
                                      Table 2 Package management parameters

                                      Parameter

                                      +

                                      Description

                                      +

                                      Group Name

                                      Name of a package.

                                      +

                                      Name of the group to which the package belongs. If the package is not grouped, -- is displayed.

                                      Owner

                                      +

                                      Package Name

                                      Name of the user who uploads the package.

                                      +

                                      Name of a package.

                                      Type

                                      +

                                      Owner

                                      Type of a package. The following package types are supported:

                                      +

                                      Name of the user who uploads the package.

                                      +

                                      Type

                                      +

                                      Type of a package. The following package types are supported:

                                      • JAR: JAR file
                                      • PyFile: User Python file
                                      • File: User file

                                      Status

                                      +

                                      Status

                                      Status of the package to be created.

                                      +

                                      Status of the package to be created.

                                      • Uploading: The file is being uploaded.
                                      • Finished: The resource package has been uploaded.
                                      • Failed: The resource package upload failed.

                                      Created

                                      +

                                      Created

                                      Time when a package is created.

                                      +

                                      Time when a package is created.

                                      Updated

                                      +

                                      Updated

                                      Time when the package is updated.

                                      +

                                      Time when the package is updated.

                                      Operation

                                      +

                                      Operation

                                      Manage Permissions: Manage user permissions for a package.

                                      +

                                      Manage Permissions: Manage user permissions for a package.

                                      Delete: Delete the package.

                                      More:

                                      • Modify Owner: Modify the owner of the package.
                                      @@ -67,7 +83,7 @@ diff --git a/docs/dli/umn/dli_01_0408.html b/docs/dli/umn/dli_01_0408.html index 97078b449..92875a062 100644 --- a/docs/dli/umn/dli_01_0408.html +++ b/docs/dli/umn/dli_01_0408.html @@ -6,8 +6,6 @@
                                      • Overview
                                        • -
                                      • Creating an IAM User and Granting Permissions
                                        -
                                      • Creating a Custom Policy
                                      • DLI Resources
                                        diff --git a/docs/dli/umn/dli_01_0410.html b/docs/dli/umn/dli_01_0410.html index 810edd3c2..f20ea2504 100644 --- a/docs/dli/umn/dli_01_0410.html +++ b/docs/dli/umn/dli_01_0410.html @@ -1,16 +1,16 @@ -

                                        Cross-Source Analysis Development Methods

                                        +

                                        Common Development Methods for DLI Cross-Source Analysis

                                        Cross-Source Analysis

                                        If DLI needs to access external data sources, you need to establish enhanced datasource connections to enable the network between DLI and the data sources, and then develop different types of jobs to access the data sources. This is the process of DLI cross-source analysis.

                                        This section describes how to develop data sources supported by DLI for cross-source analysis.

                                        Notes

                                        • Flink jobs can directly access DIS, OBS, and SMN data sources without using datasource connections.
                                        • You are advised to use enhanced datasource connections to connect DLI to data sources.
                                        -

                                        DLI Supported Data Sources

                                        Table 1 lists the data sources supported by DLI. For details about how to use the data sources, see Data Lake Insight SQL Syntax Reference.

                                        +

                                        DLI Supported Data Sources

                                        Table 1 lists the data sources supported by DLI. For how to use the data sources, see the Data Lake Insight SQL Syntax Reference.

                                        - @@ -148,7 +148,7 @@ diff --git a/docs/dli/umn/dli_01_0418.html b/docs/dli/umn/dli_01_0418.html index cd6d3aea7..5800d891c 100644 --- a/docs/dli/umn/dli_01_0418.html +++ b/docs/dli/umn/dli_01_0418.html @@ -9,17 +9,16 @@

                                        Process Flow

                                        Figure 1 Process for granting DLI permissions

                                        -
                                        1. Create a user group and grant the permission to it.

                                          Create a user group on the IAM console, and assign the DLI ReadOnlyAccess permission to the group.

                                          +
                                          1. Create a user group and grant the permission to it.

                                            Create a user group on the IAM console and grant the DLI ReadOnlyAccess permission to it.

                                          2. Create a user and add the user to the user group.

                                            Create a user on the IAM console and add the user to the group created in 1.

                                          3. Log in and verify the permission.

                                            Log in to the management console using the newly created user, and verify that the user's permissions.

                                            -
                                            • Choose Service List > Data Lake Insight. The DLI management console is displayed. If you can view the queue list on the Queue Management page but cannot buy DLI queues by clicking Buy Queue in the upper right corner (assume that the current permission contains only DLI ReadOnlyAccess), the DLI ReadOnlyAccess permission has taken effect.
                                            • Choose any other service in Service List. If a message appears indicating that you have insufficient permissions to access the service, the DLI ReadOnlyAccess permission has already taken effect.
                                            +
                                            • Choose Service List > Data Lake Insight. The DLI management console is displayed. If you can view the queue list on the Queue Management page but cannot buy DLI queues by clicking Buy Queue in the upper right corner (assume that the current permission contains only DLI ReadOnlyAccess), the DLI ReadOnlyAccess permission has taken effect.
                                            • Choose any other service in Service List. If a message appears indicating that you have insufficient permissions to access the service, the DLI ReadOnlyAccess permission has already taken effect.
                                          -

                                        -
                                        Parent topic: Permissions Management
                                        +
                                        Parent topic: Preparations
                                        diff --git a/docs/dli/umn/dli_01_0419.html b/docs/dli/umn/dli_01_0419.html new file mode 100644 index 000000000..a76418ab3 --- /dev/null +++ b/docs/dli/umn/dli_01_0419.html @@ -0,0 +1,93 @@ + + +

                                        DLI Agency Overview

                                        +

                                        What Is an Agency?

                                        Cloud services often interact with each other, with some of which dependent on other services. You can create an agency to delegate DLI to use other cloud services and perform resource O&M on your behalf.

                                        +

                                        For example, the AK/SK required by DLI Flink jobs is stored in DEW. To allow DLI to access DEW data during job execution, you need to provide an IAM agency to delegate the permissions to perform operations on DEW data to DLI.

                                        +
                                        Figure 1 DLI service agency
                                        +
                                        +

                                        DLI Agencies

                                        Before using DLI, you are advised to set up DLI agency permissions to ensure the proper functioning of DLI.

                                        +
                                        • By default, DLI provides the following agencies: dli_admin_agency, dli_management_agency, and dli_data_clean_agency. The names of these agencies are fixed, but the permissions contained in them can be customized. In other scenarios, you need to create custom agencies. For details about the agencies, see Table 1.
                                        • DLI upgrades dli_admin_agency to dli_management_agency to meet the demand for fine-grained agency permissions management. The new agency has the necessary permissions for datasource operations, notifications, and user authorization operations. For details, see Configuring DLI Agency Permissions.
                                        • To use Flink 1.15, Spark 3.3.1 (Spark general queue scenario), or a later version to execute jobs, perform the following operations:
                                          Create an agency on the IAM console and add the agency information to the job configuration. For details, see Creating a Custom DLI Agency.
                                          • Common scenarios for creating an agency: DLI is allowed to read and write data from and to OBS, dump logs, and read and write Flink checkpoints. DLI is allowed to access DEW to obtain data access credentials and access catalogs to obtain metadata.
                                          • You cannot use the default agency names dli_admin_agency, dli_management_agency, or dli_data_clean_agency. It must be unique.
                                          +
                                          +
                                        • If the engine version is earlier than Flink 1.15, dli_admin_agency is used by default during job execution. If the engine version is earlier than Spark 3.3.1, user authentication information (AK/SK and security token) is used during job execution.

                                          This means that jobs whose engine versions are earlier than Flink 1.15 or Spark 3.3.1 are not affected by the update of agency permissions and do not require custom agencies.

                                          +
                                        • To maintain compatibility with existing job agency permission requirements, dli_admin_agency will still be listed in the IAM agency list even after the update.
                                        +
                                        • Only the tenant account or a member account of user group admin can authorize the service.
                                        • Do not delete the agency created by the system by default.
                                        +
                                        + +
                                        Table 1 Supported data sources

                                        Service

                                        Spark SQL Job

                                        +

                                        SQL Job

                                        Spark Jar Job
                                        + + + + + + + + + + + + + + + + + + + + + +
                                        Table 1 DLI agencies

                                        Permission

                                        +

                                        Type

                                        +

                                        Description

                                        +

                                        dli_admin_agency

                                        +

                                        Default agency

                                        +

                                        This agency has been discarded and is not recommended. Upgrade the agency to dli_management_agency as soon as possible.

                                        +

                                        For details about how to update an agency, see Configuring DLI Agency Permissions.

                                        +

                                        dli_management_agency

                                        +

                                        Default agency

                                        +

                                        DLI system agency, which is used to delegate operation permissions to DLI so that DLI can use other cloud services and perform resource O&M operations on your behalf. This agency grants permissions for datasource operations, message notifications, and user authorization operations. For details about the permissions of an agency, see Table 2.

                                        +

                                        dli_data_clean_agency

                                        +

                                        Default agency, which needs to be authorized by users

                                        +

                                        Data cleanup agency, which is used to clean up data according to the lifecycle of a table and clean up lakehouse table data. You need to create a DLI agency named dli_data_clean_agency on IAM and grant permissions to it.

                                        +

                                        You need to create an agency and customize permissions for it. However, the agency name is fixed to dli_data_clean_agency.

                                        +

                                        For details about the permission policies of an agency, see Agency Permission Policies in Common Scenarios.

                                        +

                                        Other custom agencies

                                        +

                                        Custom agency

                                        +

                                        When using Flink 1.15, Spark 3.3, or a later version to execute jobs, create an agency on the IAM console and add new agency information to the job configuration. For details, see Creating a Custom DLI Agency.

                                        +

                                        Common scenarios for creating an agency: DLI is allowed to read and write data from and to OBS to transfer logs. DLI is allowed to access DEW to obtain data access credentials and access catalogs to obtain metadata.

                                        +

                                        You cannot use the default agency names dli_admin_agency, dli_management_agency, or dli_data_clean_agency. It must be unique.

                                        +

                                        For details about the permission policies of an agency, see Agency Permission Policies in Common Scenarios.

                                        +
                                        +
                                        + +
                                        + + + + + + + + + + + + + +
                                        Table 2 Permissions contained in the dli_management_agency agency

                                        Policy

                                        +

                                        Description

                                        +

                                        IAM ReadOnlyAccess

                                        +

                                        To authorize IAM users who have not logged in to DLI, you need to obtain their information. So, the permissions contained in the IAM ReadOnlyAccess policy are required.

                                        +

                                        DLI Datasource Connections Agency Access

                                        +

                                        Permissions to access and use VPCs, subnets, routes, and VPC peering connections

                                        +

                                        DLI Notification Agency Access

                                        +

                                        Permissions to send notifications through SMN when a job fails to be executed

                                        +
                                        +
                                        +
                                        +
                                        + + diff --git a/docs/dli/umn/dli_01_0420.html b/docs/dli/umn/dli_01_0420.html deleted file mode 100644 index 2667210ee..000000000 --- a/docs/dli/umn/dli_01_0420.html +++ /dev/null @@ -1,19 +0,0 @@ - - -

                                        Importing Data to a DLI Table

                                        -

                                        Importing Data Using OBS

                                        On the DLI management console, you can import data stored in OBS into DLI tables.

                                        -

                                        To import OBS data to a DLI table, either choose Data Management > Databases and Tables in the navigation pane on the left, locate a desired database, and click Tables in the Operation column, or choose SQL Editor in the navigation pane on the left.

                                        -

                                        For details, see Importing Data to the Table.

                                        -
                                        -

                                        Importing Data Using DIS

                                        Use the Data Ingestion Service (DIS) service to import data to DLI. You need to create a DIS stream.

                                        -

                                        -

                                        For details, see "Creating a DIS Stream" in Data Ingestion Service User Guide.

                                        -

                                        When configuring the DIS stream, set the Dump Destination to DLI and select the database and table in DLI.

                                        -
                                        -
                                        -
                                        -
                                        -
                                        Parent topic: Other Common Operations
                                        -
                                        -
                                        - diff --git a/docs/dli/umn/dli_01_0421.html b/docs/dli/umn/dli_01_0421.html index 9b18a6146..0549eaf29 100644 --- a/docs/dli/umn/dli_01_0421.html +++ b/docs/dli/umn/dli_01_0421.html @@ -13,7 +13,7 @@
                                        -
                                        Parent topic: Queue Management
                                        +
                                        Parent topic: Managing Queues
                                        diff --git a/docs/dli/umn/dli_01_0422.html b/docs/dli/umn/dli_01_0422.html index ddfaa2934..0c81534b2 100644 --- a/docs/dli/umn/dli_01_0422.html +++ b/docs/dli/umn/dli_01_0422.html @@ -1,6 +1,6 @@ -

                                        Datasource Authentication

                                        +

                                        Using DLI Datasource Authentication to Manage Access Credentials for Data Sources

                                        diff --git a/docs/dli/umn/dli_01_0426.html b/docs/dli/umn/dli_01_0426.html index 8be3272ea..9463f9968 100644 --- a/docs/dli/umn/dli_01_0426.html +++ b/docs/dli/umn/dli_01_0426.html @@ -1,31 +1,19 @@ -

                                        Enhanced Datasource Connections

                                        +

                                        Configuring the Network Connection Between DLI and Data Sources (Enhanced Datasource Connection)

                                        diff --git a/docs/dli/umn/dli_01_0427.html b/docs/dli/umn/dli_01_0427.html index 813298be6..edeeb87fd 100644 --- a/docs/dli/umn/dli_01_0427.html +++ b/docs/dli/umn/dli_01_0427.html @@ -54,7 +54,7 @@ diff --git a/docs/dli/umn/dli_01_0440.html b/docs/dli/umn/dli_01_0440.html index 55c6c466b..b37f04734 100644 --- a/docs/dli/umn/dli_01_0440.html +++ b/docs/dli/umn/dli_01_0440.html @@ -71,84 +71,72 @@

                                        DLI Permission Types

                                        Table 2 lists the DLI service permissions. For details about the resources that can be controlled by DLI, see Table 4.

                                        -
                                        Table 2 DLI permission types

                                        Permission Type

                                        +
                                        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/dli/umn/dli_01_0443.html b/docs/dli/umn/dli_01_0443.html index 498a9c893..7de634b1a 100644 --- a/docs/dli/umn/dli_01_0443.html +++ b/docs/dli/umn/dli_01_0443.html @@ -1,15 +1,16 @@ -

                                        Modifying the CIDR Block

                                        +

                                        Changing the CIDR Block for a Standard Queue

                                        If the CIDR block of the DLI queue conflicts with that of the user data source, you can change the CIDR block of the queue.

                                        -

                                        If the queue whose CIDR block is to be modified has jobs that are being submitted or running, or the queue has been bound to enhanced datasource connections, the CIDR block cannot be modified.

                                        +

                                        If the queue whose CIDR block is to be modified has jobs that are being submitted or running, or the queue has been bound to enhanced datasource connections, the CIDR block cannot be modified.

                                        +

                                        The operations described in this section only apply to standard queues.

                                        -

                                        Procedure

                                        1. On the left of the DLI management console, click Resources >Queue Management.
                                        2. Select the queue to be modified and click Modify CIDR Block in the Operation column.
                                        3. Enter the required CIDR block and click OK. After the CIDR block of the queue is successfully changed, wait for 5 to 10 minutes until the cluster to which the queue belongs is restarted and then run jobs on the queue.
                                        +

                                        Procedure

                                        1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                                        2. Select the queue to be modified and click Modify CIDR Block in the Operation column.
                                        3. Enter the required CIDR block and click OK. After the CIDR block of the queue is successfully changed, wait for 5 to 10 minutes until the cluster to which the queue belongs is restarted and then run jobs on the queue.
                                        -
                                        Parent topic: Queue Management
                                        +
                                        Parent topic: Managing Queues
                                        diff --git a/docs/dli/umn/dli_01_0445.html b/docs/dli/umn/dli_01_0445.html index 59ecdd159..d99131da0 100644 --- a/docs/dli/umn/dli_01_0445.html +++ b/docs/dli/umn/dli_01_0445.html @@ -1,6 +1,6 @@ -

                                        Viewing Monitoring Metrics

                                        +

                                        Using Cloud Eye to Monitor DLI

                                        Description

                                        This section describes metrics reported by DLI to Cloud Eye as well as their namespaces and dimensions. You can use the management console or APIs provided by Cloud Eye to query the metrics of the monitored object and alarms generated for DLI.

                                        Namespace

                                        SYS.DLI

                                        @@ -155,6 +155,7 @@
                                        @@ -168,6 +169,7 @@ @@ -181,6 +183,7 @@ @@ -194,6 +197,7 @@ @@ -207,6 +211,7 @@ @@ -353,7 +358,7 @@ - @@ -407,9 +412,4 @@

                                        Viewing DLI Monitoring Metrics on Cloud Eye

                                        1. Search for Cloud Eye on the management console.
                                        2. In the navigation pane on the left of the Cloud Eye console, click Cloud Service Monitoring > Data Lake Insight.
                                        3. Select a queue to view its information.
                                        -
                                        -
                                        -
                                        Parent topic: Other Common Operations
                                        -
                                        -
                                        diff --git a/docs/dli/umn/dli_01_0447.html b/docs/dli/umn/dli_01_0447.html index 46b309c2a..45fe1cf34 100644 --- a/docs/dli/umn/dli_01_0447.html +++ b/docs/dli/umn/dli_01_0447.html @@ -1,9 +1,9 @@ -

                                        Managing Database Permissions

                                        +

                                        Configuring Database Permissions on the DLI Console

                                        Scenario

                                        • By setting permissions, you can assign varying database permissions to different users.
                                        • The administrator and database owner have all permissions, which cannot be set or modified by other users.
                                        -

                                        Precautions

                                        • Lower-level objects automatically inherit permissions granted to upper-level objects. The hierarchical relationship is database > table > column.
                                        • The database owner, table owner, and authorized users can assign permissions on the database and tables.
                                        • Columns can only inherit the query permission. For details about Inheritable Permissions, see Managing Database Permissions.
                                        • The permissions can be revoked only at the initial level to which the permissions are granted. You need to grant and revoke permissions at the same level. You need to grant and revoke permissions at the same level. For example, after you are granted the insertion permission on a database, you can obtain the insertion permission on the tables in the database. Your insertion permission can be revoked only at the database level.
                                        • If you create a database with the same name as a deleted database, the database permissions will not be inherited. In this case, you need to grant the database permissions to users or projects.

                                          For example, user A is granted with the permission to delete the testdb database. Delete the database and create another one with the same name. You need to grant user A the deletion permission of the testdb database again.

                                          +

                                          Precautions

                                          • Lower-level objects automatically inherit permissions granted to upper-level objects. The hierarchical relationship is database > table > column.
                                          • The database owner, table owner, and authorized users can assign permissions on the database and tables.
                                          • Columns can only inherit the query permission. For details about Inheritable Permissions, see Configuring Database Permissions on the DLI Console.
                                          • The permissions can be revoked only at the initial level to which the permissions are granted. You need to grant and revoke permissions at the same level. You need to grant and revoke permissions at the same level. For example, after you are granted the insertion permission on a database, you can obtain the insertion permission on the tables in the database. Your insertion permission can be revoked only at the database level.
                                          • If you create a database with the same name as a deleted database, the database permissions will not be inherited. In this case, you need to grant the database permissions to users or projects.

                                            For example, user A is granted with the permission to delete the testdb database. Delete the database and create another one with the same name. You need to grant user A the deletion permission of the testdb database again.

                                          Viewing Database Permissions

                                          1. On the left of the management console, choose Data Management > Databases and Tables.
                                          2. Locate the row where the target database resides and click Manage Permissions in the Operation column.

                                            Permissions can be granted to new users or projects, modified for users or projects with existing permissions, or revoked from a user or project.

                                            @@ -61,13 +61,13 @@

                                            Modifying Permissions for an Existing User or Project

                                            For a user or project that has some permissions on the database, you can revoke the existing permissions or grant new ones.

                                            If the options in Set Permission are gray, the corresponding account does not have the permission to modify the database. You can apply to the administrator, database owner, or other authorized users for granting and revoking permissions of databases.

                                            -
                                            1. In the User Permission Info list, find the user whose permission needs to be set.
                                              • If the user is a sub-user, you can set permissions for it.
                                              • If the user is already an administrator, you can only view the permissions information.
                                              +
                                              1. In the User Permission Info list, find the user whose permission needs to be set.
                                                • If the user is an IAM user, you can set permissions for it.
                                                • If the user is already an administrator, you can only view the permissions information.

                                                In the Project Permission Info list, locate the project for which you want to set permissions and click Set Permission.

                                                -
                                              2. In the Operation column of the sub-user or project, click Set Permission. The Set Permission dialog box is displayed.

                                                For details about the permissions of database users or projects, see Table 1.

                                                +
                                              3. In the Operation column of the IAM user or project, click Set Permission. The Set Permission dialog box is displayed.

                                                For details about the permissions of database users or projects, see Table 1.

                                              4. Click OK.
                                            -

                                            Revoking All Permissions of a User or Project

                                            Revoke all permissions of a user or a project.
                                            • In the user list under User Permission Info, locate the row where the target sub-user resides and click Revoke Permission in the Operation column. In the displayed dialog box, click OK. In this case, the user has no permissions on the database.

                                              If a user is an administrator, Revoke Permission is gray, indicating that the user's permission cannot be revoked.

                                              +

                                              Revoking All Permissions of a User or Project

                                              Revoke all permissions of a user or a project.
                                              • In the user list under User Permission Info, locate the row where the target IAM user resides and click Revoke Permission in the Operation column. In the displayed dialog box, click OK. In this case, the user has no permissions on the database.

                                                If a user is an administrator, Revoke Permission is gray, indicating that the user's permission cannot be revoked.

                                              • In the Project Permission Info area, select the project whose permissions need to be revoked and click Revoke Permission in the Operation column. After you click OK, the project does not have any permissions on the database.
                                              @@ -75,7 +75,7 @@
                                              diff --git a/docs/dli/umn/dli_01_0448.html b/docs/dli/umn/dli_01_0448.html index 0507e6750..8851ae2f9 100644 --- a/docs/dli/umn/dli_01_0448.html +++ b/docs/dli/umn/dli_01_0448.html @@ -1,6 +1,6 @@ -

                                              Managing Table Permissions

                                              +

                                              Configuring Table Permissions on the DLI Console

                                              Operation Scenario

                                              • By setting permissions, you can assign varying table permissions to different users.
                                              • The administrator and table owner have all permissions, which cannot be set or modified by other users.
                                              • When setting table permissions for a new user, ensure that the user group the user belongs to has the Tenant Guest permission.

                                              Precautions

                                              • If you create a table with the same name as a deleted table, the table permissions will not be inherited. In this case, you need to grant the table permissions to users or projects.

                                                For example, user A is granted with the permission to delete the testTable table. Delete the table and create another one with the same name. You need to grant user A the deletion permission of the testTable table again.

                                                @@ -55,21 +55,21 @@
                                        - - - -
                                        Table 2 DLI permission types

                                        Permission Type

                                        Subtype

                                        +

                                        Subtype

                                        Console Operations

                                        +

                                        Console Operations

                                        SQL Syntax

                                        -

                                        API Definition

                                        +

                                        SQL Syntax

                                        Queue Permissions

                                        +

                                        Queue Permissions

                                        Queue management permissions

                                        +

                                        Queue management permissions

                                        For details, see Queue Permission Management.

                                        +

                                        For details, see Queue Permission Management.

                                        None

                                        -

                                        For details, see "Granting Users with the Queue Usage Permission" in the Data Lake Insight API Reference.

                                        +

                                        None

                                        Queue usage permission

                                        +

                                        Queue usage permission

                                        Data Permissions

                                        +

                                        Data Permissions

                                        Database permissions

                                        +

                                        Database permissions

                                        For details, see Managing Database Permissions and Managing Table Permissions.

                                        +

                                        For details, see Configuring Database Permissions on the DLI Console and Configuring Table Permissions on the DLI Console.

                                        For details, see SQL Syntax of Batch Jobs > Data Permissions Management > Data Permissions List in the Data Lake Insight SQL Syntax Reference.

                                        -

                                        For details, see Permission-related APIs > Granting Users with the Data Usage Permission in the Data Lake Insight API Reference.

                                        +

                                        For details, see SQL Syntax of Batch Jobs > Data Permissions Management > Data Permissions List in the Data Lake Insight SQL Syntax Reference.

                                        Table permissions

                                        +

                                        Table permissions

                                        Column permissions

                                        +

                                        Column permissions

                                        Job Permissions

                                        +

                                        Job Permissions

                                        Flink job permissions

                                        +

                                        Flink job permissions

                                        For details, see Managing Flink Job Permissions.

                                        +

                                        For details, see Configuring Flink Job Permissions.

                                        None

                                        -

                                        For details, see Permission-related APIs > Granting Users with the Data Usage Permission in the Data Lake Insight API Reference.

                                        +

                                        None

                                        Package Permissions

                                        +

                                        Package Permissions

                                        Package group permissions

                                        +

                                        Package group permissions

                                        For details, see Managing Permissions on Packages and Package Groups.

                                        +

                                        For details, see Configuring DLI Package Permissions.

                                        None

                                        -

                                        For details, see Permission-related APIs > Granting Users with the Data Usage Permission in the Data Lake Insight API Reference.

                                        +

                                        None

                                        Package permissions

                                        +

                                        Package permissions

                                        Datasource Connection Permissions

                                        +

                                        Datasource Connection Permissions

                                        Datasource connection permissions

                                        +

                                        Datasource connection permissions

                                        For details, see Datasource Authentication Permission Management.

                                        +

                                        For details, see Datasource Authentication Permission Management.

                                        None

                                        -

                                        For details, see Permission-related APIs > Granting Users with the Data Usage Permission in the Data Lake Insight API Reference.

                                        +

                                        None

                                        Queue CPU Usage

                                        Displays the CPU usage of user queues.

                                        +

                                        This metric only applies to standard queues.

                                        0–100

                                        Queue Disk Usage

                                        Displays the disk usage of user queues.

                                        +

                                        This metric only applies to standard queues.

                                        0–100

                                        Max Disk Usage

                                        Displays the maximum disk usage of user queues.

                                        +

                                        This metric only applies to standard queues.

                                        0–100

                                        Queue Memory Usage

                                        Displays the memory usage of user queues.

                                        +

                                        This metric only applies to standard queues.

                                        0–100

                                        Used Memory

                                        Displays the memory usage rate of the user queues.

                                        +

                                        This metric only applies to standard queues.

                                        ≥ 0

                                        Flink Job Max Operator Latency

                                        Displays the maximum operator delay of a Flink job. The unit is ms.

                                        +

                                        Displays the maximum operator delay of a Flink job. The unit is ms.

                                        ≥ 0

                                        Authorization Object

                                        Select User or Project.

                                        +

                                        Select User or Project.

                                        Username/Project

                                        • If you select User, enter the IAM username when granting table permissions to the user.
                                          NOTE:

                                          The username is an existing IAM user name and has logged in to the DLI management console.

                                          +
                                        • If you select User, enter the IAM username when granting table permissions to the user.
                                          NOTE:

                                          The username is an existing IAM user name and has logged in to the DLI management console.

                                          -
                                        • If you select Project, select the project to be authorized in the current region.
                                          NOTE:

                                          If you select Project, you can only view information about the authorized tables and their databases.

                                          +
                                        • If you select Project, select the project to be authorized in the current region.
                                          NOTE:

                                          If you select Project, you can only view information about the authorized tables and their databases.

                                        Non-inheritable Permissions

                                        Select a permission to grant it to the user, or deselect a permission to revoke it.
                                        • The following permissions are applicable to both user and project authorization:
                                          • View Table Creation Statement: This permission allows you to view the statement for creating the current table.
                                          • View Table Information: This permission allows you to view information about the current table.
                                          • Select Table: This permission allows you to query data of the current table.
                                          • Drop Table: This permission allows you to delete the current table.
                                          • Rename Table: Rename the current table.
                                          • Insert: This permission allows you to insert data into the current table.
                                          • Overwrite: This permission allows you to insert data to overwrite the data in the current table.
                                          • Add Column: This permission allows you to add columns to the current table.
                                          • Grant Permission: This permission allows you to grant table permissions to other users or projects.
                                          • Revoke Permission: This permission allows you to revoke the table's permissions that other users or projects have but cannot revoke the table owner's permissions.
                                          • View Other Users' Permissions: This permission allows you to query other users' permission on the current table.
                                          +
                                        Select a permission to grant it to the user, or deselect a permission to revoke it.
                                        • The following permissions are applicable to both user and project authorization:
                                          • View Table Creation Statement: This permission allows you to view the statement for creating the current table.
                                          • View Table Information: This permission allows you to view information about the current table.
                                          • Select Table: This permission allows you to query data of the current table.
                                          • Drop Table: This permission allows you to delete the current table.
                                          • Rename Table: Rename the current table.
                                          • Insert: This permission allows you to insert data into the current table.
                                          • Overwrite: This permission allows you to insert data to overwrite the data in the current table.
                                          • Add Column: This permission allows you to add columns to the current table.
                                          • Grant Permission: This permission allows you to grant table permissions to other users or projects.
                                          • Revoke Permission: This permission allows you to revoke the table's permissions that other users or projects have but cannot revoke the table owner's permissions.
                                          • View Other Users' Permissions: This permission allows you to query other users' permission on the current table.
                                          The partition table also has the following permissions:
                                          • Add Partition: This permission allows you to add a partition to a partition table.
                                          • Delete Partition: This permission allows you to delete existing partitions from a partition table.
                                          • Configure Path for Partition: This permission allows you to set the path of a partition in a partition table to a specified OBS path.
                                          • Rename Table Partition: This permission allows you to rename partitions in a partition table.
                                          • Restore Table Partition: This permission allows you to export partition information from the file system and save the information to metadata.
                                          • View All Partitions: This permission allows you to view all partitions in a partition table.
                                        @@ -92,7 +92,7 @@

                                        Authorization Object

                                        Select User or Project.

                                        +

                                        Select User or Project.

                                        Username/Project

                                        @@ -122,15 +122,15 @@

                                        Modifying Permissions for an Existing User or Project

                                        For a user or project that has some permissions on the database, you can revoke the existing permissions or grant new ones.

                                        If all options under Set Permission are gray, you are not allowed to change permissions on this table. You can apply to the administrator, table owner, or other authorized users for granting and revoking table permissions.

                                        -
                                        1. In the User Permission Info list, find the user whose permission needs to be set.
                                          • If the user is a sub-user and is not the owner of the table, you can set permissions.
                                          • If the user is an administrator or table owner, you can only view permissions.
                                          +
                                          1. In the User Permission Info list, find the user whose permission needs to be set.
                                            • If the user is an IAM user and is not the owner of the table, you can set permissions.
                                            • If the user is an administrator or table owner, you can only view permissions.

                                            In the Project Permission Info list, locate the project for which you want to set permissions and click Set Permission.

                                            -
                                          2. In the Operation column of the sub-user or project, click Set Permission. The Set Permission dialog box is displayed.
                                            • For details about DLI table user or project permissions, see Table 1.
                                            • For details about OBS table user or project permissions, see Table 2.
                                            • For details about View table user or project permissions, see Table 3.
                                            +
                                          3. In the Operation column of the IAM user or project, click Set Permission. The Set Permission dialog box is displayed.
                                            • For details about DLI table user or project permissions, see Table 1.
                                            • For details about OBS table user or project permissions, see Table 2.
                                            • For details about View table user or project permissions, see Table 3.
                                          4. Click OK.

                                        Revoking All Permissions of a User or Project

                                        Revoke all permissions of a user or a project.

                                        -
                                        • In the user list under User Permission Info, locate the row where the target sub-user resides and click Revoke Permission in the Operation column. In the displayed dialog box, click OK. In this case, the user has no permissions on the table.

                                          In the following cases, Revoke Permission is gray, indicating that the permission of the user cannot be revoked.

                                          -
                                          • The user is an administrator.
                                          • The sub-user is the owner of the table.
                                          • The sub-user has only inheritable permissions.
                                          +
                                          • In the user list under User Permission Info, locate the row where the target IAM user resides and click Revoke Permission in the Operation column. In the displayed dialog box, click OK. In this case, the user has no permissions on the table.

                                            In the following cases, Revoke Permission is gray, indicating that the permission of the user cannot be revoked.

                                            +
                                            • The user is an administrator.
                                            • The IAM user is the owner of the table.
                                            • The IAM user has only inheritable permissions.
                                          • In the Project Permission Info area, select the project whose permissions need to be revoked and click Revoke Permission in the Operation column. After you click OK, the project does not have any permissions on the table.

                                            If a project has only inheritable permissions, Revoke Permission is gray, indicating that the permissions of the project cannot be revoked.

                                            @@ -139,7 +139,7 @@
                                          diff --git a/docs/dli/umn/dli_01_0454.html b/docs/dli/umn/dli_01_0454.html deleted file mode 100644 index c10532f05..000000000 --- a/docs/dli/umn/dli_01_0454.html +++ /dev/null @@ -1,28 +0,0 @@ - - -

                                          Preparing Flink Job Data

                                          -

                                          To create a Flink job, you need to enter the data source and data output channel, that is, source and sink. To use another service as the source or sink stream, you need to apply for the service first.

                                          -

                                          Flink jobs support the following data sources and output channels:

                                          -
                                          • DIS as the data input and output channel

                                            To use DIS as the data source and output channel, you need to enable DIS first.

                                            -

                                            For details about how to create a DIS stream, see Creating a DIS Stream in the Data Ingestion Service User Guide.

                                            -

                                            After applying for a DIS stream, you can upload local data to DIS to provide data sources for Flink jobs in real time. For details, see Sending Data to DIS in the Data Ingestion Service User Guide.

                                            -

                                            An example is provided as follows:

                                            -
                                            1,lilei,bmw320i,28
-2,hanmeimei,audia4,27
                                            -
                                          • OBS as the data source

                                            To use OBS as the data source, enable OBS first. For details about how to enable OBS, see Enabling OBS in the Object Storage Service Console Operation Guide.

                                            -

                                            After you enable OBS, upload local files to OBS using the Internet. For detailed operations, see Uploading a File in the Object Storage Service Console Operation Guide.

                                            -
                                          • RDS as the output channel

                                            To use RDS as the output channel, create an RDS instance. For details, see Creating a DB Instance in the Relational Database Service User Guide.

                                            -
                                          • SMN as the output channel

                                            To use SMN as the output channel, create an SMN topic to obtain the URN resource ID and then add topic subscription. For detailed operations, see Getting Started in the Simple Message Notification User Guide.

                                            -
                                          • Kafka as the data input and output channel

                                            If Kafka serves as both the source and sink streams, create an enhanced datasource connection between Flink jobs and Kafka. For details, see Enhanced Datasource Connections.

                                            -

                                            If the port of the Kafka server is listened on by the host name, you need to add the mapping between the host name and IP address of the Kafka Broker node to the datasource connection.

                                            -
                                          • CloudTable as the data input and output channel

                                            To use CloudTable as the data input and output channel, create a cluster in CloudTable and obtain the cluster ID.

                                            -
                                          • CSS as the output channel

                                            To use CSS as the data output channel, create a cluster in CSS and obtain the cluster's private network address. For details, see Getting Started in the Cloud Search Service User Guide.

                                            -
                                          • DCS as the output channel

                                            To use DCS as the output channel, create a Redis cache instance in DCS and obtain the address used for Flink jobs to connect to the Redis instance. For details, see "Buying a DCS Redis Instance" in Distributed Cache Service User Guide.

                                            -
                                          -
                                          -
                                          -
                                          -
                                          Parent topic: Flink Job Management
                                          -
                                          -
                                          - diff --git a/docs/dli/umn/dli_01_0455.html b/docs/dli/umn/dli_01_0455.html deleted file mode 100644 index 49ac41b50..000000000 --- a/docs/dli/umn/dli_01_0455.html +++ /dev/null @@ -1,181 +0,0 @@ - - -

                                          Creating a Flink SQL Job

                                          -

                                          This section describes how to create a Flink SQL job. You can use Flink SQLs to develop jobs to meet your service requirements. Using SQL statements simplifies logic implementation. You can edit Flink SQL statements for your job in the DLI SQL editor. This section describes how to use the SQL editor to write Flink SQL statements.

                                          -

                                          DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink 1.10 and 1.12 provided by the community. In addition, Redis, GaussDB(DWS), and DIS data source types are added based on the community connector.

                                          -

                                          Prerequisites

                                          • You have prepared the data input and data output channels. For details, see Preparing Flink Job Data.
                                          • When you use a Flink SQL job to access other external data sources, such as OpenTSDB, HBase, Kafka, GaussDB(DWS), RDS, CSS, CloudTable, DCS Redis, and DDS, you need to create a datasource connection to connect the job running queue to the external data source.
                                            • For details about the external data sources that can be accessed by Flink jobs, see Cross-Source Analysis Development Methods.
                                            • For details about how to create a datasource connection, see Enhanced Datasource Connections. After a datasource connection is created, you can choose More > Test Address Connectivity in the Operation column on the Queue Management page to check whether the network connection between the queue and the external data source is normal. For details, see Testing Address Connectivity.
                                            -
                                          -
                                          -

                                          Creating a Flink SQL Job

                                          1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                          2. In the upper right corner of the Flink Jobs page, click Create Job.
                                          3. Specify job parameters.

                                            -

                                            - - - - - - - - - - - - - - - - - - - -
                                            Table 1 Job configuration information

                                            Parameter

                                            -

                                            Description

                                            -

                                            Type

                                            -

                                            Set Type to Flink SQL. You will need to rewrite SQL statements to start the job.

                                            -

                                            Name

                                            -

                                            Name of a job. Enter 1 to 57 characters. Only letters, digits, hyphens (-), and underscores (_) are allowed.

                                            -
                                            NOTE:

                                            The job name must be globally unique.

                                            -
                                            -

                                            Description

                                            -

                                            Description of a job. It can contain a maximum of 512 characters.

                                            -

                                            Template Name

                                            -

                                            You can select a sample template or a custom job template. For details about templates, see Managing Flink Templates.

                                            -

                                            Tag

                                            -

                                            Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                            -
                                            NOTE:
                                            • A maximum of 20 tags can be added.
                                            • Only one tag value can be added to a tag key.
                                            • The key name in each resource must be unique.
                                            -
                                            -
                                            • Tag key: Enter a tag key name in the text box.
                                              NOTE:

                                              A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                              -
                                              -
                                            -
                                            • Tag value: Enter a tag value in the text box.
                                              NOTE:

                                              A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                              -
                                              -
                                            -
                                            -
                                            -

                                          4. Click OK to enter the editing page.
                                          5. Edit a Flink SQL job.

                                            Enter details SQL statements in the statement editing area. For details about SQL syntax, see the Data Lake Insight SQL Syntax Reference.

                                            -

                                          6. Click Check Semantics.

                                            • You can Debug or Start a job only after the semantic verification is successful.
                                            • If verification is successful, the message "The SQL semantic verification is complete. No error." will be displayed.
                                            • If verification fails, a red "X" mark will be displayed in front of each SQL statement that produced an error. You can move the cursor to the "X" mark to view error details and change the SQL statement as prompted.
                                            -

                                          7. Set job running parameters.

                                            -

                                            - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                                            Table 2 Running parameters

                                            Parameter

                                            -

                                            Description

                                            -

                                            Queue

                                            -

                                            A shared queue is selected by default. You can select a custom queue as needed.

                                            -
                                            NOTE:
                                            • During job creation, a sub-user can only select a queue that has been allocated to the user.
                                            • If the remaining capacity of the selected queue cannot meet the job requirements, the system automatically scales up the capacity. When a queue is idle, the system automatically scales in the queue.
                                            -
                                            -

                                            UDF Jar

                                            -

                                            If you selected custom queues, you need to configure this parameter.

                                            -

                                            You can customize a UDF Jar file. Before you select a JAR file, upload the corresponding JAR package to the OBS bucket and choose Data Management > Package Management to create a package. For details, see Creating a Package.

                                            -

                                            In SQL, you can call a user-defined function that is inserted into a JAR file.

                                            -

                                            CUs

                                            -

                                            Sum of the number of compute units and job manager CUs of DLI. One CU equals 1 vCPU and 4 GB.

                                            -

                                            The configured number of CUs is the number of CUs required for job running and cannot exceed the number of CUs in the bound queue.

                                            -

                                            Job Manager CUs

                                            -

                                            Number of CUs of the management unit.

                                            -

                                            Parallelism

                                            -

                                            Number of Flink SQL jobs that run at the same time Properly increasing the number of parallel threads improves the overall computing capability of the job. However, the switchover overhead caused by the increase of threads must be considered.

                                            -
                                            NOTE:
                                            • This value cannot be greater than four times the compute units (number of CUs minus the number of job manager CUs).
                                            • The priority of the number of parallel tasks on this page is lower than that set in the code.
                                            -
                                            -

                                            Task Manager Configuration

                                            -

                                            Whether to set Task Manager resource parameters

                                            -

                                            If this option is selected, you need to set the following parameters:

                                            -
                                            • CU(s) per TM: Number of resources occupied by each Task Manager.
                                            • Slot(s) per TM: Number of slots contained in each Task Manager.
                                            -

                                            OBS Bucket

                                            -

                                            OBS bucket to store job logs and checkpoint information. If the selected OBS bucket is not authorized, click Authorize.

                                            -
                                            NOTE:

                                            If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                                            -
                                            -

                                            Save Job Log

                                            -

                                            Whether to save the job running logs to OBS The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID. To go to this path, go to the job list and click the job name. On the Run Log tab page, click the provided OBS link.

                                            -
                                            CAUTION:

                                            You are advised to select this parameter. Otherwise, no run log is generated after the job is executed. If the job is abnormal, the run log cannot be obtained for fault locating.

                                            -
                                            -

                                            If this option is selected, you need to set the following parameters:

                                            -
                                            OBS Bucket: Select an OBS bucket to store user job logs. If the selected OBS bucket is not authorized, click Authorize.
                                            NOTE:

                                            If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                                            -
                                            -
                                            -

                                            Alarm Generation upon Job Exception

                                            -

                                            Whether to report job exceptions, for example, abnormal job running or exceptions due to an insufficient balance, to users via SMS or email

                                            -

                                            If this option is selected, you need to set the following parameters:

                                            -

                                            SMN Topic

                                            -

                                            Select a user-defined SMN topic. For details about how to create a custom SMN topic, see "Creating a Topic" in Simple Message Notification User Guide.

                                            -

                                            Enable Checkpointing

                                            -

                                            Whether to enable job snapshots. If this function is enabled, jobs can be restored based on the checkpoints.

                                            -
                                            If this option is selected, you need to set the following parameters:
                                            • Checkpoint Interval indicates the interval for creating checkpoints. The value ranges from 1 to 999999, and the default value is 30.
                                            • Checkpoint Mode can be set to either of the following values:
                                              • At least once: Events are processed at least once.
                                              • Exactly once: Events are processed only once.
                                              -
                                            -
                                            • OBS Bucket: Select an OBS bucket to store your checkpoints. If the selected OBS bucket is not authorized, click Authorize.
                                              The checkpoint path is Bucket name/jobs/checkpoint/Directory starting with the job ID.
                                              NOTE:

                                              If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                                              -
                                              -
                                              -
                                            -
                                            -

                                            Auto Restart upon Exception

                                            -

                                            Whether to enable automatic restart. If this function is enabled, any job that has become abnormal will be automatically restarted.

                                            -

                                            If this option is selected, you need to set the following parameters:

                                            -
                                            • Max. Retry Attempts: maximum number of retry times upon an exception. The unit is times/hour.
                                              • Unlimited: The number of retries is unlimited.
                                              • Limited: The number of retries is user-defined.
                                              -
                                            • Restore Job from Checkpoint: This parameter is available only when Enable Checkpointing is selected.
                                            -

                                            Idle State Retention Time

                                            -

                                            Defines for how long the state of a key is retained without being updated before it is removed in GroupBy or Window. The default value is 1 hour.

                                            -

                                            Dirty Data Policy

                                            -

                                            Select a policy for processing dirty data. The following policies are supported: Ignore, Trigger a job exception, and Save.

                                            -
                                            NOTE:

                                            Save indicates that the dirty data is stored to the OBS bucket selected above.

                                            -
                                            -

                                            Dirty Data Dump Address

                                            -

                                            Set this parameter when Dirty Data Policy is set to Save. Click the address box to select the OBS path for storing dirty data.

                                            -
                                            -
                                            -

                                          8. (Optional) Set the runtime configuration as required.
                                          9. Click Save.
                                          10. Click Start. On the displayed Start Flink Jobs page, confirm the job specifications, and click Start Now to start the job.

                                            After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column. After a job is successfully submitted, the job status will change from Submitting to Running. After the execution is complete, the message Completed is displayed.

                                            -

                                            If the job status is Submission failed or Running exception, the job submission failed or the job did not execute successfully. In this case, you can move the cursor over the status icon in the Status column of the job list to view the error details. You can click to copy error information. After handling the fault based on the provided information, resubmit the job.

                                            -

                                            Other available buttons are as follows:

                                            -
                                            • Save As: Save the created job as a new job.
                                            • Format: Format the SQL statements in the editing box.
                                            • Set as Template: Set the created SQL statements as a job template.
                                            • Theme Settings: Set the theme related parameters, including Font Size, Wrap, and Page Style.
                                            -
                                            -

                                          -
                                          -
                                          -
                                          -
                                          -
                                          Parent topic: Flink Job Management
                                          -
                                          -
                                          - diff --git a/docs/dli/umn/dli_01_0457.html b/docs/dli/umn/dli_01_0457.html index 63d5939e5..73ecc2617 100644 --- a/docs/dli/umn/dli_01_0457.html +++ b/docs/dli/umn/dli_01_0457.html @@ -1,49 +1,49 @@

                                          Creating a Flink Jar Job

                                          -

                                          This section describes how to create a Flink Jar job. You can perform secondary development based on Flink APIs, build your own JAR file, and submit the JAR file to DLI queues. DLI is fully compatible with open-source community APIs. To create a custom Flink job, you need to compile and build application JAR files. You must have a certain understanding of Flink secondary development and have high requirements related to stream computing complexity.

                                          -

                                          Prerequisites

                                          • Ensure that a dedicated queue has been created. To create a dedicated queue, select Dedicated Resource Mode when you choose the type of a queue during purchase.
                                          • When you use a Flink Jar job to access other external data sources, such as OpenTSDB, HBase, Kafka, GaussDB(DWS), RDS, CSS, CloudTable, DCS Redis, and DDS, you need to create a datasource connection to connect the job running queue to the external data source.
                                            • For details about the external data sources that can be accessed by Flink jobs, see Cross-Source Analysis Development Methods.
                                            • For details about how to create a datasource connection, see Enhanced Datasource Connections.

                                              On the Resources > Queue Management page, locate the queue you have created, and choose More > Test Address Connectivity in the Operation column to check whether the network connection between the queue and the data source is normal. For details, see Testing Address Connectivity.

                                              +

                                              A Flink Jar job involves developing a custom application Jar package based on Flink's capabilities and submitting it to a DLI queue for execution.

                                              +

                                              To create a Flink Jar job, you need to write and build your own application Jar package. This is suitable for users who require stream data processing and are proficient in Flink's secondary development capabilities.

                                              +

                                              This section describes how to create a Flink Jar job on the DLI management console.

                                              +

                                              Prerequisites

                                              • When you use a Flink Jar job to access other external data sources, such as OpenTSDB, HBase, Kafka, GaussDB(DWS), RDS, CSS, CloudTable, DCS Redis, and DDS, you need to create a datasource connection to connect the job running queue to the external data source. -
                                              • When running a Flink Jar job, you need to build the secondary development application code into a Jar package and upload the JAR package to the created OBS bucket. Choose Data Management > Package Management to create a package. For details, see Creating a Package.

                                                DLI does not support the download function. If you need to modify the uploaded data file, please edit the local file and upload it again.

                                                -
                                                -
                                              • Flink dependencies have been built in the DLI server and security hardening has been performed based on the open-source community version. To avoid dependency package compatibility issues or log output and dump issues, be careful to exclude the following files when packaging:
                                                • Built-in dependencies (or set the package dependency scope to provided in Maven or SBT)
                                                • Log configuration files (example, log4j.properties/logback.xml)
                                                • JAR package for log output implementation (example, log4j).
                                                +
                                              • To run a Flink Jar job, you need to build your custom application code into a JAR file and upload it to the OBS bucket that has already been created.
                                              • Flink dependencies have been built in the DLI server and security hardening has been performed based on the open-source community version. To avoid dependency package compatibility issues or log output and dump issues, be careful to exclude the following files when packaging:
                                                • Built-in dependencies (or set the package dependency scope to provided in Maven or SBT)
                                                • Log configuration files (example, log4j.properties/logback.xml)
                                                • JAR package for log output implementation (example, log4j).

                                              Creating a Flink Jar Job

                                              1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                              2. In the upper right corner of the Flink Jobs page, click Create Job.

                                              3. Specify job parameters.

                                                -

                                                Table 1 Job configuration information

                                                Parameter

                                                +
                                                - - - - - - - - - @@ -51,64 +51,77 @@
                                                Table 1 Job configuration information

                                                Parameter

                                                Description

                                                +

                                                Description

                                                Type

                                                +

                                                Type

                                                Select Flink Jar.

                                                +

                                                Select Flink Jar.

                                                Name

                                                +

                                                Name

                                                Name of a job. Enter 1 to 57 characters. Only letters, digits, hyphens (-), and underscores (_) are allowed.

                                                +

                                                Job name. Enter 1 to 57 characters. Only letters, numbers, hyphens (-), and underscores (_) are allowed.

                                                NOTE:

                                                The job name must be globally unique.

                                                Description

                                                +

                                                Description

                                                Description of a job. It can be up to 512 characters long.

                                                +

                                                Description of a job. It can be up to 512 characters long.

                                                Tag

                                                +

                                                Tags

                                                Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                                +

                                                Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                                For details, see .

                                                NOTE:
                                                • A maximum of 20 tags can be added.
                                                • Only one tag value can be added to a tag key.
                                                • The key name in each resource must be unique.
                                                -
                                                • Tag key: Enter a tag key name in the text box.
                                                  NOTE:

                                                  A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                  +
                                                  • Tag key: Enter a tag key name in the text box.
                                                    NOTE:

                                                    A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                  -
                                                  • Tag value: Enter a tag value in the text box.
                                                    NOTE:

                                                    A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                                    +
                                                    • Tag value: Enter a tag value in the text box.
                                                      NOTE:

                                                      A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                                                -

                                              4. Click OK to enter the editing page.
                                              5. Select a queue. Flink Jar jobs can run only on dedicated queues.

                                                • A Flink Jar job can run only on a pre-created dedicated queue.
                                                • If no dedicated queue is available in the Queue drop-down list, create a dedicated queue and bind it to the current user.
                                                -
                                                -

                                              6. Configuring Flink Jar Job parameters

                                                +

                                              7. Click OK to enter the editing page.
                                              8. Select a queue.
                                              9. Configuring Flink Jar Job parameters

                                                -
                                                Table 2 Parameter description

                                                Name

                                                +
                                                - - - - - - - - - - - - - - - - - + + + @@ -116,78 +129,86 @@
                                                Table 2 Parameters

                                                Parameter

                                                Description

                                                +

                                                Description

                                                Queue

                                                +

                                                Queue

                                                A shared queue is selected by default. You can select a custom queue as needed.

                                                +

                                                Select a queue where you want to run your job.

                                                Application

                                                +

                                                Application

                                                User-defined package. Before selecting a JAR file to be inserted, upload the corresponding JAR file to the OBS bucket and choose Data Management > Package Management to create a package. For details, see Creating a Package.

                                                -

                                                For details about built-in dependency packages, see Built-in Dependencies.

                                                +

                                                Select a Jar job package.

                                                +

                                                There are the following ways to manage JAR files:

                                                +
                                                • Upload packages to OBS: Upload Jar packages to an OBS bucket in advance and select the corresponding OBS path.
                                                • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.
                                                +

                                                For Flink 1.15 or later, you can only select packages from OBS, instead of DLI.

                                                Main Class

                                                +

                                                Main Class

                                                The name of the JAR package to be loaded, for example, KafkaMessageStreaming.

                                                -
                                                • Default: Specified based on the Manifest file in the JAR package.
                                                • Manually assign: You must enter the class name and confirm the class arguments (separate arguments with spaces).
                                                +

                                                The name of the JAR package to be loaded, for example, KafkaMessageStreaming.

                                                +
                                                • Default: Specified based on the Manifest file in the JAR package.
                                                • Manually assign: You must enter the class name and confirm the class arguments (separated by spaces).
                                                NOTE:

                                                When a class belongs to a package, the main class path must contain the complete package path, for example, packagePath.KafkaMessageStreaming.

                                                Class Arguments

                                                +

                                                Class Arguments

                                                List of arguments of a specified class. The arguments are separated by spaces.

                                                +

                                                List of arguments of a specified class. The arguments are separated by spaces.

                                                Flink parameters support replacement of non-sensitive global variables. For example, if you add the global variable windowsize in Global Configuration > Global Variables, you can add the -windowsSize {{windowsize}} parameter for the Flink Jar job.

                                                JAR Package Dependencies

                                                +

                                                JAR Package Dependencies

                                                Select a user-defined package dependency. The dependent program packages are stored in the classpath directory of the cluster.

                                                -

                                                Before selecting a JAR file to be inserted, upload the corresponding JAR file to the OBS bucket and choose Data Management > Package Management to create a package. Select JAR as the package type. For details, see Creating a Package.

                                                -

                                                For details about built-in dependency packages, see Built-in Dependencies.

                                                +

                                                Select a user-defined package dependency. The dependent program packages are stored in the classpath directory of the cluster.

                                                +

                                                There are the following ways to manage JAR files:

                                                +
                                                • Upload packages to OBS: Upload Jar packages to an OBS bucket in advance and select the corresponding OBS path.
                                                • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.
                                                +

                                                For Flink 1.15 or later, you can only select packages from OBS, instead of DLI.

                                                +

                                                When creating a JAR file for a Flink Jar job, you do not need to upload existing built-in dependency packages to avoid package information conflicts.

                                                +

                                                For details about built-in dependency packages, see DLI Built-in Dependencies.

                                                Other Dependencies

                                                +

                                                Other Dependencies

                                                User-defined dependency files. Other dependency files need to be referenced in the code.

                                                -

                                                Before selecting a dependency file, upload the file to the OBS bucket and choose Data Management > Package Management to create a package. The package type is not limited. For details, see Creating a Package.

                                                +

                                                User-defined dependency files. Other dependency files need to be referenced in the code.

                                                +

                                                There are the following ways to manage dependency files:

                                                +
                                                • Upload packages to OBS: Upload dependency files to an OBS bucket in advance and select the corresponding OBS path.
                                                • Upload packages to DLI: Upload dependency files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.
                                                +

                                                For Flink 1.15 or later, you can only select packages from OBS, instead of DLI.

                                                You can add the following command to the application to access the corresponding dependency file. In the command, fileName indicates the name of the file to be accessed, and ClassName indicates the name of the class that needs to access the file.

                                                ClassName.class.getClassLoader().getResource("userData/fileName")

                                                Flink Version

                                                +

                                                Flink Version

                                                Before selecting a Flink version, you need to select the queue to which the Flink version belongs.

                                                +

                                                Set Queue before setting this parameter.

                                                +
                                                NOTE:

                                                You are advised not to use Flink of different versions for a long time.

                                                +
                                                • Doing so can lead to code incompatibility, which can negatively impact job execution efficiency.
                                                • Doing so may result in job execution failures due to conflicts in dependencies. Jobs rely on specific versions of libraries or components.
                                                +

                                                Runtime Configuration

                                                +

                                                Agency

                                                User-defined optimization parameters. The parameter format is key=value.

                                                +

                                                If you choose Flink 1.15 to execute your job, you can create a custom agency to allow DLI to access other services.

                                                +

                                                Runtime Configuration

                                                +

                                                User-defined optimization parameters. The parameter format is key=value.

                                                Flink optimization parameters support replacement non-sensitive global variable. For example, if you create global variable phase in Global Configuration > Global Variables, optimization parameter table.optimizer.agg-phase.strategy={{phase}} can be added to the Flink Jar job.
                                                -

                                              10. Configure job parameters.

                                                -

                                                Table 3 Parameter description

                                                Name

                                                +

                                              11. Set compute resource specification parameters.

                                                +

                                                - - - - - - - - - - - - - + + + + + + - - + + + - -
                                                Table 3 Parameters

                                                Parameter

                                                Description

                                                +

                                                Description

                                                CUs

                                                +

                                                CUs

                                                One CU has one vCPU and 4-GB memory. The number of CUs ranges from 2 to 10,000.

                                                -

                                                Job Manager CUs

                                                -

                                                Set the number of CUs on a management unit. The value ranges from 1 to 4. The default value is 1.

                                                -

                                                Parallelism

                                                -

                                                Maximum number of parallel operators in a job. The value ranges from 1 to 10,000.

                                                -
                                                NOTE:
                                                • The value must be less than or equal to four times the number of compute units (CUs minus the number of job manager CUs).
                                                • You are advised to set this parameter to a value greater than that configured in the code. Otherwise, job submission may fail.
                                                +

                                                One CU consists of one vCPU and 4 GB of memory. The number of CUs ranges from 2 to 10000.

                                                +
                                                NOTE:

                                                When Task Manager Config is selected, elastic resource pool queue management is optimized by automatically adjusting CUs to match Actual CUs after setting Slot(s) per TM.

                                                +

                                                CUs = Actual number of CUs = max[Job Manager CPUs + Task Manager CPU, (Job Manager Memory + Task Manager Memory/4)]

                                                +
                                                • Job Manager CPUs + Task Manager CPUs = Actual TMs x CU(s) per TM + Job Manager CUs.
                                                • Job Manager Memory + Task Manager Memory = Actual TMs x Memory per TM + Job Manager Memory
                                                • If Slot(s) per TM is set, then: Actual TMs = Parallelism/Slot(s) per TM.
                                                • If Slot(s) per TM is not set, then: Actual TMs = (CUs – Job Manager CUs)/CU(s) per TM.
                                                • If Memory per TM and Job Manager Memory in the optimization parameters are not set, then: Memory per TM = CU(s) per TM x 4. Job Manager Memory = Job Manager CUs x 4.
                                                • The parallelism degree of Spark resources is jointly determined by the number of Executors and the number of Executor CPU cores.

                                                Task Manager Configuration

                                                +

                                                Job Manager CUs

                                                Whether to set Task Manager resource parameters

                                                -

                                                If this option is selected, you need to set the following parameters:

                                                -
                                                • CU(s) per TM: Number of resources occupied by each Task Manager.
                                                • Slot(s) per TM: Number of slots contained in each Task Manager.
                                                +

                                                Number of CUs for the job management unit.

                                                Save Job Log

                                                +

                                                Parallelism

                                                Whether to save the job running logs to the OBS bucket.

                                                +

                                                Number of tasks concurrently executed by each operator in a job.

                                                +
                                                NOTE:
                                                • The value must be less than or equal to four times the number of compute units (CUs minus the number of job manager CUs).
                                                • Set this parameter to a value greater than that configured in the code to avoid job submission failures.
                                                +
                                                +

                                                Task Manager Config

                                                +

                                                Whether Task Manager resource parameters are set

                                                +
                                                • If this option is selected, you need to set the following parameters:
                                                  • CU(s) per TM: Number of resources occupied by each Task Manager.
                                                  • Slot(s) per TM: Number of slots contained in each Task Manager.
                                                  +
                                                +
                                                • If not selected, the system automatically uses the default values.
                                                  • CU(s) per TM: The default value is 1.
                                                  • Slot(s) per TM: The default value is (Parallelism x CU(s) per TM)/(CUs – Job Manager CUs).
                                                  +
                                                +

                                                Save Job Log

                                                +

                                                Whether to save the job running logs to the OBS bucket.

                                                +

                                                If you select this parameter, you also need to set OBS Bucket.

                                                CAUTION:

                                                You are advised to select this parameter. Otherwise, no run log is generated after the job is executed. If the job is abnormal, the run log cannot be obtained for fault locating.

                                                -

                                                If this option is selected, you need to set the following parameters:

                                                -

                                                OBS Bucket: Select an OBS bucket to store job logs. If the selected OBS bucket is not authorized, click Authorize.

                                                Alarm Generation upon Job Exception

                                                +

                                                OBS Bucket

                                                Whether to report job exceptions, for example, abnormal job running or exceptions due to an insufficient balance, to users via SMS or email

                                                +

                                                This parameter is mandatory when you select Save Job Log.

                                                +

                                                Select an OBS bucket to store job logs. If the OBS bucket you select is not authorized, click Authorize.

                                                +

                                                Alarm on Job Exception

                                                +

                                                Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                                                If this option is selected, you need to set the following parameters:

                                                SMN Topic

                                                -

                                                Select a user-defined SMN topic. For details about how to customize SMN topics, see Creating a Topic in the Simple Message Notification User Guide.

                                                +

                                                Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                                                Auto Restart upon Exception

                                                +

                                                Auto Restart upon Exception

                                                Whether to enable automatic restart. If this function is enabled, any job that has become abnormal will be automatically restarted.

                                                +

                                                Whether automatic restart is enabled. If enabled, jobs will be automatically restarted and restored when exceptions occur.

                                                If this option is selected, you need to set the following parameters:

                                                -
                                                • Max. Retry Attempts: maximum number of retry times upon an exception. The unit is Times/hour.
                                                  • Unlimited: The number of retries is unlimited.
                                                  • Limited: The number of retries is user-defined.
                                                  -
                                                • Restore Job from Checkpoint: Restore the job from the saved checkpoint.

                                                  If you select this parameter, you also need to set Checkpoint Path.

                                                  -

                                                  Checkpoint Path: Select the checkpoint saving path. The checkpoint path must be the same as that you set in the application package. Note that the checkpoint path for each job must be unique. Otherwise, the checkpoint cannot be obtained.

                                                  +
                                                  • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                                                    • Unlimited: The number of retries is unlimited.
                                                    • Limited: The number of retries is user-defined.
                                                    +
                                                  • Restore Job from Checkpoint: Restore the job from the saved checkpoint.

                                                    If you select this parameter, you need to set Checkpoint Path.

                                                    +

                                                    Checkpoint Path: Select the checkpoint save path. This path must match that configured in the application package. Each job must have a unique checkpoint path, or, you will not be able to obtain the checkpoint.
                                                -

                                              12. Click Save on the upper right of the page.
                                              13. Click Start on the upper right side of the page. On the displayed Start Flink Job page, confirm the job specifications, and click Start Now to start the job.

                                                After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column. After a job is successfully submitted, the job status will change from Submitting to Running. After the execution is complete, the message Completed is displayed.

                                                -

                                                If the job status is Submission failed or Running exception, the job submission failed or the job did not execute successfully. In this case, you can move the cursor over the status icon in the Status column of the job list to view the error details. You can click to copy these details. After handling the fault based on the provided information, resubmit the job.

                                                -

                                                Other available buttons are as follows:

                                                -

                                                Save As: Save the created job as a new job.

                                                -
                                                +

                                                +

                                              14. Click Save on the upper right of the page.
                                              15. Click Start in the upper right corner. On the displayed Start Flink Job page, confirm the job specifications, and click Start Now to start the job. After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column.

                                                • Once a job is successfully submitted, its status changes from Submitting to Running. After the execution is complete, the status changes to Completed.
                                                • If the job status is Submission failed or Running exception, the job fails to submit or run. In this case, you can hover over the status icon in the Status column of the job list to view the error details. You can click to copy these details. Rectify the fault based on the error information and resubmit the job.

                                                16. diff --git a/docs/dli/umn/dli_01_0461.html b/docs/dli/umn/dli_01_0461.html index 1381b13ed..1352ac049 100644 --- a/docs/dli/umn/dli_01_0461.html +++ b/docs/dli/umn/dli_01_0461.html @@ -1,14 +1,12 @@ -

                                                Performing Operations on a Flink Job

                                                -

                                                After a job is created, you can perform operations on the job as required.

                                                - -

                                                Editing a Job

                                                You can edit a created job, for example, by modifying the SQL statement, job name, job description, or job configurations.

                                                -
                                                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                2. In the row where the job you want to edit locates, click Edit in the Operation column to switch to the editing page.
                                                3. Edit the job as required.

                                                  For details about how to edit a Flink SQL job, see Step 5 to Step 7 in Creating a Flink SQL Job.

                                                  -

                                                  For details about how to edit a user-defined Flink job, see Step 5 to Step 7 in Creating a Flink Jar Job.

                                                  +

                                                  Common Operations of Flink Jobs

                                                  +

                                                  After creating a job, you can manage it by performing various operations such as editing its basic information, starting or stopping it, and importing or exporting it.

                                                  +

                                                  Editing a Job

                                                  You can edit a created job, for example, by modifying the SQL statement, job name, job description, or job configurations.

                                                  +
                                                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                  2. In the row where the job you want to edit locates, click Edit in the Operation column to switch to the editing page.
                                                  3. Edit the job as required.

                                                    For details, see Creating a Flink OpenSource SQL Job and Creating a Flink Jar Job.

                                                  -

                                                  Starting a Job

                                                  You can start a saved or stopped job.

                                                  +

                                                  Starting a Job

                                                  You can start a saved or stopped job.

                                                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                  2. Use either of the following methods to start jobs:

                                                    • Starting a single job

                                                      Select a job and click Start in the Operation column.

                                                      Alternatively, you can select the row where the job you want to start locates and click Start in the upper left of the job list.

                                                    • Batch starting jobs

                                                      Select the rows where the jobs you want to start locate and click Start in the upper left of the job list.

                                                      @@ -17,9 +15,9 @@

                                                  1. On the Start Flink Jobs page, confirm the job information. If they are correct, click Start Now.

                                                    After a job is started, you can view the job execution result in the Status column.

                                                  -

                                                  Stopping a Job

                                                  You can stop a job in the Running or Submitting state.

                                                  +

                                                  Stopping a Job

                                                  You can stop a job in the Running or Submitting state.

                                                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                  2. Stop a job using either of the following methods:

                                                    • Stopping a job

                                                      Locate the row that contains the job to be stopped, click More in the Operation column, and select Stop.

                                                      -

                                                      Alternatively, you can select the row where the job you want to stop locates and click Stop in the upper left of the job list.

                                                      +

                                                      Alternatively, you can select the row where the job you want to stop locates and click Stop in the upper left of the job list.

                                                    • Batch stopping jobs

                                                      Locate the rows containing the jobs you want to stop and click Stop in the upper left of the job list.

                                                  3. In the displayed Stop Job dialog box, click OK to stop the job.

                                                    • Before stopping a job, you can trigger a savepoint to save the job status information. When you start the job again, you can choose whether to restore the job from the savepoint.
                                                    • If you select Trigger savepoint, a savepoint is created. If Trigger savepoint is not selected, no savepoint is created. By default, the savepoint function is disabled.
                                                    • The lifecycle of a savepoint starts when the savepoint is triggered and stops the job, and ends when the job is restarted. The savepoint is automatically deleted after the job is restarted.
                                                    @@ -29,70 +27,68 @@

                                                  -

                                                  Deleting a Job

                                                  If you do not need to use a job, perform the following operations to delete it. A deleted job cannot be restored. Therefore, exercise caution when deleting a job.

                                                  +

                                                  Deleting a Job

                                                  If you do not need to use a job, perform the following operations to delete it. A deleted job cannot be restored. Therefore, exercise caution when deleting a job.

                                                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                  1. Perform either of the following methods to delete jobs:

                                                    • Deleting a single job

                                                      Locate the row containing the job you want to delete and click More > Delete in the Operation column.

                                                      Alternatively, you can select the row containing the job you want to delete and click Delete in the upper left of the job list.

                                                      -

                                                    • Deleting jobs in batches

                                                      Select the rows containing the jobs you want to delete and click Delete in the upper left of the job list.

                                                  2. Click Yes.
                                                  -

                                                  Exporting a Job

                                                  You can export the created Flink jobs to an OBS bucket.

                                                  +

                                                  Exporting a Job

                                                  You can export the created Flink jobs to an OBS bucket.

                                                  This mode is applicable to the scenario where a large number of jobs need to be created when you switch to another region, project, or user. In this case, you do not need to create a job. You only need to export the original job, log in to the system in a new region or project, or use a new user to import the job.

                                                  -

                                                  When switching to another project or user, you need to grant permissions to the new project or user. For details, see Managing Flink Job Permissions.

                                                  +

                                                  When switching to another project or user, you need to grant permissions to the new project or user. For details, see Configuring Flink Job Permissions.

                                                  -
                                                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                  1. Click Export Job in the upper right corner. The Export Job dialog box is displayed.

                                                    -

                                                  2. Select the OBS bucket where the job is stored. Click Next.
                                                  3. Select job information you want to export.

                                                    By default, configurations of all jobs are exported. You can enable the Custom Export function to export configurations of the desired jobs.

                                                    +
                                                    1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                    1. Click Export Job in the upper right corner. The Export Job dialog box is displayed.
                                                    2. Select the OBS bucket where the job is stored. Click Next.
                                                    3. Select job information you want to export.

                                                      By default, configurations of all jobs are exported. You can enable the Custom Export function to export configurations of the desired jobs.

                                                    4. Click Confirm to export the job.
                                                  -

                                                  Importing a Job

                                                  You can import the Flink job configuration file stored in the OBS bucket to the Flink Jobs page of DLI.

                                                  +

                                                  Importing a Job

                                                  You can import the Flink job configuration file stored in the OBS bucket to the Flink Jobs page of DLI.

                                                  This mode is applicable to the scenario where a large number of jobs need to be created when you switch to another region, project, or user. In this case, you do not need to create a job. You only need to export the original job, log in to the system in a new region or project, or use a new user to import the job.

                                                  -

                                                  If you need to import a self-created job, you are advised to use the job creation function. For details, see Creating a Flink SQL Job and Creating a Flink Jar Job.

                                                  -
                                                  • When switching to another project or user, you need to grant permissions to the new project or user. For details, see Managing Flink Job Permissions.
                                                  • Only jobs whose data format is the same as that of Flink jobs exported from DLI can be imported.
                                                  +

                                                  To import a self-created job, use the job creation function.

                                                  +

                                                  For details, see Creating a Flink OpenSource SQL Job and Creating a Flink Jar Job.

                                                  +
                                                  • When switching to another project or user, you need to grant permissions to the new project or user. For details, see Configuring Flink Job Permissions.
                                                  • Only jobs whose data format is the same as that of Flink jobs exported from DLI can be imported.
                                                  -
                                                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                  1. Click Import Job in the upper right corner. The Import Job dialog box is displayed.

                                                    -

                                                  2. Select the complete OBS path of the job configuration file to be imported. Click Next.
                                                  3. Configure the same-name job policy and click next. Click Next.

                                                    • Select Overwrite job of the same name. If the name of the job to be imported already exists, the existing job configuration will be overwritten and the job status switches to Draft.
                                                    • If Overwrite job of the same name is not selected and the name of the job to be imported already exists, the job will not be imported.
                                                    +
                                                    1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                    1. Click Import Job in the upper right corner. The Import Job dialog box is displayed.
                                                    2. Select the complete OBS path of the job configuration file to be imported. Click Next.
                                                    3. Configure the same-name job policy and click next. Click Next.

                                                      • Select Overwrite job of the same name. If the name of the job to be imported already exists, the existing job configuration will be overwritten and the job status switches to Draft.
                                                      • If Overwrite job of the same name is not selected and the name of the job to be imported already exists, the job will not be imported.

                                                    4. Ensure that Config File and Overwrite Same-Name Job are correctly configured. Click Confirm to import the job.
                                                  -

                                                  Modifying Name and Description

                                                  You can change the job name and description as required.

                                                  +

                                                  Modifying the Name and Description of a Flink Job

                                                  You can change the job name and description as required.

                                                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                  2. In the Operation column of the job whose name and description need to be modified, choose More > Modify Name and Description. The Modify Name and Description dialog box is displayed. Change the name or modify the description of a job.
                                                  3. Click OK.
                                                  -

                                                  -

                                                  Triggering a Savepoint

                                                  When you need to stop a job, you can create a savepoint to save the job status information. In this case, when you restart the job, you can choose to restore the job from the latest savepoint.

                                                  +

                                                  Triggering a Savepoint

                                                  Before stopping a job, you can trigger a savepoint to save the status information of your job. When you restart the job, you can choose whether to quickly recover it from the most recent savepoint.

                                                  +
                                                  1. In the navigation pane of the DLI console, choose Job Management > Flink Jobs.
                                                  2. Locate the job you want to stop, click More in the Operation column, and select Trigger Savepoint. In the displayed dialog box, select a save path.
                                                  3. Click OK.
                                                  • You can click Trigger Savepoint for jobs in the Running status to save the job status.
                                                  • The lifecycle of a savepoint starts when the savepoint is triggered and stops the job, and ends when the job is restarted. The savepoint is automatically deleted after the job is restarted.
                                                  -

                                                  Importing to a Savepoint

                                                  You can import a savepoint to restore the job status. For details about the savepoint, see Checkpointing at the official website of Flink.

                                                  -

                                                  You need to select the OBS path of the save point.

                                                  +

                                                  Importing to a Savepoint

                                                  Flink jobs can be restored based on imported savepoints.

                                                  +
                                                  1. In the navigation pane of the DLI console, choose Job Management > Flink Jobs.
                                                  2. Locate the job you want to stop, click More in the Operation column, and select Import Savepoint. In the displayed dialog box, select a save path.
                                                  3. Click OK.
                                                  -

                                                  Runtime Configuration

                                                  You can select Runtime Configuration to configure job exception alarms and restart options.

                                                  -

                                                  Flink SQL jobs and Flink Jar jobs are supported.

                                                  +

                                                  Runtime Configuration

                                                  You can configure job exception alarms and restart options by selecting Runtime Configuration.

                                                  +

                                                  This configuration is only available for Flink OpenSource SQL jobs and Flink Jar jobs.

                                                  -
                                                  1. In the Operation column of the Flink job, choose More > Runtime Configuration.
                                                  2. In the Runtime Configuration dialog box, set the following parameters: -
                                                    Table 1 Running parameters

                                                    Parameter

                                                    +
                                                    1. Locate the desired Flink job, click More in the Operation column, and select Runtime Configuration.
                                                    2. In the Runtime Configuration dialog box, set the following parameters: +
                                                      - - - - - - -
                                                      Table 1 Running parameters

                                                      Parameter

                                                      Description

                                                      +

                                                      Description

                                                      Name

                                                      +

                                                      Name

                                                      Job name.

                                                      +

                                                      Job name.

                                                      Alarm Generation upon Job Exception

                                                      +

                                                      Alarm Generation upon Job Exception

                                                      Whether to report job exceptions, for example, abnormal job running or exceptions due to an insufficient balance, to users via SMS or email.

                                                      +

                                                      Whether to report job exceptions, for example, abnormal job running or exceptions due to an insufficient balance, to users via SMS or email.

                                                      If this option is selected, you need to set the following parameters:

                                                      SMN Topic

                                                      -

                                                      Select a user-defined SMN topic. For details about how to customize SMN topics, see Creating a Topic in the Simple Message Notification User Guide.

                                                      +

                                                      Select a user-defined SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                                                      Auto Restart upon Exception

                                                      +

                                                      Auto Restart upon Exception

                                                      Whether to enable automatic restart. If this function is enabled, any job that has become abnormal will be automatically restarted.

                                                      +

                                                      Whether to enable automatic restart. If this function is enabled, any job that has become abnormal will be automatically restarted.

                                                      If this option is selected, you need to set the following parameters:

                                                      -
                                                      • Max. Retry Attempts: maximum number of retry times upon an exception. The unit is times/hour.
                                                        • Unlimited: The number of retries is unlimited.
                                                        • Limited: The number of retries is user-defined.
                                                        +
                                                        • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                                                          • Unlimited: The number of retries is unlimited.
                                                          • Limited: The number of retries is user-defined.
                                                        • Restore Job from Checkpoint: Restore the job from the saved checkpoint.
                                                          NOTE:

                                                          This parameter cannot be configured for Flink SQL jobs or Flink OpenSource SQL jobs.

                                                          If this parameter is selected, you need to set Checkpoint Path for Flink Jar jobs.

                                                          @@ -108,7 +104,7 @@
                                                          -
                                                          Parent topic: Flink Job Management
                                                          +
                                                          Parent topic: Managing Flink Jobs
                                                          diff --git a/docs/dli/umn/dli_01_0462.html b/docs/dli/umn/dli_01_0462.html index 989e5407e..3e3423a9a 100644 --- a/docs/dli/umn/dli_01_0462.html +++ b/docs/dli/umn/dli_01_0462.html @@ -1,155 +1,270 @@ -

                                                          Flink Job Details

                                                          -

                                                          After creating a job, you can view the job details to learn about the following information:

                                                          - -

                                                          Viewing Job Details

                                                          This section describes how to view job details. After you create and save a job, you can click the job name to view job details, including SQL statements and parameter settings. For a Jar job, you can only view its parameter settings.

                                                          -
                                                          1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                          2. Click the name of the job to be viewed. The Job Detail tab is displayed.

                                                            In the Job Details tab, you can view SQL statements, configured parameters.

                                                            -
                                                            The following uses a Flink SQL job as an example. -
                                                            Table 1 Description

                                                            Parameter

                                                            +

                                                            Viewing Flink Job Details

                                                            +

                                                            After creating a Flink job, you can check the basic information, job details, task list, and execution plan of the job on the DLI console.

                                                            +

                                                            This section describes how to check information about a Flink job.

                                                            + +
                                                            - + - - + - - + - - + - - + - - + - +
                                                            Table 1 Viewing Flink job information

                                                            Type

                                                            Description

                                                            +

                                                            Description

                                                            +

                                                            Instruction

                                                            Type

                                                            +

                                                            Basic information

                                                            Job type, for example, Flink SQL

                                                            +

                                                            Includes the job ID, job type, job execution status, and more.

                                                            +

                                                            Viewing Basic Information

                                                            Name

                                                            +

                                                            Job details

                                                            Flink job name

                                                            +

                                                            Includes SQL statements and the parameter settings for Flink Jar jobs.

                                                            +

                                                            Viewing Details

                                                            Description

                                                            +

                                                            Job monitoring

                                                            Description of a Flink job

                                                            +

                                                            You can use Cloud Eye to check job data input and output details.

                                                            +

                                                            Viewing Monitoring Information

                                                            Status

                                                            +

                                                            Task list

                                                            Running status of a job

                                                            +

                                                            You can view details about each task running on a job, including the task start time, number of received and transmitted bytes, and running duration.

                                                            +

                                                            Viewing the Task List

                                                            Running Mode

                                                            +

                                                            Execution plan

                                                            If your job runs on a shared queue, this parameter is Shared.

                                                            -

                                                            If your job runs on a custom queue with dedicated resources, this parameter is Exclusive.

                                                            +

                                                            You can understand the operator flow direction of a running job.

                                                            +

                                                            Viewing the Execution Plan

                                                            Flink Version

                                                            +
                                                            +
                                                            +

                                                            Viewing Basic Information

                                                            In the navigation pane of the DLI console, choose Job Management > Flink Jobs. The Flink Jobs page displays all Flink jobs. You can check basic information about any Flink jobs in the list.

                                                            + +
                                                            + + + + - - - - - - - - - - - - - - - - - - - - +
                                                            Table 2 Basic information about a Flink job

                                                            Parameter

                                                            +

                                                            Description

                                                            +

                                                            ID

                                                            Version of Flink selected for the job.

                                                            +

                                                            ID of a submitted Flink job, which is generated by the system by default.

                                                            Runtime Configuration

                                                            +

                                                            Name

                                                            Displayed when a user-defined parameter is added to a job

                                                            +

                                                            Name of the submitted Flink job.

                                                            CUs

                                                            +

                                                            Type

                                                            Number of CUs configured for a job

                                                            +

                                                            Type of the submitted Flink job, which includes:

                                                            +
                                                            • Flink SQL
                                                            • Flink Jar

                                                            Job Manager CUs

                                                            +

                                                            Status

                                                            Number of job manager CUs configured for a job.

                                                            +

                                                            Job status, which is subject to the console.

                                                            Parallelism

                                                            +

                                                            Description

                                                            Number of jobs that can be concurrently executed by a Flink job

                                                            +

                                                            Description of the submitted Flink job.

                                                            CU(s) per TM

                                                            +

                                                            Username

                                                            Number of CUs occupied by each Task Manager configured for a job

                                                            +

                                                            Name of the user who submits the job.

                                                            Slot(s) per TM

                                                            +

                                                            Created

                                                            Number of Task Manager slots configured for a job

                                                            +

                                                            Time when the job was created.

                                                            OBS Bucket

                                                            +

                                                            Started

                                                            OBS bucket name. After Enable Checkpointing and Save Job Log are enabled, checkpoints and job logs are saved in this bucket.

                                                            +

                                                            Time when the Flink job started to run.

                                                            Save Job Log

                                                            +

                                                            Duration

                                                            Whether the job running logs are saved to OBS

                                                            +

                                                            Time consumed by job running.

                                                            Alarm Generation upon Job Exception

                                                            +

                                                            Operation

                                                            Whether job exceptions are reported

                                                            +
                                                            • Edit: Edit a created job.
                                                            • Start: Start and run a job.
                                                            • More
                                                              • FlinkUI: Selecting this will display the Flink job execution page.
                                                                NOTE:

                                                                If you select FlinkUI immediately after submitting a job to a new queue, an empty projectID will be cached and the FlinkUI page cannot be displayed as it takes about 10 minutes to create a cluster in the background.

                                                                +

                                                                You are advised to use a dedicated queue to ensure immediate availability of clusters for your jobs. Alternatively, select FlinkUI when the job is in the Running state.

                                                                +
                                                                +
                                                              • Stop: Stop a Flink job. If it is grayed out, jobs in the current state cannot be stopped.
                                                              • Delete: Delete a job.
                                                                NOTE:

                                                                Deleted jobs cannot be restored.

                                                                +
                                                                +
                                                              • Modify Name and Description: You can modify the name and description of a job.
                                                              • Import Savepoint: Import the data exported from the original Cloud Stream Service (CS) job.
                                                              • Trigger Savepoint: You can select this operation for jobs in the Running state to save the job status.
                                                              • Permissions: You can view the user permissions corresponding to the job and grant permissions to other users.
                                                              • Runtime Configuration: You can enable Alarm Generation on Job Exception and Auto Restart on Exception.
                                                              +

                                                            SMN Topic

                                                            +
                                                            +
                                                            +
                                                            +

                                                            Viewing Details

                                                            This section describes how to view job details. After you create and save a job, you can click the job name to view job details, including SQL statements and parameter settings. For a Jar job, you can only view its parameter settings.

                                                            +
                                                            1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                            2. Click the name of the job to be viewed. The Job Detail tab is displayed.

                                                              In the Job Details tab, you can view SQL statements, configured parameters.

                                                              +
                                                              The following uses a Flink SQL job as an example. +
                                                              + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - @@ -158,74 +273,74 @@

                                                              -

                                                              Checking Job Monitoring Information

                                                              You can use Cloud Eye to view details about job data input and output.

                                                              +

                                                              Viewing Monitoring Information

                                                              You can use Cloud Eye to view details about job data input and output.

                                                              1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                              2. Click the name of the job you want. The job details are displayed.

                                                                Click Job Monitoring in the upper right corner of the page to switch to the Cloud Eye console.

                                                                The following table describes monitoring metrics related to Flink jobs.

                                                                -
                                                              Table 3 Parameter descriptions

                                                              Parameter

                                                              +

                                                              Description

                                                              +

                                                              Type

                                                              Name of the SMN topic. This parameter is displayed when Alarm Generation upon Job Exception is enabled.

                                                              +

                                                              Job type, for example, Flink SQL

                                                              Auto Restart upon Exception

                                                              +

                                                              Name

                                                              Whether automatic restart is enabled.

                                                              +

                                                              Flink job name

                                                              Max. Retry Attempts

                                                              +

                                                              Description

                                                              Maximum number of retry times upon an exception. Unlimited means the number is not limited.

                                                              +

                                                              Description of a Flink job

                                                              Savepoint

                                                              +

                                                              Status

                                                              OBS path of the savepoint

                                                              +

                                                              Running status of a job

                                                              Enable Checkpointing

                                                              +

                                                              Running Mode

                                                              Whether checkpointing is enabled

                                                              +

                                                              The dedicated resource mode is used by default.

                                                              Checkpoint Interval

                                                              +

                                                              Flink Version

                                                              Interval between storing intermediate job running results to OBS. The unit is second.

                                                              +

                                                              Version of Flink selected for the job.

                                                              Checkpoint Mode

                                                              +

                                                              Runtime Configuration

                                                              Checkpoint mode. Available values are as follows:

                                                              +

                                                              Displayed when a user-defined parameter is added to a job

                                                              +

                                                              CUs

                                                              +

                                                              Number of CUs configured for a job

                                                              +

                                                              Job Manager CUs

                                                              +

                                                              Number of job manager CUs configured for a job.

                                                              +

                                                              Parallelism

                                                              +

                                                              Number of jobs that can be concurrently executed by a Flink job

                                                              +

                                                              CU(s) per TM

                                                              +

                                                              Number of CUs occupied by each Task Manager configured for a job

                                                              +

                                                              Slot(s) per TM

                                                              +

                                                              Number of Task Manager slots configured for a job

                                                              +

                                                              OBS Bucket

                                                              +

                                                              OBS bucket name. After Enable Checkpointing and Save Job Log are enabled, checkpoints and job logs are saved in this bucket.

                                                              +

                                                              Save Job Log

                                                              +

                                                              Whether the job running logs are saved to OBS

                                                              +

                                                              Alarm on Job Exception

                                                              +

                                                              Whether job exceptions are reported

                                                              +

                                                              SMN Topic

                                                              +

                                                              Name of the SMN topic. This parameter is displayed when Alarm Generation upon Job Exception is enabled.

                                                              +

                                                              Auto Restart upon Exception

                                                              +

                                                              Whether automatic restart is enabled.

                                                              +

                                                              Max. Retry Attempts

                                                              +

                                                              Maximum number of retry times upon an exception. Unlimited means the number is not limited.

                                                              +

                                                              Savepoint

                                                              +

                                                              OBS path of the savepoint

                                                              +

                                                              Enable Checkpointing

                                                              +

                                                              Whether checkpointing is enabled

                                                              +

                                                              Checkpoint Interval

                                                              +

                                                              Interval between storing intermediate job running results to OBS. The unit is second.

                                                              +

                                                              Checkpoint Mode

                                                              +

                                                              Checkpoint mode. Available values are as follows:

                                                              • At least once: Events are processed at least once.
                                                              • Exactly once: Events are processed only once.

                                                              Idle State Retention Time

                                                              +

                                                              Idle State Retention Time

                                                              Defines for how long the state of a key is retained without being updated before it is removed in GroupBy or Window.

                                                              +

                                                              Clears intermediate states of operators such as GroupBy, RegularJoin, Rank, and Depulicate that have not been updated after the maximum retention time. The default value is 1 hour.

                                                              Dirty Data Policy

                                                              +

                                                              Dirty Data Policy

                                                              Policy for processing dirty data. The value is displayed only when there is a dirty data policy. Available values are as follows:

                                                              +

                                                              Policy for processing dirty data. The value is displayed only when there is a dirty data policy. Available values are as follows:

                                                              Ignore

                                                              Trigger a job exception

                                                              Save

                                                              Dirty Data Dump Address

                                                              +

                                                              Dirty Data Dump Address

                                                              OBS path for storing dirty data when Dirty Data Policy is set to Save.

                                                              +

                                                              OBS path for storing dirty data when Dirty Data Policy is set to Save.

                                                              Created

                                                              +

                                                              Created

                                                              Time when a job is created

                                                              +

                                                              Time when a job is created

                                                              Updated

                                                              +

                                                              Updated

                                                              Time when a job was last updated

                                                              +

                                                              Time when a job was last updated
                                                              Table 2 Monitoring metrics related to Flink jobs

                                                              Name

                                                              +
                                                              - - - - - - - - - - - - - - - - - - - - - - - - -
                                                              Table 4 Monitoring metrics related to Flink jobs

                                                              Name

                                                              Description

                                                              +

                                                              Description

                                                              Flink Job Data Read Rate

                                                              +

                                                              Flink Job Data Read Rate

                                                              Displays the data input rate of a Flink job for monitoring and debugging. Unit: record/s.

                                                              +

                                                              Displays the data input rate of a Flink job for monitoring and debugging. Unit: record/s.

                                                              Flink Job Data Write Rate

                                                              +

                                                              Flink Job Data Write Rate

                                                              Displays the data output rate of a Flink job for monitoring and debugging. Unit: record/s.

                                                              +

                                                              Displays the data output rate of a Flink job for monitoring and debugging. Unit: record/s.

                                                              Flink Job Total Data Read

                                                              +

                                                              Flink Job Total Data Read

                                                              Displays the total number of data inputs of a Flink job for monitoring and debugging. Unit: records

                                                              +

                                                              Displays the total number of data inputs of a Flink job for monitoring and debugging. Unit: records

                                                              Flink Job Total Data Write

                                                              +

                                                              Flink Job Total Data Write

                                                              Displays the total number of output data records of a Flink job for monitoring and debugging. Unit: records

                                                              +

                                                              Displays the total number of output data records of a Flink job for monitoring and debugging. Unit: records

                                                              Flink Job Byte Read Rate

                                                              +

                                                              Flink Job Byte Read Rate

                                                              Displays the number of input bytes per second of a Flink job. Unit: byte/s

                                                              +

                                                              Displays the number of input bytes per second of a Flink job. Unit: byte/s

                                                              Flink Job Byte Write Rate

                                                              +

                                                              Flink Job Byte Write Rate

                                                              Displays the number of output bytes per second of a Flink job. Unit: byte/s

                                                              +

                                                              Displays the number of output bytes per second of a Flink job. Unit: byte/s

                                                              Flink Job Total Read Byte

                                                              +

                                                              Flink Job Total Read Byte

                                                              Displays the total number of input bytes of a Flink job. Unit: byte

                                                              +

                                                              Displays the total number of input bytes of a Flink job. Unit: byte

                                                              Flink Job Total Write Byte

                                                              +

                                                              Flink Job Total Write Byte

                                                              Displays the total number of output bytes of a Flink job. Unit: byte

                                                              +

                                                              Displays the total number of output bytes of a Flink job. Unit: byte

                                                              Flink Job CPU Usage

                                                              +

                                                              Flink Job CPU Usage

                                                              Displays the CPU usage of Flink jobs. Unit: %

                                                              +

                                                              Displays the CPU usage of Flink jobs. Unit: %

                                                              Flink Job Memory Usage

                                                              +

                                                              Flink Job Memory Usage

                                                              Displays the memory usage of Flink jobs. Unit: %

                                                              +

                                                              Displays the memory usage of Flink jobs. Unit: %

                                                              Flink Job Max Operator Latency

                                                              +

                                                              Flink Job Max Operator Latency

                                                              Displays the maximum operator delay of a Flink job. The unit is ms.

                                                              +

                                                              Displays the maximum operator delay of a Flink job. The unit is ms.

                                                              Flink Job Maximum Operator Backpressure

                                                              +

                                                              Flink Job Maximum Operator Backpressure

                                                              Displays the maximum operator backpressure value of a Flink job. A larger value indicates severer backpressure.

                                                              +

                                                              Displays the maximum operator backpressure value of a Flink job. A larger value indicates severer backpressure.

                                                              0: OK

                                                              50: low

                                                              100: high

                                                              @@ -236,81 +351,81 @@

                                                              -

                                                              Viewing the Task List of a Job

                                                              You can view details about each task running on a job, including the task start time, number of received and transmitted bytes, and running duration.

                                                              +

                                                              Viewing the Task List

                                                              You can view details about each task running on a job, including the task start time, number of received and transmitted bytes, and running duration.

                                                              If the value is 0, no data is received from the data source.

                                                              1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                              2. Click the name of the job you want. The job details are displayed.
                                                              3. On Task List and view the node information about the task.

                                                                View the operator task list. The following table describes the task parameters. -
                                                                Table 3 Parameter description

                                                                Parameter

                                                                +
                                                                - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -319,24 +434,15 @@

                                                                -

                                                                Viewing the Job Execution Plan

                                                                You can view the execution plan to understand the operator stream information about the running job.

                                                                +

                                                                Viewing the Execution Plan

                                                                You can view the execution plan to understand the operator stream information about the running job.

                                                                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                                2. Click the name of the job you want. The job details are displayed.
                                                                3. Click the Execution Plan tab to view the operator flow direction.

                                                                  Click a node. The corresponding information is displayed on the right of the page.
                                                                  • Scroll the mouse wheel to zoom in or out.
                                                                  • The stream diagram displays the operator stream information about the running job in real time.

                                                                -

                                                                Viewing Job Submission Logs

                                                                You can view the submission logs to locate the fault.

                                                                -
                                                                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                                2. Click the name of the job you want. The job details are displayed.
                                                                3. In the Commit Logs tab, view the information about the job submission process.
                                                                -
                                                                -

                                                                Viewing Job Running Logs

                                                                You can view the run logs to locate the faults occurring during job running.

                                                                -
                                                                1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                                2. Click the name of the job you want. The job details are displayed.
                                                                3. On the Run Log tab page, you can view the Job Manager and Task Manager information of the running job.

                                                                  Information about JobManager and TaskManager is updated every minute. Only run logs of the last minute are displayed by default.

                                                                  -

                                                                  If you select an OBS bucket for saving job logs during the job configuration, you can switch to the OBS bucket and download log files to view more historical logs.

                                                                  -

                                                                  If the job is not running, information on the Task Manager page cannot be viewed.

                                                                  -

                                                                -
                                                                -
                                                                Parent topic: Flink Job Management
                                                                +
                                                                Parent topic: Managing Flink Jobs
                                                                diff --git a/docs/dli/umn/dli_01_0463.html b/docs/dli/umn/dli_01_0463.html index 3b873fc22..6dd84558d 100644 --- a/docs/dli/umn/dli_01_0463.html +++ b/docs/dli/umn/dli_01_0463.html @@ -1,6 +1,6 @@ -

                                                                Tag Management

                                                                +

                                                                Adding Tags to a Flink Job

                                                                A tag is a key-value pair customized by users and used to identify cloud resources. It helps users to classify and search for cloud resources. A tag consists of a tag key and a tag value.

                                                                DLI allows you to add tags to Flink jobs. You can add tags to Flink jobs to identify information such as the project name, service type, and background. If you use tags in other cloud services, you are advised to create the same tag key-value pairs for cloud resources used by the same business to keep consistency.

                                                                DLI supports the following two types of tags:

                                                                @@ -23,7 +23,7 @@
                                                                • Click the text box and select a predefined tag key from the drop-down list.

                                                                  To add a predefined tag, you need to create one on TMS and then select it from the Tag key drop-down list. You can click View predefined tags to go to the Predefined Tags page of the TMS console. Then, click Create Tag in the upper corner of the page to create a predefined tag.

                                                                -
                                                                • Enter a tag key in the text box.
                                                                  NOTE:

                                                                  A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                  +
                                                                  • Enter a tag key in the text box.
                                                                    NOTE:

                                                                    A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                  @@ -31,7 +31,7 @@
                                                                @@ -49,7 +49,7 @@ diff --git a/docs/dli/umn/dli_01_0464.html b/docs/dli/umn/dli_01_0464.html index 5077550b0..2c950b7bf 100644 --- a/docs/dli/umn/dli_01_0464.html +++ b/docs/dli/umn/dli_01_0464.html @@ -1,9 +1,9 @@ -

                                                                Managing Flink Templates

                                                                +

                                                                Managing Flink Job Templates

                                                                Flink templates include sample templates and custom templates. You can modify an existing sample template to meet the actual job logic requirements and save time for editing SQL statements. You can also customize a job template based on your habits and methods so that you can directly invoke or modify the template in later jobs.

                                                                Flink template management provides the following functions:

                                                                - +

                                                                Flink SQL Sample Template

                                                                The template list displays existing sample templates for Flink SQL jobs. Table 1 describes the parameters in the template list.

                                                                The scenarios of sample templates can be different, which are subject to the console.

                                                                @@ -15,12 +15,12 @@
                                                                - -
                                                                Table 5 Parameter descriptions

                                                                Parameter

                                                                Description

                                                                +

                                                                Description

                                                                Name

                                                                +

                                                                Name

                                                                Name of an operator.

                                                                +

                                                                Name of an operator.

                                                                Duration

                                                                +

                                                                Duration

                                                                Running duration of an operator.

                                                                +

                                                                Running duration of an operator.

                                                                Max Concurrent Jobs

                                                                +

                                                                Max Concurrent Jobs

                                                                Number of parallel tasks in an operator.

                                                                +

                                                                Number of parallel tasks in an operator.

                                                                Task

                                                                +

                                                                Task

                                                                Operator tasks are categorized as follows:

                                                                +

                                                                Operator tasks are categorized as follows:

                                                                • The digit in red indicates the number of failed tasks.
                                                                • The digit in light gray indicates the number of canceled tasks.
                                                                • The digit in yellow indicates the number of tasks that are being canceled.
                                                                • The digit in green indicates the number of finished tasks.
                                                                • The digit in blue indicates the number of running tasks.
                                                                • The digit in sky blue indicates the number of tasks that are being deployed.
                                                                • The digit in dark gray indicates the number of tasks in a queue.

                                                                Status

                                                                +

                                                                Status

                                                                Status of an operator task.

                                                                +

                                                                Status of an operator task.

                                                                Back Pressure Status

                                                                +

                                                                Back Pressure Status

                                                                Working load status of an operator. Available options are as follows:

                                                                +

                                                                Working load status of an operator. Available options are as follows:

                                                                • OK: indicates that the operator is in normal working load.
                                                                • LOW: indicates that the operator is in slightly high working load. DLI processes data quickly.
                                                                • HIGH: indicates that the operator is in high working load. The data input speed at the source end is slow.

                                                                Delay

                                                                +

                                                                Delay

                                                                Duration from the time when source data starts being processed to the time when data reaches the current operator. The unit is millisecond.

                                                                +

                                                                Duration from the time when source data starts being processed to the time when data reaches the current operator. The unit is millisecond.

                                                                Sent Records

                                                                +

                                                                Sent Records

                                                                Number of data records sent by an operator.

                                                                +

                                                                Number of data records sent by an operator.

                                                                Sent Bytes

                                                                +

                                                                Sent Bytes

                                                                Number of bytes sent by an operator.

                                                                +

                                                                Number of bytes sent by an operator.

                                                                Received Bytes

                                                                +

                                                                Received Bytes

                                                                Number of bytes received by an operator.

                                                                +

                                                                Number of bytes received by an operator.

                                                                Received Records

                                                                +

                                                                Received Records

                                                                Number of data records received by an operator.

                                                                +

                                                                Number of data records received by an operator.

                                                                Started

                                                                +

                                                                Started

                                                                Time when an operator starts running.

                                                                +

                                                                Time when an operator starts running.

                                                                Ended

                                                                +

                                                                Ended

                                                                Time when an operator stops running.

                                                                +

                                                                Time when an operator stops running.

                                                                Tag value

                                                                You can perform the following operations:

                                                                -
                                                                • Click the text box and select a predefined tag value from the drop-down list.
                                                                • Enter a tag value in the text box.
                                                                  NOTE:

                                                                  A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                                                  +
                                                                  • Click the text box and select a predefined tag value from the drop-down list.
                                                                  • Enter a tag value in the text box.
                                                                    NOTE:

                                                                    A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.

                                                                Name

                                                                Name of a template, which has 1 to 64 characters and only contains letters, digits, hyphens (-), and underlines (_).

                                                                +

                                                                Template name. Enter 1 to 64 characters. Only letters, numbers, hyphens (-), and underscores (_) are allowed.

                                                                Description

                                                                Description of a template. It contains 0 to 512 characters.

                                                                +

                                                                Description of a template. It can contain 0 to 512 characters.

                                                                Operation

                                                                @@ -32,37 +32,64 @@
                                                                -

                                                                Custom Templates

                                                                The custom template list displays all Jar job templates. Table 1 describes parameters in the custom template list.

                                                                +

                                                                Flink OpenSource SQL Sample Template

                                                                The template list displays existing sample templates for Flink SQL OpenSource jobs. Table 1 describes the parameters in the template list.

                                                                -
                                                                Table 2 Parameters in the custom template list

                                                                Parameter

                                                                +
                                                                - - - - - - - - +
                                                                Table 2 Parameters in the sample template list for Flink OpenSource SQL jobs

                                                                Parameter

                                                                Description

                                                                +

                                                                Description

                                                                Name

                                                                +

                                                                Name

                                                                Name of a template, which has 1 to 64 characters and only contains letters, digits, hyphens (-), and underlines (_).

                                                                +

                                                                Template name. Enter 1 to 64 characters. Only letters, numbers, hyphens (-), and underscores (_) are allowed.

                                                                Description

                                                                +

                                                                Description

                                                                Description of a template. It contains 0 to 512 characters.

                                                                +

                                                                Description of a template. It can contain 0 to 512 characters.

                                                                Created

                                                                +

                                                                Operation

                                                                Time when a template is created.

                                                                +

                                                                Create Job: Create a job directly by using the template. After a job is created, the system switches to the Edit page under Job Management.

                                                                Updated

                                                                +
                                                                +
                                                                + +

                                                                Custom Templates

                                                                The custom template list displays all Jar job templates. Table 1 describes parameters in the custom template list.

                                                                + +
                                                                + + + + - - - + + + + + + + + + @@ -72,33 +99,39 @@

                                                                Creating a Template

                                                                You can create a template using any of the following methods:

                                                                • Creating a template on the Template Management page
                                                                  1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates.
                                                                  2. Click Create Template in the upper right corner of the page. The Create Template dialog box is displayed.
                                                                  3. Specify Name and Description. -
                                                                Table 3 Parameters in the custom template list

                                                                Parameter

                                                                +

                                                                Description

                                                                +

                                                                Name

                                                                Latest time when a template is modified.

                                                                +

                                                                Template name. Enter 1 to 64 characters. Only letters, numbers, hyphens (-), and underscores (_) are allowed.

                                                                Operation

                                                                +

                                                                Description

                                                                • Edit: Modify a template that has been created.
                                                                • Create Job: Create a job directly by using the template. After a job is created, the system switches to the Edit page under Job Management.
                                                                • More:
                                                                  • Delete: Delete a created template.
                                                                  +

                                                                Description of a template. It can contain 0 to 512 characters.

                                                                +

                                                                Created

                                                                +

                                                                Time when a template is created.

                                                                +

                                                                Updated

                                                                +

                                                                Latest time when a template is modified.

                                                                +

                                                                Operation

                                                                +
                                                                • Edit: Modify a template that has been created.
                                                                • Create Job: Create a job directly by using the template. After a job is created, the system switches to the Edit page under Job Management.
                                                                • More:
                                                                  • Delete: Delete a created template.
                                                                Table 3 Template parameters

                                                                Parameter

                                                                +
                                                                - - - + + + - - - - @@ -106,58 +139,58 @@
                                                                Table 4 Template parameters

                                                                Parameter

                                                                Description

                                                                +

                                                                Description

                                                                Name

                                                                +

                                                                Type

                                                                Name of a template, which has 1 to 64 characters and only contains letters, digits, hyphens (-), and underlines (_).

                                                                +

                                                                Template type

                                                                +
                                                                • Flink SQL job template
                                                                • Flink OpenSource SQL job template
                                                                +

                                                                Name

                                                                +

                                                                Template name. Enter 1 to 64 characters. Only letters, numbers, hyphens (-), and underscores (_) are allowed.

                                                                NOTE:

                                                                The template name must be unique.

                                                                Description

                                                                +

                                                                Description

                                                                Description of a template. It contains 0 to 512 characters.

                                                                +

                                                                Description of a template. It can contain 0 to 512 characters.

                                                                Tags

                                                                +

                                                                Tags

                                                                Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                                                +

                                                                Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                                                NOTE:
                                                                • A maximum of 20 tags can be added.
                                                                • Only one tag value can be added to a tag key.
                                                                • The key name in each resource must be unique.
                                                                -
                                                                • Tag key: Enter a tag key name in the text box.
                                                                  NOTE:

                                                                  A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                  +
                                                                  • Tag key: Enter a tag key name in the text box.
                                                                    NOTE:

                                                                    A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                  -
                                                                  • Tag value: Enter a tag value in the text box.
                                                                    NOTE:

                                                                    A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                                                    +
                                                                    • Tag value: Enter a tag value in the text box.
                                                                      NOTE:

                                                                      A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                                                                -
                                                              4. Click OK to enter the editing page.
                                                                The Table 4 describes the parameters on the template editing page. -
                                                                Table 4 Template parameters

                                                                Parameter

                                                                +
                                                              5. Click OK to enter the editing page.
                                                                The Table 5 describes the parameters on the template editing page. +
                                                                - - - - - - - - - - - - - - - - -
                                                                Table 5 Template parameters

                                                                Parameter

                                                                Description

                                                                +

                                                                Description

                                                                Name

                                                                +

                                                                Name

                                                                You can modify the template name.

                                                                +

                                                                You can modify the template name.

                                                                Description

                                                                +

                                                                Description

                                                                You can modify the template description.

                                                                +

                                                                You can modify the template description.

                                                                Saving Mode

                                                                +

                                                                Saving Mode

                                                                • Save Here: Save the modification to the current template.
                                                                • Save as New: Save the modification as a new template.
                                                                +
                                                                • Save Here: Save the modification to the current template.
                                                                • Save as New: Save the modification as a new template.

                                                                SQL statement editing area

                                                                +

                                                                SQL statement editing area

                                                                In the area, you can enter detailed SQL statements to implement business logic. For details about how to compile SQL statements, see Data Lake Insight SQL Syntax Reference.

                                                                +

                                                                In the area, you can enter detailed SQL statements to implement business logic. For how to compile SQL statements, see Data Lake Insight SQL Syntax Reference.

                                                                Save

                                                                +

                                                                Save

                                                                Save the modifications.

                                                                +

                                                                Save the modifications.

                                                                Create Job

                                                                +

                                                                Create Job

                                                                Use the current template to create a job.

                                                                +

                                                                Use the current template to create a job.

                                                                Format

                                                                +

                                                                Format

                                                                Format SQL statements. After SQL statements are formatted, you need to compile SQL statements again.

                                                                +

                                                                Format SQL statements. After SQL statements are formatted, you need to compile SQL statements again.

                                                                Theme Settings

                                                                +

                                                                Theme Settings

                                                                Change the font size, word wrap, and page style (black or white background).

                                                                +

                                                                Change the font size, word wrap, and page style (black or white background).
                                                                -
                                                              6. In the SQL statement editing area, enter SQL statements to implement service logic. For details about how to compile SQL statements, see Data Lake Insight SQL Syntax Reference.
                                                              7. After the SQL statement is edited, click Save in the upper right corner to complete the template creation.
                                                              8. (Optional) If you do not need to modify the template, click Create Job in the upper right corner to create a job based on the current template. For details about how to create a job, see Creating a Flink SQL Job, and Creating a Flink Jar Job.
                                                                9. +
                                                              9. In the SQL statement editing area, enter SQL statements to implement service logic. For how to compile SQL statements, see Data Lake Insight SQL Syntax Reference.
                                                              10. After the SQL statement is edited, click Save in the upper right corner to complete the template creation.
                                                              11. (Optional) If you do not need to modify the template, click Create Job in the upper right corner to create a job based on the current template. For how to create a job, see Creating a Flink Jar Job.
                                                              12. Creating a template based on an existing job template
                                                                1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
                                                                2. In the row where the desired template is located in the custom template list, click Edit under Operation to enter the Edit page.
                                                                3. After the modification is complete, set Saving Mode to Save as New.
                                                                4. Click Save in the upper right corner to save the template as a new one.
                                                                • Creating a template using a created job
                                                                  1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                                  2. Click Create Job in the upper right corner. The Create Job page is displayed.
                                                                  3. Specify parameters as required.
                                                                  4. Click OK to enter the editing page.
                                                                  5. After the SQL statement is compiled, click Set as Template.
                                                                  6. In the Set as Template dialog box that is displayed, specify Name and Description and click OK.
                                                                  @@ -165,7 +198,7 @@

                                                                Creating a Job Based on a Template

                                                                You can create jobs based on sample templates or custom templates.

                                                                -
                                                                1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates.
                                                                2. In the sample template list, click Create Job in the Operation column of the target template. For details about how to create a job, see Creating a Flink SQL Job and Creating a Flink Jar Job.
                                                                +
                                                                1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates.
                                                                2. In the sample template list, click Create Job in the Operation column of the target template. For how to create a job, see Creating a Flink OpenSource SQL Job and Creating a Flink Jar Job.

                                                                Modifying a Template

                                                                After creating a custom template, you can modify it as required. The sample template cannot be modified, but you can view the template details.

                                                                1. In the left navigation pane of the DLI management console, choose Job Templates > Flink Templates. Click the Custom Templates tab.
                                                                2. In the row where the template you want to modify is located in the custom template list, click Edit in the Operation column to enter the Edit page.
                                                                3. In the SQL statement editing area, modify the SQL statements as required.
                                                                4. Set Saving Mode to Save Here.
                                                                5. Click Save in the upper right corner to save the modification.
                                                                @@ -177,7 +210,7 @@
                                                                -
                                                                Parent topic: Job Templates
                                                                +
                                                                Parent topic: Submitting a Flink Job Using DLI
                                                                diff --git a/docs/dli/umn/dli_01_0465.html b/docs/dli/umn/dli_01_0465.html index f5d625a17..8d36c8297 100644 --- a/docs/dli/umn/dli_01_0465.html +++ b/docs/dli/umn/dli_01_0465.html @@ -1,17 +1,19 @@ -

                                                                Spark Job Management

                                                                +

                                                                Submitting a Spark Job Using DLI

                                                                diff --git a/docs/dli/umn/dli_01_0476.html b/docs/dli/umn/dli_01_0476.html index b1eff87f8..c35f52f96 100644 --- a/docs/dli/umn/dli_01_0476.html +++ b/docs/dli/umn/dli_01_0476.html @@ -1,6 +1,6 @@ -

                                                                Global Variables

                                                                +

                                                                Managing DLI Global Variables

                                                                What Is a Global Variable?

                                                                DLI allows you to set variables that are frequently used during job development as global variables on the DLI management console. This avoids repeated definitions during job editing and reduces development and maintenance costs. Global variables can be used to replace long and difficult variables, simplifying complex parameters and improving the readability of SQL statements.

                                                                This section describes how to create a global variable.

                                                                @@ -28,20 +28,50 @@ partitioned by (int1 int,int2 int,int3 int)

                                                                Do not use global variables in OPTIONS of the table creation statements.

                                                                - -
                                                                -

                                                                Modifying a Global Variable

                                                                On the Global Variables page, click Modify in the Operation column of a variable to modify the variable value.

                                                                -

                                                                If there are multiple global variables with the same name in the same project under an account, delete the redundant global variables to ensure that the global variables are unique in the same project. In this case, all users who have the permission to modify the global variables can change the variable values.

                                                                +

                                                                Related operations:

                                                                +
                                                                • Modifying a global variable
                                                                  On the Global Variables page, locate a desired variable and click Modify in the Operation column.

                                                                  If there are multiple global variables with the same name in the same project under an account, delete the redundant global variables to ensure that the global variables are unique in the same project. In this case, all users who have the permission to modify the global variables can change the variable values.

                                                                  -

                                                                  Deleting a Global Variable

                                                                  On the Global Variables page, click Delete in the Operation column of a variable to delete the variable value.

                                                                  +
                                                                • Deleting a global variable

                                                                  On the Global Variables page, click Delete in the Operation column of a variable to delete the variable value.

                                                                  • If there are multiple global variables with the same name in the same project under an account, delete the global variables created by the user first. If there are only unique global variables, all users who have the delete permission can delete the global variables.
                                                                  • After a variable is deleted, the variable cannot be used in SQL statements.
                                                                  +
                                                                + +
                                                                +

                                                                Permissions Management for Global Variables

                                                                You can assign different users different global variables through permission settings. The administrator and owners of global variables have all permissions. You do not need to set permissions for them, and their global variable permissions cannot be modified by other users.

                                                                +

                                                                When setting global variables for a new user, the user's group must have the Tenant Guest permission.

                                                                +
                                                                • Granting permissions on a global variable to a user
                                                                  1. In the navigation pane on the left of the DLI console, choose Global Configuration > Global Variables. On the displayed page, locate a desired global variable and click Set Permission in the Operation column. On the displayed User Permissions page, you can grant, set, and revoke permissions on the global variable to users.
                                                                  2. Click Grant Permission in the upper right corner. +
                                                                    + + + + + + + + + + +
                                                                    Table 2 Global variable parameters

                                                                    Parameter

                                                                    +

                                                                    Description

                                                                    +

                                                                    Username

                                                                    +

                                                                    Name of the IAM user who is granted permissions

                                                                    +
                                                                    NOTE:

                                                                    This username must be an existing IAM username.

                                                                    +
                                                                    +

                                                                    Permissions

                                                                    +
                                                                    • Update: This permission allows you to update the global variable.
                                                                    • Delete: This permission allows you to delete the global variable.
                                                                    • Grant Permission: This permission allows you to grant permissions of the global variable to other users.
                                                                    • Revoke Permission: This permission allows you to revoke the global variable permissions that other users have but cannot revoke the global variable owner's permissions.
                                                                    • View Other User's Permissions: This permission allows you to view the global variable permissions of other users.
                                                                    +
                                                                    +
                                                                    +
                                                                  +
                                                                • Granting permissions on a global variable to a user

                                                                  On the User Permissions page, locate a desired IAM user and click Set Permission in the Operation column. Table 2 lists the permission parameters.

                                                                  +

                                                                  If all permission options are grayed out, it means you do not have the authority to modify the permissions on this global variable. You can request the modification permission from users who have authorization, such as the administrator or group owners.

                                                                  +
                                                                • Revoking permissions on a global variable from a user

                                                                  On the User Permissions page, locate a desired IAM user and click Revoke Permission in the Operation column. Once the revoke operation is complete, the IAM user will no longer have any permissions on the global variable.

                                                                  +
                                                                diff --git a/docs/dli/umn/dli_01_0477.html b/docs/dli/umn/dli_01_0477.html index 0f57661e2..952137d07 100644 --- a/docs/dli/umn/dli_01_0477.html +++ b/docs/dli/umn/dli_01_0477.html @@ -1,51 +1,52 @@ -

                                                                Managing Permissions on Packages and Package Groups

                                                                -

                                                                Scenario

                                                                • You can isolate package groups or packages allocated to different users by setting permissions to ensure data query performance.
                                                                • The administrator and the owner of a package group or package have all permissions. You do not need to set permissions and the permissions cannot be modified by other users.
                                                                • When you set permissions on a package group or a package to a new user, the user group the user belong to must have the Tenant Guest permission.
                                                                -
                                                                -

                                                                On the Package Management page, click Manage Permissions in the Operation column of the target package. On the displayed User Permission Info page, you can grant permissions for the package group or package, set and revoke user permissions.

                                                                -
                                                                • If you select a group when creating a package, you can manage permissions of the corresponding program package group.
                                                                • If you select No grouping when creating a package, you can manage permissions of the corresponding package.
                                                                +

                                                                Configuring DLI Package Permissions

                                                                +

                                                                By configuring permissions, you can grant different package groups or packages to various users, ensuring that job efficiency remains unaffected and job performance is maintained.

                                                                +
                                                                • Administrators and package group owners have all permissions for the package group. No permission settings are required, and other users cannot modify their package group permissions.
                                                                • Administrators and package owners have all permissions for the package. No permission settings are required, and other users cannot modify their package permissions.
                                                                • Package groups are used to manage packages with consistent behavior, so they support granting related permissions to package groups, but do not support granting individual permissions to packages within a package group.
                                                                • When an administrator assigns package group or package permissions to a new user, the administrator's user group must have Tenant Guest permissions.

                                                                  +

                                                                  +

                                                                  +
                                                                +

                                                                Configuring Permissions on Package Groups or Packages

                                                                1. On the Data Management > Package Management page, locate the package group or package whose permissions you want to grant and click Manage Permission in the Operation column.
                                                                2. On the displayed User Permissions page, click Grant Permission in the upper right corner of the page. In the dialog box that appears, enter a username in Username, select required permissions, and click OK.
                                                                  • If you select a group when creating a package, you can manage permissions of the corresponding program package group.
                                                                  • If you select No grouping when creating a package, you can manage permissions of the corresponding package.
                                                                  -

                                                                  Granting Permissions on Package Groups/Packages

                                                                  Click Grant Permission in the upper right corner of the page.

                                                                  -
                                                                  • Granting permissions on package groups -
                                                                    Table 1 Permission parameters

                                                                    Parameter

                                                                    +
                                                                    • Granting permissions on package groups +
                                                                      - - - - -
                                                                      Table 1 Permission parameters

                                                                      Parameter

                                                                      Description

                                                                      +

                                                                      Description

                                                                      Username

                                                                      +

                                                                      Username

                                                                      Name of the authorized IAM user.

                                                                      +

                                                                      Name of the authorized IAM user.

                                                                      NOTE:

                                                                      The username is the name of an existing IAM user.

                                                                      Select the permissions to be granted to the user

                                                                      +

                                                                      Select the permissions to be granted to the user

                                                                      • Use Group: This permission allows you to use the package of this group.
                                                                      • Update Group: This permission allows you to update the packages in the group, including creating a package in the group.
                                                                      • Query Group: This permission allows you to query the details of a package in a group.
                                                                      • Delete Group: This permission allows you to delete the package of the group.
                                                                      • Grant Permission: This permission allows you to grant group permissions to other users.
                                                                      • Revoke Permission: This permission allows you to revoke the group permissions that other users have but cannot revoke the group owner's permissions.
                                                                      • View Other User's Permissions: This permission allows you to view the group permissions of other users.
                                                                      +
                                                                      • Use Group: This permission allows you to use the package of this group.
                                                                      • Update Group: This permission allows you to update the packages in the group, including creating a package in the group.
                                                                      • Query Group: This permission allows you to query the details of a package in a group.
                                                                      • Delete Group: This permission allows you to delete the package of the group.
                                                                      • Grant Permission: This permission allows you to grant group permissions to other users.
                                                                      • Revoke Permission: This permission allows you to revoke the group permissions that other users have but cannot revoke the group owner's permissions.
                                                                      • View Other User's Permissions: This permission allows you to view the group permissions of other users.
                                                                    • Granting permissions on packages -
                                                                      Table 2 Permission parameters

                                                                      Parameter

                                                                      +
                                                                      - - - - - @@ -53,20 +54,25 @@ -

                                                                      Setting Permissions on Package Groups and Packages

                                                                      Click Set Permission in the Operation column of the sub-user to modify the permission of the user. Table 1 and Table 2 list the detailed permission descriptions.

                                                                      -

                                                                      If the Set Permission button is gray, you do not have the permission to modify the package group or package. You can apply to the administrator, group owner, or other users who have the permissions on granting and revoking permissions of package groups or packages.

                                                                      +
                                                                      -

                                                                      Revoking Permissions on Package Groups and Packages

                                                                      Click Revoke Permission in the Operation column of a sub-user to revoke the user's permissions. After the operation, the sub-user does not have any permission on the package group or package.

                                                                      +

                                                                      Modifying Permissions on Package Groups or Packages

                                                                      1. On the Data Management > Package Management page, locate the desired package group or package and click Manage Permission in the Operation column.
                                                                      2. On the displayed User Permissions page, click Set Permission in the Operation column of the IAM user for whom you want to modify the permissions.
                                                                        Table 1 and Table 2 list the detailed permission descriptions.
                                                                        • If you set Group to Use existing or Use new when creating a package, you will modify the permissions on the group you selected.
                                                                        • If you set Group to Do not use when creating a package, you will modify the permissions on the package.
                                                                        +
                                                                        -

                                                                        Permissions Description

                                                                        • Package group permissions

                                                                          Querying permissions. A group owner can view the created package group and all packages in the group, and can also view package groups on which they have all permissions.

                                                                          -

                                                                          A package group is a unit. If you select a group when creating a package, you can grant only the permissions of the package group to other users.

                                                                          -
                                                                        • Package permissions

                                                                          Querying permissions. A package owner can view the created packages, and can also view packages on which they have all permissions.

                                                                          -
                                                                        +

                                                                        If Set Permission on the User Permissions page is grayed out, you do not have the permission to modify the permissions on the package group or package.

                                                                        +

                                                                        You can request the Grant Permission and Revoke Permission permissions on package groups or packages from users who have authorization privileges, such as administrators or group owners.

                                                                        +
                                                                      +
                                                                      +

                                                                      Revoking Permissions on Package Groups or Packages

                                                                      DLI allows you to revoke permissions on package groups or packages with just one click.

                                                                      +
                                                                      1. On the Data Management > Package Management page, locate the desired package group or package and click Manage Permission in the Operation column.
                                                                      2. On the displayed User Permissions page, click Revoke Permission in the Operation column of the IAM user for whom you want to revoke the permissions.

                                                                        Once the permissions are revoked, the IAM user does not have any permissions on the package group or package.

                                                                        +
                                                                        • If you set Group to Use existing or Use new when creating a package, you will revoke the permissions on the group you selected.
                                                                        • If you set Group to Do not use when creating a package, you will revoke the permissions on the package.
                                                                        +
                                                                        +
                                                                      diff --git a/docs/dli/umn/dli_01_0478.html b/docs/dli/umn/dli_01_0478.html index 4fa1f05ee..3d2096865 100644 --- a/docs/dli/umn/dli_01_0478.html +++ b/docs/dli/umn/dli_01_0478.html @@ -1,33 +1,33 @@ -

                                                                      Modifying the Owner

                                                                      -

                                                                      To change the owner of a package, click More > Modify Owner in the Operation column of a package on the Package Management page.

                                                                      -
                                                                      • If the package has been grouped, you can modify the owner of the Group or Resource of it.
                                                                      • If the package has not been grouped, change its owner directly.
                                                                      +

                                                                      Changing the DLI Package Owner

                                                                      +

                                                                      DLI allows you to change the owner of a package group or package.

                                                                      +
                                                                      1. Log in to the DLI management console and choose Data Management > Package Management.
                                                                      2. On the Package Management page, locate the package whose owner you want to change, click More in the Operation column, and select Modify Owner.
                                                                        • If the package has been grouped, you can change the owner of the group or package by selecting Group or Packages for Select Type and entering the new owner's username in Username.
                                                                        • If the package has not been grouped, change its owner directly.
                                                                        -
                                                                      Table 2 Permission parameters

                                                                      Parameter

                                                                      Description

                                                                      +

                                                                      Description

                                                                      Username

                                                                      +

                                                                      Username

                                                                      Name of the authorized IAM user.

                                                                      +

                                                                      Name of the authorized IAM user.

                                                                      NOTE:

                                                                      The username is the name of an existing IAM user.

                                                                      Select the permissions to be granted to the user

                                                                      +

                                                                      Select the permissions to be granted to the user

                                                                      • Use Package: This permission allows you to use the package.
                                                                      • Update Package: This permission allows you to update the package.
                                                                      • Query Package: This permission allows you to query the package.
                                                                      • Delete Package: This permission allows you to delete the package.
                                                                      • Grant Permission: This permission allows you to grant package permissions to other users.
                                                                      • Revoke Permission: This permission allows you to revoke the package permissions that other users have but cannot revoke the package owner's permissions.
                                                                      • View Other User's Permissions: This permission allows you to view the package permissions of other users.
                                                                      +
                                                                      • Use Package: This permission allows you to use the package.
                                                                      • Update Package: This permission allows you to update the package.
                                                                      • Query Package: This permission allows you to query the package.
                                                                      • Delete Package: This permission allows you to delete the package.
                                                                      • Grant Permission: This permission allows you to grant package permissions to other users.
                                                                      • Revoke Permission: This permission allows you to revoke the package permissions that other users have but cannot revoke the package owner's permissions.
                                                                      • View Other User's Permissions: This permission allows you to view the package permissions of other users.
                                                                      @@ -36,11 +36,11 @@ -

                                                                      Flink Job Permissions

                                                                      • View Job Details
                                                                        • Tenants and the admin user can view and operate all jobs.
                                                                        • Subusers and users with the read-only permission can only view their own jobs.

                                                                          If another user grants any permission other than the job viewing permission to a subuser, the job is displayed in the job list, but the details cannot be viewed by the subuser.

                                                                          +

                                                                          Flink Job Permissions

                                                                          • View Job Details
                                                                            • Tenants and the admin user can view and operate all jobs.
                                                                            • Subusers and users with the read-only permission can only view their own jobs.

                                                                              If another user grants any permission other than the job viewing permission to a subuser, the job is displayed in the job list, but the details cannot be viewed by the subuser.

                                                                            -
                                                                          • Start Job
                                                                            • To use a dedicated queue, you must have the permission to submit and start jobs.
                                                                            • To use a shared queue, you only need to have the permission to start jobs.
                                                                            -
                                                                          • Stop Job
                                                                            • To use a dedicated queue, you must have the permission to stop jobs and queues.
                                                                            • To use a shared queue, you only need to have the permission to stop jobs.
                                                                            +
                                                                          • Start Job

                                                                            You must have the permission to submit and start jobs.

                                                                            +
                                                                          • Stop Job

                                                                            You must have the permission to stop queues and jobs.

                                                                          • Delete Job
                                                                            • If a job can be deleted, you can delete the job if you were granted this permission.
                                                                            • If a job cannot be deleted, the system stops the job before you delete it. For details about how to stop a job, see Stop Job. In addition, you must have the permission to delete the job.
                                                                          • Create Job
                                                                            • By default, sub-users cannot create jobs.
                                                                            • To create a job, you must have this permission. Currently, only the admin user has the permission to create jobs. In addition, the user must have the permission of the related package group or package used by the job.
                                                                          • Modify Job

                                                                            When modifying a job, you need to have the permission to update the job and the permission to the package group or package used by the job belongs.

                                                                            @@ -49,7 +49,7 @@
                                                                          diff --git a/docs/dli/umn/dli_01_0480.html b/docs/dli/umn/dli_01_0480.html index aa94b6b97..19d4f9021 100644 --- a/docs/dli/umn/dli_01_0480.html +++ b/docs/dli/umn/dli_01_0480.html @@ -39,7 +39,7 @@
                                                                          diff --git a/docs/dli/umn/dli_01_0485.html b/docs/dli/umn/dli_01_0485.html deleted file mode 100644 index ed0c3da67..000000000 --- a/docs/dli/umn/dli_01_0485.html +++ /dev/null @@ -1,17 +0,0 @@ - - - -

                                                                          Global Configuration

                                                                          - -

                                                                          -
                                                                          - - - diff --git a/docs/dli/umn/dli_01_0486.html b/docs/dli/umn/dli_01_0486.html new file mode 100644 index 000000000..b355ba983 --- /dev/null +++ b/docs/dli/umn/dli_01_0486.html @@ -0,0 +1,16 @@ + + +

                                                                          Configuring an Agency to Allow DLI to Access Other Cloud Services

                                                                          +

                                                                          +
                                                                          + + diff --git a/docs/dli/umn/dli_01_0487.html b/docs/dli/umn/dli_01_0487.html index 779e7c280..41c883daa 100644 --- a/docs/dli/umn/dli_01_0487.html +++ b/docs/dli/umn/dli_01_0487.html @@ -1,7 +1,9 @@ -

                                                                          Elastic Scaling of Queues

                                                                          -

                                                                          Prerequisites

                                                                          Elastic scaling can be performed for a newly created queue only when there were jobs running in this queue.

                                                                          +

                                                                          Auto Scaling of Standard Queues

                                                                          +

                                                                          Prerequisites

                                                                          Newly created queues need to run jobs before they can be scaled in or out.

                                                                          +

                                                                          The operations described in this section only apply to standard queues.

                                                                          +

                                                                          Notes and Constraints

                                                                          • Queues with 16 CUs do not support scale-out or scale-in.
                                                                          • Queues with 64 CUs do not support scale-in.
                                                                          • If Status of queue xxx is assigning, which is not available is displayed on the Elastic Scaling page, the queue can be scaled only after the queue resources are allocated.
                                                                          • If there are not enough physical resources, a queue may not be able to scale out to the desired target size.
                                                                          • The system does not guarantee that a queue will be scaled in to the desired target size. Typically, the system checks the resource usage before scaling in the queue to determine if there is enough space for scaling in. If the existing resources cannot be scaled in according to the minimum scaling step, the queue may not be scaled in successfully or only partially.

                                                                            The scaling step may vary depending on the resource specifications, usually 16 CUs, 32 CUs, 48 CUs, 64 CUs, etc.

                                                                            For example, if the queue size is 48 CUs and job execution uses 18 CUs, the remaining 30 CUs do not meet the requirement for scaling in by the minimum step of 32 CUs. If a scaling in task is executed, it will fail.

                                                                            @@ -11,18 +13,18 @@

                                                                            Scale-out is time-consuming. After you perform scale-out on the Elastic Scaling page of DLI, wait for about 10 minutes. The duration is related to the CU amount to add. After a period of time, refresh the Queue Management page and check whether values of Specifications and Actual CUs are the same to determine whether the scale-out is successful. Alternatively, on the Job Management page, check the status of the SCALE_QUEUE SQL job. If the job status is Scaling, the queue is being scaled out.

                                                                            The procedure is as follows:

                                                                            -
                                                                            1. On the left of the DLI management console, click Resources > Queue Management.
                                                                            2. Select the queue to be scaled out and choose More > Elastic Scaling in the Operation column.
                                                                            3. On the displayed page, select Scale-out for Operation and set the scale-out amount.
                                                                            4. Click .
                                                                            +
                                                                            1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                                                                            2. Select the queue to be scaled out, click More in the Operation column, and select Elastic Scaling.
                                                                            3. On the displayed page, select Scale-out for Operation and set the scale-out amount.
                                                                            4. Click .

                                                                          Scaling In

                                                                          If the current queue specifications are too much for your computing service, you can reduce the number of CUs to scale in the queue.

                                                                          • Scale-in is time-consuming. After you perform scale-in on the Elastic Scaling page of DLI, wait for about 10 minutes. The duration is related to the CU amount to reduce. After a period of time, refresh the Queue Management page and check whether values of Specifications and Actual CUs are the same to determine whether the scale-in is successful. Alternatively, on the Job Management page, check the status of the SCALE_QUEUE SQL job. If the job status is Scaling, the queue is being scaled in.
                                                                          • By default, the minimum number of CUs is 16. That is, when the queue specifications are 16 CUs, you cannot scale in the queue.

                                                                          The procedure is as follows:

                                                                          -
                                                                          1. On the left of the DLI management console, click Resources > Queue Management.
                                                                          2. Select the queue to be scaled out, click More in the Operation column, and select Elastic Scaling.
                                                                          3. On the displayed page, select Scale-in for Operation and set the scale-in amount.
                                                                          4. Click .
                                                                          +
                                                                          1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                                                                          2. Select the queue to be scaled in, click More in the Operation column, and select Elastic Scaling.
                                                                          3. On the displayed page, select Scale-in for Operation and set the scale-in amount.
                                                                          4. Click .
                                                                          -
                                                                          Parent topic: Queue Management
                                                                          +
                                                                          Parent topic: Managing Queues
                                                                          diff --git a/docs/dli/umn/dli_01_0488.html b/docs/dli/umn/dli_01_0488.html index ecb782dfb..2baf2137b 100644 --- a/docs/dli/umn/dli_01_0488.html +++ b/docs/dli/umn/dli_01_0488.html @@ -1,15 +1,17 @@ -

                                                                          Scheduling CU Changes

                                                                          +

                                                                          Setting a Scheduled Auto Scaling Task for a Standard Queue

                                                                          Scenario

                                                                          When services are busy, you might need to use more compute resources to process services in a period. After this period, you do not require the same amount of resources. If the purchased queue specifications are small, resources may be insufficient during peak hours. If the queue specifications are large, resources may be wasted.

                                                                          DLI provides scheduled tasks for elastic scale-in and -out in the preceding scenario. You can set different queue sizes (CUs) at different time or in different periods based on your service period or usage and the existing queue specifications to meet your service requirements and reduce costs.

                                                                          +

                                                                          The operations described in this section only apply to standard queues.

                                                                          +
                                                                          -

                                                                          Precautions

                                                                          • Periodic scaling can be performed for a newly created queue only when there were jobs running in this queue.
                                                                          • Scheduled scaling tasks are available only for a queue with more than 64 CUs. That is, the minimum specifications of a queue are 64 CUs.
                                                                          • A maximum of 12 scheduled tasks can be created for each queue.
                                                                          • When each scheduled task starts, the actual start time of the specification change has a deviation of 5 minutes. It is recommended that the task start time be at least 20 minutes earlier than the time when the queue is actually used.
                                                                          • The interval between two scheduled tasks must be at least 2 hours.
                                                                          • Changing the specifications of a queue is time-consuming. The time required for changing the specifications depends on the difference between the target specifications and the current specifications. You can view the specifications of the current queue on the Queue Management page.
                                                                          • If a job is running in the current queue, the queue may fail to be scaled in to the target CU amount value. Instead, it will be scaled in to a value between the current queue specifications and the target specifications. The system will try to scale in again 1 hour later until the next scheduled task starts.
                                                                          • If a scheduled task does not scale out or scale in to the target CU amount value, the system triggers the scaling plan again 15 minutes later until the next scheduled task starts.
                                                                          +

                                                                          Precautions

                                                                          • Newly created queues need to run jobs before they can be scaled in or out.
                                                                          • Scheduled scaling tasks are available only for a queue with more than 64 CUs. That is, the minimum specifications of a queue are 64 CUs.
                                                                          • A maximum of 12 scheduled tasks can be created for each queue.
                                                                          • When each scheduled task starts, the actual start time of the specification change has a deviation of 5 minutes. It is recommended that the task start time be at least 20 minutes earlier than the time when the queue is actually used.
                                                                          • The interval between two scheduled tasks must be at least 2 hours.
                                                                          • Changing the specifications of a queue is time-consuming. The time required for changing the specifications depends on the difference between the target specifications and the current specifications. You can view the specifications of the current queue on the Queue Management page.
                                                                          • If a job is running in the current queue, the queue may fail to be scaled in to the target CU amount value. Instead, it will be scaled in to a value between the current queue specifications and the target specifications. The system will try to scale in again 1 hour later until the next scheduled task starts.
                                                                          • If a scheduled task does not scale out or scale in to the target CU amount value, the system triggers the scaling plan again 15 minutes later until the next scheduled task starts.

                                                                          Creating Periodic Task

                                                                          • If only scale-out or scale-in is required, you need to create only one task for changing specifications. Set the Task Name, Final CU Count, and Executed parameters. For details, see Table 1.
                                                                          • To set both scale-out and scale-in parameters, you need to create two periodic tasks, and set the Task Name, Final CU Count, and Executed parameters. For details, see Table 1.

                                                                          The procedure is as follows:

                                                                          -
                                                                          1. On the left of the DLI management console, click Resources > Queue Management.
                                                                          2. Locate the queue for which you want to schedule a periodic task for elastic scaling, and choose More > Schedule CU Changes in the Operation column.
                                                                          3. On the displayed page, click Create Periodic Task in the upper right corner.
                                                                          4. On the Create Periodic Task page, set the required parameters. Click OK. -
                                                                      Table 1 Description

                                                                      Parameter

                                                                      +
                                                                      - - - - - - - - - @@ -35,10 +35,11 @@
                                                                      Table 1 Description

                                                                      Parameter

                                                                      Description

                                                                      +

                                                                      Description

                                                                      Group Name

                                                                      +

                                                                      Group Name

                                                                      • If you select a group when creating a package, the name of the group is displayed.
                                                                      • If no group is selected when creating a package, this parameter is not displayed.
                                                                      +
                                                                      • If you select a group when creating a package, the name of the group is displayed.
                                                                      • If no group is selected when creating a package, this parameter is not displayed.

                                                                      Name

                                                                      +

                                                                      Name

                                                                      Name of a package.

                                                                      +

                                                                      Name of a package.

                                                                      Select Type

                                                                      +

                                                                      Select Type

                                                                      • If you select a group when creating a package, you can change the owner of the group or package.
                                                                      • If no group is selected when creating a package, this parameter is not displayed.
                                                                      +
                                                                      • If you select a group when creating a package, you can change the owner of the group or package.
                                                                      • If no group is selected when creating a package, this parameter is not displayed.

                                                                      Username

                                                                      +

                                                                      Username

                                                                      Name of the package owner.

                                                                      +

                                                                      Name of the package owner.

                                                                      NOTE:

                                                                      The username is the name of an existing IAM user.
                                                                      +
                                                                    • Click OK.
                                                                      • diff --git a/docs/dli/umn/dli_01_0479.html b/docs/dli/umn/dli_01_0479.html index 39e5383e7..bb4870e92 100644 --- a/docs/dli/umn/dli_01_0479.html +++ b/docs/dli/umn/dli_01_0479.html @@ -1,6 +1,6 @@ -

                                                                      Managing Flink Job Permissions

                                                                      +

                                                                      Configuring Flink Job Permissions

                                                                      Scenario

                                                                      • You can isolate Flink jobs allocated to different users by setting permissions to ensure data query performance.
                                                                      • The administrator and job creator have all permissions, which cannot be set or modified by other users.

                                                                      Flink Job Permission Operations

                                                                      1. On the left of the DLI management console, choose Job Management > Flink Jobs.
                                                                      2. Select the job to be configured and choose More > Permissions in the Operation column. The User Permissions area displays the list of users who have permissions on the job.

                                                                        @@ -16,7 +16,7 @@

                                                                      Username

                                                                      Name of the user you want to grant permissions to.

                                                                      -
                                                                      NOTE:

                                                                      The username is the name of an existing IAM user. In addition, the user can perform authorization operations only after logging in to the platform.

                                                                      +
                                                                      NOTE:

                                                                      This username must be an existing IAM username. In addition, the user can perform authorization operations only after logging in to the platform.
                                                                      Table 1 Parameter description

                                                                      Parameter

                                                                      +
                                                                      1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                                                                      2. Locate the queue for which you want to schedule a periodic task for elastic scaling, and choose More > Schedule CU Changes in the Operation column.
                                                                      3. On the displayed page, click Create Periodic Task in the upper right corner.
                                                                      4. On the Create Periodic Task page, set the required parameters. Click OK. +
                                                                        @@ -18,7 +20,7 @@
                                                                        Table 1 Parameters

                                                                        Parameter

                                                                        Description

                                                                        Task Name

                                                                        Enter the name of the periodic task.

                                                                        -
                                                                        • The task name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_) or be left unspecified.
                                                                        • The name can contain a maximum of 128 characters.
                                                                        +
                                                                        • Only numbers, letters, and underscores (_) are allowed. The value cannot contain only numbers or start with an underscore (_) or be left unspecified.
                                                                        • The value can contain a maximum of 128 characters.

                                                                        Enable Task

                                                                        @@ -66,15 +68,15 @@

                                                                        Modifying a Scheduled Task

                                                                        If a periodic task cannot meet service requirements anymore, you can modify it on the Schedule CU Changes page.

                                                                        -
                                                                        1. In the navigation pane of the DLI management console, choose Resources >Queue Management.
                                                                        2. Locate the queue for which you want to schedule a periodic task for elastic scaling, and choose More > Schedule CU Changes in the Operation column.
                                                                        3. On the displayed page, click Modify in the Operation column. In the displayed dialog box, modify the task parameters as needed.
                                                                        +
                                                                        1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                                                                        2. Locate the queue for which you want to schedule a periodic task for elastic scaling, and choose More > Schedule CU Changes in the Operation column.
                                                                        3. On the displayed page, click Modify in the Operation column. In the displayed dialog box, modify the task parameters as needed.

                                                                        Deleting a Scheduled Task

                                                                        If you do not need the task anymore, delete the task on the Schedule CU Changes page.

                                                                        -
                                                                        1. In the navigation pane of the DLI management console, choose Resources >Queue Management.
                                                                        2. Locate the queue for which you want to schedule a periodic task for elastic scaling, and choose More > Schedule CU Changes in the Operation column.
                                                                        3. On the displayed page, click Delete in the Operation column. In the displayed dialog box, click OK.
                                                                        +
                                                                        1. In the navigation pane on the left of the DLI management console, choose Resources > Queue Management.
                                                                        2. Locate the queue for which you want to schedule a periodic task for elastic scaling, and choose More > Schedule CU Changes in the Operation column.
                                                                        3. On the displayed page, click Delete in the Operation column. In the displayed dialog box, click OK.
                                                                        -
                                                                        Parent topic: Queue Management
                                                                        +
                                                                        Parent topic: Managing Queues
                                                                        diff --git a/docs/dli/umn/dli_01_0489.html b/docs/dli/umn/dli_01_0489.html index 9bae064ae..c59b1c359 100644 --- a/docs/dli/umn/dli_01_0489.html +++ b/docs/dli/umn/dli_01_0489.html @@ -1,14 +1,22 @@

                                                                        Testing Address Connectivity

                                                                        -

                                                                        It can be used to test the connectivity between the DLI queue and the peer IP address specified by the user in common scenarios, or the connectivity between the DLI queue and the peer IP address bound to the datasource connection in datasource connection scenarios. The operation is as follows:

                                                                        -
                                                                        1. On the Queue Management page, locate the row containing the target queue, click More in the Operation column, and select Test Address Connectivity.
                                                                        2. On the Test Address Connectivity page, enter the address to be tested. The domain name and IP address are supported, and the port number can be specified.
                                                                        3. Click Test.

                                                                          If the test address is reachable, a message is displayed on the page, indicating that the address is reachable.

                                                                          -

                                                                          If the test address is unreachable, the system displays a message indicating that the address is unreachable. Check the network configurations and retry. Network configurations include the VPC peering and the datasource connection. Check whether they have been activated.

                                                                          +

                                                                          DLI's address connectivity testing feature can be used to verify network connectivity between DLI queues and destination addresses.

                                                                          +

                                                                          This feature is typically utilized for reading and writing external data sources. Once a datasource connection is configured, the communication capability between the DLI queue and the bound peer address is verified.

                                                                          +

                                                                          Testing the Address Connectivity Between a Queue and the Data Source

                                                                          1. Log in to the DLI management console. In the navigation pane on the left, choose Resources > Queue Management.
                                                                          2. On the Queue Management page, locate the row containing the target queue, click More in the Operation column, and select Test Address Connectivity.
                                                                          3. On the Test Address Connectivity page, enter the address to be tested. The domain name and IP address are supported, and the port number can be specified.

                                                                            You can input the data source address in the following formats: IPv4 address; IPv4 address + Port number; Domain name; Domain name + Port number.

                                                                            +

                                                                            · IPv4 address: 192.168.x.x

                                                                            +

                                                                            · IPv4 + Port number: 192.168.x.x:8080

                                                                            +

                                                                            · Domain name: domain-xxxxxx.com

                                                                            +

                                                                            · Domain name + Port number: domain-xxxxxx.com:8080

                                                                            +
                                                                          4. Click Test.
                                                                            • If the test address is reachable, a message is displayed on the page, indicating that the address is reachable.
                                                                            • If the test address is unreachable, the system displays a message indicating that the address is unreachable. Check the network configurations and retry. Network configurations include the VPC peering and the datasource connection. Check whether they have been activated.
                                                                          +

                                                                          Related Operations

                                                                          Why Is a Datasource Connection Successfully Created But the Network Connectivity Test Fails?

                                                                          +
                                                                          +
                                                                          -
                                                                          Parent topic: Queue Management
                                                                          +
                                                                          Parent topic: Managing Queues
                                                                          diff --git a/docs/dli/umn/dli_01_0494.html b/docs/dli/umn/dli_01_0494.html new file mode 100644 index 000000000..4e262bc62 --- /dev/null +++ b/docs/dli/umn/dli_01_0494.html @@ -0,0 +1,62 @@ + + +

                                                                          Using a Custom Image to Enhance the Job Running Environment

                                                                          +

                                                                          Scenario

                                                                          To enhance the functions and performance of Spark and Flink jobs, you can create custom images by downloading the base images provided by DLI and adding dependencies (files, JAR files, or software) and private capabilities required for job execution. This changes the container runtime environment for the jobs.

                                                                          +

                                                                          For example, you can add a Python package or C library related to machine learning to a custom image to help you extend functions.

                                                                          +

                                                                          To use the custom image function, you need to have basic knowledge of Docker.

                                                                          +
                                                                          +
                                                                          +

                                                                          Notes and Constraints

                                                                          • The base images provided by DLI must be used to create custom images.
                                                                          • You cannot modify the DLI components and directories in the base images.
                                                                          • Only Spark Jar and Flink Jar jobs are supported.
                                                                          +
                                                                          +

                                                                          Use Process

                                                                          Figure 1 Process of using a custom image
                                                                          +
                                                                          1. Obtain DLI base images.
                                                                          2. Use Dockerfile to pack dependencies (files, JAR files, or software) required for job execution into the base image to create a custom image.
                                                                          3. Publish the custom image to SoftWare Repository for Container (SWR).
                                                                          4. On the DLI job editing page, select the created image and run the job.
                                                                          5. Check the job execution status.
                                                                          +
                                                                          +

                                                                          Obtaining DLI Base Images

                                                                          Contact the administrator to obtain DLI base images.

                                                                          +

                                                                          Select the base image of the same type as the architecture of the queue.

                                                                          +

                                                                          For the CPU architecture type of a queue, see Viewing Basic Information About a Queue.

                                                                          +
                                                                          +

                                                                          Creating a Custom Image

                                                                          The following describes how to package TensorFlow into an image to generate a custom image with TensorFlow installed. Then, you can use the image to run jobs in DLI.

                                                                          +
                                                                          1. Prepare the container environment.
                                                                          2. Log in to the prepared container environment as user root and run a command to obtain the base image.

                                                                            In this example, the Spark base image is used and downloaded to the container image environment in 1 by running the following command:

                                                                            +

                                                                            docker pull Address for downloading the base image

                                                                            +

                                                                            For details about the address, see Using a Custom Image to Enhance the Job Running Environment.

                                                                            +
                                                                            For example, to download the Spark base image, run the following command:
                                                                            docker pull swr.xxx/dli-public/spark_general-x86_64:3.3.1-2.3.7.1720240419835647952528832.202404250955
                                                                            +
                                                                            +

                                                                          3. Access SWR.

                                                                            1. Log in to the SWR management console.
                                                                            2. In the navigation pane on the left, choose Dashboard and click Generate Login Command in the upper right corner. On the displayed page, click to copy the login command.
                                                                            3. Run the login command on the VM where the container engine is installed.
                                                                            +

                                                                          4. Create an organization. If an organization has been created, skip this step.

                                                                            1. Log in to the SWR management console.
                                                                            2. In the navigation pane on the left, choose Organization Management. On the displayed page, click Create Organization in the upper right corner.
                                                                            3. Enter the organization name and click OK.
                                                                            +

                                                                          5. Write a Dockerfile.

                                                                            vi Dockerfile

                                                                            +

                                                                            Pack TensorFlow into the image as follows:

                                                                            +
                                                                            ARG BASE_IMG=swr.xxx/dli-public/spark_general-x86_64:3.3.1-2.3.7.1720240419835647952528832.202404250955//Replace xxx with the URL of the base image.
+
+FROM ${BASE_IMG} as builder
+USER omm //Run this command as user omm.
+RUN set -ex && \
+    mkdir -p /home/omm/.pip && \
+    pip3 install tensorflow==2.4.0
+Copy the content to the base image.
+USER omm
                                                                            +

                                                                            The following steps are included:

                                                                            +
                                                                            1. Set the available repository address of pip.
                                                                            2. Use pip3 to install the TensorFlow algorithm package.
                                                                            3. Copy the content in the temporary image builder where the algorithm package is installed to the base image (this step is to reduce the image size) to generate the final custom image.
                                                                            +

                                                                          6. Use Dockerfile to generate a custom image.

                                                                            Format of the image packaging command:
                                                                            docker build -t [Custom organization name]/[Custom image name]: [Image version] --build-arg BASE_IMG= [DLI base image path] -f Dockerfile .
                                                                            +
                                                                            +

                                                                            The DLI base image path is the image path in Obtaining DLI Base Images.

                                                                            +
                                                                            The following is an example:
                                                                            docker build -t mydli/spark:2.4 --build-arg BASE_IMG=swr.xxx/dli-public/spark_general-x86_64:3.3.1-2.3.7.1720240419835647952528832.202404250955 -f Dockerfile .
                                                                            +
                                                                            +

                                                                          7. Add a tag to the custom image.

                                                                            docker tag [Organization name]/[Image name]:[Image version][Image repository address]/[Organization name]/[Image name:version] in 6

                                                                            +
                                                                            The following is an example:
                                                                            docker tag mydli/spark:2.4 swr.xxx/testdli0617/spark:2.4.5.tensorflow
                                                                            +
                                                                            +

                                                                          8. Upload the custom image.

                                                                            docker push [Image repository address]/[Organization name]/[Image name:Version]

                                                                            +

                                                                            Set [Image repository address]/[Organization name]/[Image name:Version] the same as those in 7.

                                                                            +
                                                                            The following is an example:
                                                                            docker push swr.xxx/testdli0617/spark:2.4.5.tensorflow
                                                                            +
                                                                            +

                                                                          9. When submitting a Spark or Flink JAR job in DLI, select a custom image.

                                                                            • Open the Spark job or Flink job editing page on the management console, select the uploaded and shared image from the custom image list, and run the job.

                                                                              If you select a non- shared image, the system displays a message indicating that the image is not authorized. You can use the image only after it is authorized.

                                                                              +

                                                                              +
                                                                            • Specify the image parameter in job parameters on API to use a custom image to run a job.
                                                                            +

                                                                          +
                                                                          +
                                                                          +
                                                                          +
                                                                          +
                                                                          Parent topic: Common DLI Management Operations
                                                                          +
                                                                          +
                                                                          + diff --git a/docs/dli/umn/dli_01_0498.html b/docs/dli/umn/dli_01_0498.html index 6bd0eaabd..2edbd2e77 100644 --- a/docs/dli/umn/dli_01_0498.html +++ b/docs/dli/umn/dli_01_0498.html @@ -1,15 +1,15 @@ -

                                                                          (Recommended) Creating a Flink OpenSource SQL Job

                                                                          +

                                                                          Creating a Flink OpenSource SQL Job

                                                                          This section describes how to create a Flink OpenSource SQL job.

                                                                          -

                                                                          DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink 1.10 and 1.12 provided by the community. In addition, Redis, GaussDB(DWS), and DIS data source types are added based on the community connector. For details about the syntax and restrictions of Flink SQL DDL, DML, and functions, see Table API & SQL.

                                                                          -

                                                                          Prerequisites

                                                                          • You have prepared the data input and data output channels. For details, see Preparing Flink Job Data.
                                                                          • Before creating a Flink OpenSource SQL job to access other external data sources, such as OpenTSDB, HBase, Kafka, GaussDB(DWS), RDS, CSS, CloudTable, DCS Redis, and DDS, you need to create a datasource connection to connect the job running queue to the external data source.
                                                                            • For details about the external data sources that can be accessed by Flink jobs, see Cross-Source Analysis Development Methods.
                                                                            • For details about how to create a datasource connection, see Enhanced Datasource Connections.

                                                                              On the Resources > Queue Management page, locate the queue you have created, and choose More > Test Address Connectivity in the Operation column to check whether the network connection between the queue and the data source is normal. For details, see Testing Address Connectivity.

                                                                              +

                                                                              DLI Flink OpenSource SQL jobs are fully compatible with the syntax of Flink provided by the community. In addition, Redis and GaussDB(DWS) data source types are added based on the community connector. For the syntax and constraints of Flink SQL DDL, DML, and functions, see Table API & SQL.

                                                                              +

                                                                              Prerequisites

                                                                              Creating a Flink OpenSource SQL Job

                                                                              1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                                              2. In the upper right corner of the Flink Jobs page, click Create Job.

                                                                                -

                                                                              3. Specify job parameters.

                                                                                -

                                                                                Table 1 Job configuration information

                                                                                Parameter

                                                                                +

                                                                              4. Set job parameters.

                                                                                +

                                                                                @@ -22,30 +22,30 @@ - - - - @@ -53,8 +53,10 @@
                                                                                Table 1 Job parameters

                                                                                Parameter

                                                                                Description

                                                                                Name

                                                                                Name of a job. Enter 1 to 57 characters. Only letters, digits, hyphens (-), and underscores (_) are allowed.

                                                                                +

                                                                                Job name. Enter 1 to 57 characters. Only letters, numbers, hyphens (-), and underscores (_) are allowed.

                                                                                NOTE:

                                                                                The job name must be globally unique.

                                                                                Description

                                                                                Description of a job. It can be up to 512 characters long.

                                                                                +

                                                                                Description of a job. It can contain a maximum of 512 characters.

                                                                                Template Name

                                                                                You can select a sample template or a custom job template. For details about templates, see Managing Flink Templates.

                                                                                +

                                                                                You can select a sample template or a custom job template. For details about templates, see Managing Flink Job Templates.

                                                                                Tag

                                                                                +

                                                                                Tags

                                                                                Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                                                                NOTE:
                                                                                • A maximum of 20 tags can be added.
                                                                                • Only one tag value can be added to a tag key.
                                                                                • The key name in each resource must be unique.
                                                                                -
                                                                                • Tag key: Enter a tag key name in the text box.
                                                                                  NOTE:

                                                                                  A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                  +
                                                                                  • Tag key: Enter a tag key name in the text box.
                                                                                    NOTE:

                                                                                    A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                  -
                                                                                  • Tag value: Enter a tag value in the text box.
                                                                                    NOTE:

                                                                                    A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                                                                    +
                                                                                    • Tag value: Enter a tag value in the text box.
                                                                                      NOTE:

                                                                                      A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                                                                                -

                                                                              5. Click OK to enter the editing page.
                                                                              6. Edit an OpenSource SQL job.

                                                                                Enter details SQL statements in the SQL statement edit area. For details about SQL statements, see the Data Lake Insight Flink OpenSource SQL Syntax Reference.

                                                                                +

                                                                              7. Click OK to enter the editing page.
                                                                              8. Edit an OpenSource SQL job.

                                                                                Enter detailed SQL statements in the statement editing area. For details about SQL statements, see the Data Lake Insight Flink OpenSource SQL Syntax Reference.

                                                                              9. Click Check Semantics.

                                                                                • You can Start a job only after the semantic verification is successful.
                                                                                • If verification is successful, the message "The SQL semantic verification is complete. No error." will be displayed.
                                                                                • If verification fails, a red "X" mark will be displayed in front of each SQL statement that produced an error. You can move the cursor to the "X" mark to view error details and change the SQL statement as prompted.
                                                                                +

                                                                                Flink 1.15 does not support syntax verification.

                                                                                +

                                                                              10. Set job running parameters.

                                                                                @@ -64,18 +66,30 @@ - + + + + + + - - - - - - - - -
                                                                                Table 2 Running parameters

                                                                                Parameter

                                                                                Queue

                                                                                A shared queue is selected by default. Select a CCE queue with dedicated resources and configure the following parameters:

                                                                                -

                                                                                UDF Jar: You can customize a UDF Jar file. Before selecting a JAR file to be inserted, upload the corresponding JAR file to the OBS bucket and choose Data Management > Package Management to create a package. For details, see Creating a Package.

                                                                                -

                                                                                In SQL, you can call a user-defined function that is inserted into a JAR file.

                                                                                -
                                                                                NOTE:

                                                                                During job creation, a sub-user can only select a queue that has been allocated to the user.

                                                                                -

                                                                                If the remaining capacity of the selected queue cannot meet the job requirements, the system automatically scales up the capacity and you will be billed based on the increased capacity. When a queue is idle, the system automatically scales in the queue.

                                                                                -
                                                                                +

                                                                                Select a queue to run the job.

                                                                                +

                                                                                UDF Jar

                                                                                +

                                                                                UDF JAR file, which contains UDFs that can be called in subsequent jobs.

                                                                                +

                                                                                There are the following ways to manage UDF JAR files:

                                                                                +
                                                                                • Upload packages to OBS: Upload Jar packages to an OBS bucket in advance and select the corresponding OBS path.
                                                                                • Upload packages to DLI: Upload JAR files to an OBS bucket in advance and create a package on the Data Management > Package Management page of the DLI management console. For details, see Creating a DLI Package.
                                                                                +

                                                                                For Flink 1.15 or later, only OBS packages can be selected when creating jobs, and DLI packages are not supported.

                                                                                +

                                                                                Agency

                                                                                +

                                                                                If you choose Flink 1.15 to execute your job, you can create a custom agency to allow DLI to access other services.

                                                                                CUs

                                                                                Sum of the number of compute units and job manager CUs of DLI. One CU equals 1 vCPU and 4 GB.

                                                                                The value is the number of CUs required for job running and cannot exceed the number of CUs in the bound queue.

                                                                                +
                                                                                NOTE:

                                                                                When Task Manager Config is selected, elastic resource pool queue management is optimized by automatically adjusting CUs to match Actual CUs after setting Slot(s) per TM.

                                                                                +

                                                                                CUs = Actual number of CUs = max[Job Manager CPUs + Task Manager CPU, (Job Manager Memory + Task Manager Memory/4)]

                                                                                +
                                                                                • Job Manager CPUs + Task Manager CPUs = Actual TMs x CU(s) per TM + Job Manager CUs.
                                                                                • Job Manager Memory + Task Manager Memory = Actual TMs x Memory per TM + Job Manager Memory
                                                                                • If Slot(s) per TM is set, then: Actual TMs = Parallelism/Slot(s) per TM.
                                                                                • If Slot(s) per TM is not set, then: Actual TMs = (CUs – Job Manager CUs)/CU(s) per TM.
                                                                                • If Memory per TM and Job Manager Memory in the optimization parameters are not set, then: Memory per TM = CU(s) per TM x 4. Job Manager Memory = Job Manager CUs x 4.
                                                                                • The parallelism degree of Spark resources is jointly determined by the number of Executors and the number of Executor CPU cores.
                                                                                +

                                                                                Job Manager CUs

                                                                                @@ -85,87 +99,95 @@

                                                                                Parallelism

                                                                                Number of Flink OpenSource SQL jobs that run at the same time

                                                                                +

                                                                                Number of tasks concurrently executed by each operator in a job.

                                                                                NOTE:

                                                                                This value cannot be greater than four times the compute units (number of CUs minus the number of job manager CUs).

                                                                                Task Manager Configuration

                                                                                +

                                                                                Task Manager Config

                                                                                Whether to set Task Manager resource parameters.

                                                                                -

                                                                                If this option is selected, you need to set the following parameters:

                                                                                -
                                                                                • CU(s) per TM: Number of resources occupied by each Task Manager.
                                                                                • Slot(s) per TM: Number of slots contained in each Task Manager.
                                                                                +

                                                                                Whether Task Manager resource parameters are set

                                                                                +
                                                                                • If selected, you need to set the following parameters:
                                                                                  • CU(s) per TM: Number of resources occupied by each Task Manager.
                                                                                  • Slot(s) per TM: Number of slots contained in each Task Manager.
                                                                                  +
                                                                                +
                                                                                • If not selected, the system automatically uses the default values.
                                                                                  • CU(s) per TM: The default value is 1.
                                                                                  • Slot(s) per TM: The default value is (Parallelism x CU(s) per TM)/(CUs – Job Manager CUs).
                                                                                  +

                                                                                OBS Bucket

                                                                                OBS bucket to store job logs and checkpoint information. If the selected OBS bucket is not authorized, click Authorize.

                                                                                +

                                                                                OBS bucket to store job logs and checkpoint information. If the OBS bucket you selected is unauthorized, click Authorize.

                                                                                Save Job Log

                                                                                Whether to save the job running logs to OBS. The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID.

                                                                                +

                                                                                Whether job running logs are saved to OBS. The logs are saved in the following path: Bucket name/jobs/logs/Directory starting with the job ID.

                                                                                CAUTION:

                                                                                You are advised to configure this parameter. Otherwise, no run log is generated after the job is executed. If the job fails, the run log cannot be obtained for fault locating.

                                                                                If this option is selected, you need to set the following parameters:

                                                                                -
                                                                                OBS Bucket: Select an OBS bucket to store user job logs. If the selected OBS bucket is not authorized, click Authorize.
                                                                                NOTE:

                                                                                If both Enable Checkpointing and Save Job Log are selected, you only need to authorize OBS once.

                                                                                +
                                                                                OBS Bucket: Select an OBS bucket to store job logs. If the OBS bucket you selected is unauthorized, click Authorize.
                                                                                NOTE:

                                                                                If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                                                                                Alarm Generation upon Job Exception

                                                                                +

                                                                                Alarm on Job Exception

                                                                                Whether to report job exceptions, for example, abnormal job running or exceptions due to an insufficient balance, to users via SMS or email.

                                                                                +

                                                                                Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                                                                                If this option is selected, you need to set the following parameters:

                                                                                SMN Topic

                                                                                -

                                                                                Select a user-defined SMN topic. For details about how to customize SMN topics, see Creating a Topic in the Simple Message Notification User Guide.

                                                                                +

                                                                                Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                                                                                Enable Checkpointing

                                                                                Whether to enable job snapshots. If this function is enabled, jobs can be restored based on the checkpoints.

                                                                                -
                                                                                If this option is selected, you need to set the following parameters:
                                                                                • Checkpoint Interval (s) indicates the interval for creating checkpoints. The value ranges from 1 to 999999, and the default value is 30.
                                                                                • Checkpoint Mode can be set to either of the following values:
                                                                                  • At least once: Events are processed at least once.
                                                                                  • Exactly once: Events are processed only once.
                                                                                  +
                                                                                  If this option is selected, you need to set the following parameters:
                                                                                  • Checkpoint Interval: interval for creating checkpoints, in seconds. The value ranges from 1 to 999999, and the default value is 30.
                                                                                  • Checkpoint Mode can be set to either of the following values:
                                                                                    • At least once: Events are processed at least once.
                                                                                    • Exactly once: Events are processed only once.
                                                                                  -
                                                                                  • OBS Bucket: Select an OBS bucket to store your checkpoints. If the selected OBS bucket is not authorized, click Authorize.
                                                                                    The checkpoint path is Bucket name/jobs/checkpoint/Directory starting with the job ID.
                                                                                    NOTE:

                                                                                    If both Enable Checkpointing and Save Job Log are selected, you only need to authorize OBS once.

                                                                                    -
                                                                                    -
                                                                                  +

                                                                                  If you select Enable Checkpointing, you also need to set OBS Bucket.

                                                                                  +

                                                                                  OBS Bucket: Select an OBS bucket to store your checkpoints. If the OBS bucket you selected is unauthorized, click Authorize.

                                                                                  +
                                                                                  The checkpoint path is Bucket name/jobs/checkpoint/Directory starting with the job ID.
                                                                                  NOTE:

                                                                                  If Enable Checkpointing and Save Job Log are both selected, you only need to authorize OBS once.

                                                                                  +

                                                                                Auto Restart upon Exception

                                                                                Whether to enable automatic restart. If this function is enabled, any job that has become abnormal will be automatically restarted.

                                                                                +

                                                                                Whether automatic restart is enabled. If enabled, jobs will be automatically restarted and restored when exceptions occur.

                                                                                If this option is selected, you need to set the following parameters:

                                                                                -
                                                                                • Max. Retry Attempts: maximum number of retry times upon an exception. The unit is Times/hour.
                                                                                  • Unlimited: The number of retries is unlimited.
                                                                                  • Limited: The number of retries is user-defined.
                                                                                  +
                                                                                  • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                                                                                    • Unlimited: The number of retries is unlimited.
                                                                                    • Limited: The number of retries is user-defined.
                                                                                  • Restore Job from Checkpoint: This parameter is available only when Enable Checkpointing is selected.

                                                                                Idle State Retention Time

                                                                                How long the state of a key is retained without being updated before it is removed in GroupBy or Window. The default value is 1 hour.

                                                                                +

                                                                                Clears intermediate states of operators such as GroupBy, RegularJoin, Rank, and Depulicate that have not been updated after the maximum retention time. The default value is 1 hour.

                                                                                Dirty Data Policy

                                                                                Policy for processing dirty data. The following policies are supported: Ignore, Trigger a job exception, and Save.

                                                                                If you set this field to Save, the Dirty Data Dump Address must be set. Click the address box to select the OBS path for storing dirty data.

                                                                                +

                                                                                This parameter is available only when a DIS data source is used.
                                                                                -

                                                                              11. (Optional) Set the runtime configuration as required.
                                                                              12. Click Save.
                                                                              13. Click Start. On the displayed Start Flink Jobs page, confirm the job specifications, and click Start Now to start the job.

                                                                                After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column. After a job is successfully submitted, the job status will change from Submitting to Running. After the execution is complete, the message Completed is displayed.

                                                                                -

                                                                                If the job status is Submission failed or Running exception, the job submission failed or the job did not execute successfully. In this case, you can move the cursor over the status icon in the Status column of the job list to view the error details. You can click to copy these details. After handling the fault based on the provided information, resubmit the job.

                                                                                +

                                                                              14. (Optional) Set the runtime configuration as required.

                                                                                +

                                                                              15. Click Save.
                                                                              16. Click Start. On the displayed Start Flink Jobs page, confirm the job specifications, and click Start Now to start the job.

                                                                                After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column. Once a job is successfully submitted, its status changes from Submitting to Running. After the execution is complete, the status changes to Completed.

                                                                                +

                                                                                If the job status is Submission failed or Running exception, the job fails to submit or run. In this case, you can hover over the status icon in the Status column of the job list to view the error details. You can click to copy these details. Rectify the fault based on the error information and resubmit the job.

                                                                                Other buttons are as follows:

                                                                                • Save As: Save the created job as a new job.
                                                                                • Static Stream Graph: Provide the static concurrency estimation function and stream graph display function.
                                                                                • Simplified Stream Graph: Display the data processing flow from the source to the sink.
                                                                                • Format: Format the SQL statements in the editing box.
                                                                                • Set as Template: Set the created SQL statements as a job template.
                                                                                • Theme Settings: Set the theme related parameters, including Font Size, Wrap, and Page Style.

                                                                                17. Simplified Stream Graph

                                                                                On the OpenSource SQL job editing page, click Simplified Stream Graph.

                                                                                +

                                                                                Simplified stream graph viewing is only supported in Flink 1.12 and Flink 1.10.

                                                                                +
                                                                                +

                                                                                Static Stream Graph

                                                                                On the OpenSource SQL job editing page, click Static Stream Graph.

                                                                                -

                                                                                If you use a UDF in a Flink OpenSource SQL job, it is not possible to generate a static stream graph.

                                                                                +
                                                                                • Simplified stream graph viewing is only supported in Flink 1.12 and Flink 1.10.
                                                                                • If you use a UDF in a Flink OpenSource SQL job, it is not possible to generate a static stream graph.

                                                                                The Static Stream Graph page also allows you to:

                                                                                • Estimate concurrencies. Click Estimate Concurrencies on the Static Stream Graph page to estimate concurrencies. Click Restore Initial Value to restore the initial value after concurrency estimation.
                                                                                • Zoom in or out the page.
                                                                                • Expand or merge operator chains.
                                                                                • You can edit Parallelism, Output rate, and Rate factor.
                                                                                  • Parallelism: indicates the number of concurrent tasks.
                                                                                  • Output rate: indicates the data traffic of an operator. The unit is piece/s.
                                                                                  • Rate factor: indicates the retention rate after data is processed by operators. Rate factor = Data output volume of an operator/Data input volume of the operator (Unit: %)
                                                                                  @@ -174,7 +196,7 @@
                                                                                diff --git a/docs/dli/umn/dli_01_0504.html b/docs/dli/umn/dli_01_0504.html index 2f441848e..2224a7e56 100644 --- a/docs/dli/umn/dli_01_0504.html +++ b/docs/dli/umn/dli_01_0504.html @@ -1,118 +1,111 @@ -

                                                                                Overview

                                                                                -

                                                                                Elastic Resource Pool

                                                                                An elastic resource pool provides compute resources (CPU and memory) for running DLI jobs. The unit is CU. One CU contains one CPU and 4 GB memory.

                                                                                -

                                                                                You can create multiple queues in an elastic resource pool. Compute resources can be shared among queues. You can properly set the resource pool allocation policy for queues to improve compute resource utilization.

                                                                                +

                                                                                Overview of DLI Elastic Resource Pools and Queues

                                                                                +

                                                                                DLI compute resources are the foundation to run jobs. This section describes the modes of DLI compute resources and queue types.

                                                                                +

                                                                                What Are Elastic Resource Pools and Queues?

                                                                                Before we dive into the compute resource modes of DLI, let us first understand the basic concepts of elastic resource pools and queues.

                                                                                +
                                                                                • An elastic resource pool is a pooled management mode for DLI compute resources, which can be seen as a collection of DLI compute resources. DLI supports the creation of multiple queues within an elastic resource pool, and these queues can share the resources in the elastic resource pool.
                                                                                • Queues are the basic units of compute resources that are actually used and allocated in DLI. You can create different queues for different jobs or data processing tasks, and allocate and adjust resources for these queues as needed. To learn more about the queue types in DLI, refer to DLI Queue Types.
                                                                                -

                                                                                Specifications

                                                                                DLI offers compute resources in the specifications listed in Table 1.

                                                                                +

                                                                                DLI elastic resource pools are physically isolated, while queues within the same elastic resource pool are logically isolated.

                                                                                +

                                                                                You are advised to create separate elastic resource pools for testing and production scenarios to ensure the independence and security of resource management through physical isolation.

                                                                                +
                                                                                +

                                                                                DLI Compute Resource Modes

                                                                                DLI offers three compute resource management modes, each with unique advantages and use cases.

                                                                                +
                                                                                Figure 1 DLI compute resource modes
                                                                                +
                                                                                • Elastic resource pool mode: a pooled management mode for compute resources that provides dynamic scaling capabilities. Queues within the same elastic resource pool share compute resources. By setting up a reasonable compute resource allocation policy for queues, you can improve compute resource utilization and meet resource demands during peak hours.
                                                                                  • Use cases: suitable for scenarios with significant fluctuations in business volume, such as periodic data batch processing tasks or real-time data processing needs.
                                                                                  • Supported queue types: for SQL (Spark), for SQL (HetuEngine), and for general purpose. To learn more about the queue types in DLI, refer to DLI Queue Types.

                                                                                    General-purpose queues and SQL queues in elastic resource pool mode do not support cross-AZ deployment.

                                                                                    +
                                                                                    +
                                                                                  • Usage: first create an elastic resource pool, then create queues within the pool and allocate compute resources. Queues are associated with specific jobs and data processing tasks.

                                                                                    For how to buy an elastic resource pool and create queues within it, see Creating an Elastic Resource Pool and Creating Queues Within It.

                                                                                    +
                                                                                  +
                                                                                • Global sharing mode:

                                                                                  Global sharing mode is a compute resource allocation mode that allocates resources based on the actual amount of data scanned in SQL queries. It does not support specifying or reserving compute resources.

                                                                                  +

                                                                                  The default queue, which is preset by DLI, is the compute resource for global sharing mode, and the resource size is allocated on demand. Users who are unsure of the data size or occasionally need to process data can use the default queue to run jobs.

                                                                                  +
                                                                                  • Use cases: suitable for testing jobs or scenarios with low resource consumption.
                                                                                  • Supported queue types: Only the preset default queue in DLI is the compute resource for global sharing mode.

                                                                                    The default queue is typically used by users who are new to DLI but it may lead to resource contention and prevent you from getting the resources you need for your jobs, as its resources are shared among all users. You are advised to use self-built queues to run production jobs.

                                                                                    +
                                                                                  • Usage: The default queue is only applicable to submitting SQL jobs. When submitting SQL jobs on the DLI management console, select the default queue.
                                                                                  +
                                                                                • Non-elastic resource pool mode (discarded and not recommended):

                                                                                  The previous-gen of DLI's compute resource management mode is no longer recommended due to its lack of flexibility.

                                                                                  +

                                                                                  Non-elastic resource pool mode provides fixed-specification compute resources that are purchased and exclusively used, and cannot be dynamically adjusted according to demand, which may result in resource waste or insufficient resources during peak hours.

                                                                                  +
                                                                                -
                                                                                Table 1 Elastic resource pool specifications

                                                                                Edition

                                                                                +
                                                                                - - - - - - - - - - - + + + + +
                                                                                Table 1 DLI compute resource modes and supported queue types

                                                                                DLI Compute Resource Mode

                                                                                Specification

                                                                                +

                                                                                Supported Queue Type

                                                                                Constraint

                                                                                +

                                                                                Resource Characteristic

                                                                                Scenario

                                                                                +

                                                                                Use Case

                                                                                Basic

                                                                                +

                                                                                Elastic resource pool mode

                                                                                16–64 CUs

                                                                                +

                                                                                For SQL (Spark)

                                                                                +

                                                                                For SQL (HetuEngine)

                                                                                +

                                                                                For general purpose

                                                                                • High reliability and availability are not supported.
                                                                                • Queue properties and job priorities cannot be set.
                                                                                • Notebook instances cannot be interconnected with.
                                                                                -

                                                                                For more notes and constraints on elastic resource pools, see Notes and Constraints.

                                                                                +

                                                                                Resources are shared among multiple queues for a single user.

                                                                                +

                                                                                Resources are dynamically allocated and can be flexibly adjusted.

                                                                                This edition is suitable for testing scenarios with low resource consumption and low requirements for resource reliability and availability.

                                                                                +

                                                                                Suitable for scenarios with significant fluctuations in business demand, where resources need to be flexibly adjusted to meet peak and off-peak demands.

                                                                                Standard

                                                                                +

                                                                                Global sharing mode

                                                                                64 CUs or higher

                                                                                +

                                                                                default queue

                                                                                For more notes and constraints on elastic resource pools, see Notes and Constraints.

                                                                                +

                                                                                Resources are shared among multiple queues for multiple users.

                                                                                +

                                                                                You are billed on a pay-per-use basis. Resources cannot be reserved.

                                                                                This edition offers powerful computing capabilities, high availability, and flexible resource management. It is suitable for large-scale computing tasks and business scenarios with long-term resource planning needs.

                                                                                +

                                                                                Suitable for temporary or testing projects where data size is uncertain or data processing is only required occasionally.

                                                                                +

                                                                                Non-elastic resource pool mode

                                                                                +

                                                                                (discarded, not recommended)

                                                                                +

                                                                                For SQL

                                                                                +

                                                                                For general purpose

                                                                                +

                                                                                Resources are exclusively used by a single user and a single queue.

                                                                                +

                                                                                Resources cannot be dynamically adjusted and may remain idle.

                                                                                +

                                                                                Discarded and not recommended.
                                                                                -

                                                                                Notes and Constraints

                                                                                • The region of an elastic resource pool cannot be changed.
                                                                                • Jobs of Flink 1.10 or later can run in elastic resource pools.
                                                                                • The network segment of an elastic resource pool cannot be changed after being set.
                                                                                • You can only view the scaling history of resource pools in the last 30 days.
                                                                                • Elastic resource pools cannot access the Internet.

                                                                                  -

                                                                                  -
                                                                                • Changes to elastic resource pool CUs can occur when setting the CU, adding or deleting queues in an elastic resource pool, or modifying the scaling policies of queues in an elastic resource pool, or when the system automatically triggers elastic resource pool scaling. However, in some cases, the system cannot guarantee that the scaling will reach the target CUs as planned.
                                                                                  • If there are not enough physical resources, an elastic resource pool may not be able to scale out to the desired target size.
                                                                                  • The system does not guarantee that an elastic resource pool will be scaled in to the desired target size.

                                                                                    The system checks the resource usage before scaling in the elastic resource pool to determine if there is enough space for scaling in. If the existing resources cannot be scaled in according to the minimum scaling step, the pool may not be scaled in successfully or only partially.

                                                                                    -

                                                                                    The scaling step may vary depending on the resource specifications, usually 16 CUs, 32 CUs, 48 CUs, 64 CUs, etc.

                                                                                    -

                                                                                    For example, if the elastic resource pool has a capacity of 192 CUs and the queues in the pool are using 68 CUs due to running jobs, the plan is to scale in to 64 CUs.

                                                                                    -

                                                                                    When executing a scaling in task, the system determines that there are 124 CUs remaining and scales in by the minimum step of 64 CUs. However, the remaining 60 CUs cannot be scaled in any further. Therefore, after the elastic resource pool executes the scaling in task, its capacity is reduced to 128 CUs.

                                                                                    +

                                                                                    To help you understand the use cases for different DLI compute resource modes, we can compare purchasing DLI compute resources to using car services:

                                                                                    +
                                                                                    • The elastic resource pool mode can be compared to "renting a car" where you can dynamically adjust the scale of resources based on actual needs.

                                                                                      This mode is suitable for scenarios with significant fluctuations in business demand, allowing for flexible adjustment of resources based on peak and off-peak demands to optimize costs.

                                                                                      +
                                                                                    • The global sharing mode can be compared to "taking a taxi" where you only pay for the actual amount of data used.

                                                                                      This mode is suitable for scenarios where data size is uncertain or data processing is only required occasionally, allowing for on-demand use of resources without the need to pre-purchase or reserve resources.

                                                                                    +

                                                                                    Elastic Resource Pool Scaling

                                                                                    Creating or deleting queues within an elastic resource pool triggers elastic resource scaling.

                                                                                    +

                                                                                    Scaling in an elastic resource pool may affect nodes containing shuffle data, leading to the recomputation of Spark tasks. This causes automatic retries for Spark and SQL jobs, and if the retries exceed the limit, the job execution fails, requiring you to rerun the job.

                                                                                    +
                                                                                    • Spark 2.3 jobs need to be upgraded to a later Spark version to support dynamic scale-in of the jobs while they are running.
                                                                                    • Spark Streaming and Flink jobs cannot be scaled in while they are running. To perform a scale-in, suspend the jobs or migrate them to another elastic resource pool.
                                                                                    +
                                                                                    +
                                                                                    +

                                                                                    DLI Queue Types

                                                                                    DLI is divided into three queue types: default queue, for SQL, and for general purpose. You can choose the most suitable queue type based on your business scenario and job characteristics.

                                                                                    +
                                                                                    • default queue:

                                                                                      The default queue is a preset queue that is shared among all users.

                                                                                      +

                                                                                      The default queue does not support specifying the size of resources and resources are allocated on-demand during job execution, with billing based on the actual amount of data scanned.

                                                                                      +

                                                                                      As resources of the default queue are shared among all users, there may be resource contention during use, and it cannot be guaranteed that resources will be available for every job execution.

                                                                                      +

                                                                                      The default queue is suitable for small-scale or temporary data processing needs. For important jobs or jobs that require guaranteed resources, you are advised to buy an elastic resource pool and create queues within it to execute jobs.

                                                                                      +
                                                                                    • For SQL:

                                                                                      For SQL queues are used to execute SQL jobs and supports specifying engine types including Spark and HetuEngine.

                                                                                      +

                                                                                      This type of queues is suitable for businesses that require fast data query and analysis, as well as regular cache clearing or environment resetting.

                                                                                      +
                                                                                    +
                                                                                    • For general purpose:

                                                                                      For general purpose queues are used to execute Spark jobs, Flink OpenSource SQL jobs, and Flink Jar jobs.

                                                                                      +

                                                                                      It is suitable for complex data processing, real-time data stream processing, or batch data processing scenarios.

                                                                                    -

                                                                                    Scenario

                                                                                    Resources too fixed to meet a range of requirements.

                                                                                    -

                                                                                    The quantities of compute resources required for jobs change in different time of a day. If the resources cannot be scaled based on service requirements, they may be wasted or insufficient. Figure 1 shows the resource usage during a day.

                                                                                    -
                                                                                    • After ETL jobs are complete, no other jobs are running during 04:00 to 07:00 in the early morning. The resources could be released at that time.
                                                                                    • From 09:00 to 12:00 a.m. and 02:00 to 04:00 p.m., a large number of ETL report and job queries are queuing for compute resources.
                                                                                      Figure 1 Fixed resources
                                                                                      +

                                                                                      Use Cases of Elastic Resource Pools

                                                                                      Queues in an elastic resource pool are recommended, as they offer the flexibility to use resources with high utilization as needed. This part describes common use cases of elastic resource pools.

                                                                                      +

                                                                                      Resources too fixed to meet a range of requirements.

                                                                                      +

                                                                                      The quantities of compute resources required for jobs change in different time of a day. If the resources cannot be scaled based on service requirements, they may be wasted or insufficient. Figure 2 shows the resource usage during a day.

                                                                                      +
                                                                                      • After ETL jobs are complete, no other jobs are running during 04:00 to 07:00 in the early morning. The resources could be released at that time.
                                                                                      • From 09:00 to 12:00 a.m. and 02:00 to 04:00 p.m., a large number of ETL report and job queries are queuing for compute resources.
                                                                                        Figure 2 Fixed resources

                                                                                      Resources are isolated and cannot be shared.

                                                                                      -
                                                                                      A company has two departments, and each run their jobs on a DLI queue. Department A is idle from 08:00 to 12:00 a.m. and has remaining resources, while department B has a large number of service requests during this period and needs more resources to meet the requirements. Since the resources are isolated and cannot be shared between department A and B, the idle resources are wasted.
                                                                                      Figure 2 Resource waste due to resource isolation
                                                                                      +
                                                                                      A company has two departments, and each run their jobs on a DLI queue. Department A is idle from 08:00 to 12:00 a.m. and has remaining resources, while department B has a large number of service requests during this period and needs more resources to meet the requirements. Since the resources are isolated and cannot be shared between department A and B, the idle resources are wasted.
                                                                                      Figure 3 Resource waste due to resource isolation

                                                                                      Elastic resource pools can be accessed by different queues and automatically scaled to improve resource utilization and handle resource peaks.

                                                                                      You can use elastic resource pools to centrally manage and allocate resources. Multiple queues can be bound to an elastic resource pool to share the pooled resources.

                                                                                      -

                                                                                      Architecture and Advantages

                                                                                      Elastic resource pools support the CCE cluster architecture for heterogeneous resources so you can centrally manage and allocate them.

                                                                                      -

                                                                                      Elastic resource pools have the following advantages:

                                                                                      -
                                                                                      • Unified management
                                                                                        • You can manage multiple internal clusters and schedule jobs. You can manage millions of cores for compute resources.
                                                                                        • Elastic resource pools can be deployed across multiple AZs to support high availability. (This function will be supported in later versions.)
                                                                                        -
                                                                                      • Tenant resource isolation

                                                                                        Resources of different queues are isolated to reduce the impact on each other.

                                                                                        -
                                                                                      • Shared access and flexibility
                                                                                        • Specifications can be scaled in seconds to help you handle request peaks.
                                                                                        • Queue priorities and CU quotas can be set at different time to improve resource utilization.
                                                                                        -
                                                                                      • Job-level isolation (supported in later versions)

                                                                                        SQL jobs can run on independent Spark instances, reducing mutual impacts between jobs.

                                                                                        -
                                                                                      • Automatic scaling (supported in later versions)

                                                                                        The queue quota is updated in real time based on workload and priority.

                                                                                        -
                                                                                      -

                                                                                      Using elastic resource pools has the following advantages.

                                                                                      - -
                                                                                      - - - - - - - - - - - - - - - - - - - - -

                                                                                      Advantage

                                                                                      -

                                                                                      No Elastic Resource Pool

                                                                                      -

                                                                                      Use Elastic Resource Pool

                                                                                      -

                                                                                      Scale-out duration

                                                                                      -

                                                                                      You will need to spend several minutes manually scaling out.

                                                                                      -

                                                                                      No manual intervention is required, as dynamic scale out can be done in seconds.

                                                                                      -

                                                                                      Resource utilization

                                                                                      -

                                                                                      Resources cannot be shared among different queues.

                                                                                      -

                                                                                      For example, if queue 1 has 10 unused CUs and queue 2 requires more resources due to heavy load, queue 2 cannot utilize the resources of queue 1. It has to be scaled up.

                                                                                      -

                                                                                      Queues added to the same elastic resource pool can share compute resources.

                                                                                      -

                                                                                      When you set a data source, you must allocate different network segments to each queue, which requires a large number of VPC network segments.

                                                                                      -

                                                                                      You can add multiple general-purpose queues in the same elastic resource pool to one network segment, simplifying the data source configuration.

                                                                                      -

                                                                                      Resource allocation

                                                                                      -

                                                                                      If resources are insufficient for scale-out tasks of multiple queues, some queues will fail to be scaled out.

                                                                                      -

                                                                                      You can set the priority for each queue in the elastic resource pool based on the peak hours to ensure proper resource allocation.

                                                                                      -
                                                                                      -
                                                                                      -

                                                                                      You can perform the following operations on elastic resource pools:

                                                                                      - -
                                                                                      diff --git a/docs/dli/umn/dli_01_0505.html b/docs/dli/umn/dli_01_0505.html index 464fb803d..1b58f9597 100644 --- a/docs/dli/umn/dli_01_0505.html +++ b/docs/dli/umn/dli_01_0505.html @@ -1,70 +1,116 @@ -

                                                                                      Creating an Elastic Resource Pool

                                                                                      -

                                                                                      For details about the application scenarios of elastic resource pools, see the Overview. This section describes how to create an elastic resource pool.

                                                                                      -

                                                                                      Precautions

                                                                                      • If you use an enhanced datasource connection, the network segment of the elastic resource pool cannot overlap that of the data source.
                                                                                      • The network segment of an elastic resource pool cannot be changed after being set.
                                                                                      -
                                                                                      -

                                                                                      Creating an Elastic Resource Pool

                                                                                      1. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                      2. On the Resource Pool page, click Buy Resource Pool in the upper right corner.
                                                                                      3. On the displayed page, set the following parameters:

                                                                                        -

                                                                                        Table 1 Parameters

                                                                                        Parameter

                                                                                        +

                                                                                        Creating an Elastic Resource Pool and Creating Queues Within It

                                                                                        +

                                                                                        An elastic resource pool offers compute resources (CPU and memory) required for running DLI jobs, which can adapt to the changing demands of services.

                                                                                        +

                                                                                        You can create multiple queues within an elastic resource pool. These queues are associated with specific jobs and data processing tasks, and serve as the basic unit for resource allocation and usage within the pool. This means queues are specific compute resources required for executing jobs.

                                                                                        +

                                                                                        Queues within an elastic resource pool can be shared to execute jobs. This is achieved by correctly setting the queue allocation policy. This enhances queue utilization. This section describes how to create an elastic resource pool and create queues within it.

                                                                                        +

                                                                                        DLI elastic resource pools are physically isolated, while queues within the same elastic resource pool are logically isolated.

                                                                                        +

                                                                                        You are advised to create separate elastic resource pools for testing and production scenarios to ensure the independence and security of resource management through physical isolation.

                                                                                        +
                                                                                        +

                                                                                        Notes and Constraints

                                                                                        +
                                                                                        - - - - - - - + + +
                                                                                        Table 1 Notes and constraints on elastic resource pools

                                                                                        Item

                                                                                        Description

                                                                                        +

                                                                                        Description

                                                                                        Region

                                                                                        +

                                                                                        Resource specifications

                                                                                        Select a region. Select a region near you to ensure the lowest latency possible.

                                                                                        +
                                                                                        • An elastic resource pool currently supports up to 32,000 CUs.

                                                                                        Project

                                                                                        +

                                                                                        Managing elastic resource pools

                                                                                        Each region corresponds to a project.

                                                                                        +
                                                                                        • You cannot change the region of an elastic resource pool once the pool is created.
                                                                                        • Flink 1.10 or later jobs can run in elastic resource pools.
                                                                                        • The CIDR block of an elastic resource pool cannot be changed once set.
                                                                                        • You can view only the scaling history of an elastic resource pool within 30 days.
                                                                                        • Elastic resource pools cannot directly access the Internet.

                                                                                        Name

                                                                                        +

                                                                                        Elastic resource pool scaling

                                                                                        Name of the elastic resource pool.

                                                                                        -
                                                                                        • The name can contain only digits, letters, and underscores (_), but cannot contain only digits or start with an underscore (_) or digit.
                                                                                        • The description can contain a maximum of 128 characters.
                                                                                        +
                                                                                        • Changes to elastic resource pool CUs can occur when setting the CU, adding or deleting queues in an elastic resource pool, or modifying the scaling policies of queues in an elastic resource pool, or when the system automatically triggers elastic resource pool scaling. However, in some cases, the system cannot guarantee that the scaling will reach the target CUs as planned.
                                                                                          • If there are not enough physical resources, an elastic resource pool may not be able to scale out to the desired target size.
                                                                                          • The system does not guarantee that an elastic resource pool will be scaled in to the desired target size.

                                                                                            The system checks the resource usage before scaling in the elastic resource pool to determine if there is enough space for scaling in. If the existing resources cannot be scaled in according to the minimum scaling step, the pool may not be scaled in successfully or only partially.

                                                                                            +

                                                                                            The scaling step may vary depending on the resource specifications, usually 16 CUs, 32 CUs, 48 CUs, 64 CUs, etc.

                                                                                            +

                                                                                            For example, if the elastic resource pool has a capacity of 192 CUs and the queues in the pool are using 68 CUs due to running jobs, the plan is to scale in to 64 CUs.

                                                                                            +

                                                                                            When executing a scaling in task, the system determines that there are 124 CUs remaining and scales in by the minimum step of 64 CUs. However, the remaining 60 CUs cannot be scaled in any further. Therefore, after the elastic resource pool executes the scaling in task, its capacity is reduced to 128 CUs.

                                                                                            +
                                                                                          +
                                                                                        +
                                                                                        +
                                                                                        +
                                                                                        +

                                                                                        Creating an Elastic Resource Pool

                                                                                        1. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                        2. On the displayed page, click Buy Resource Pool in the upper right corner.
                                                                                        3. On the displayed page, set the following parameters:

                                                                                          +

                                                                                          + + + + + + + + + + + - - - - - - + + + - - - - @@ -72,12 +118,108 @@
                                                                                          Table 2 Parameters

                                                                                          Parameter

                                                                                          +

                                                                                          Description

                                                                                          +

                                                                                          Region

                                                                                          +

                                                                                          Select a region. Select a region near you to ensure the lowest latency possible.

                                                                                          +

                                                                                          Project

                                                                                          +

                                                                                          Each region corresponds to a project.

                                                                                          +

                                                                                          Name

                                                                                          +

                                                                                          Name of the elastic resource pool.

                                                                                          +
                                                                                          • Only numbers, letters, and underscores (_) are allowed. The value cannot contain only numbers or start with an underscore (_) or number.
                                                                                          • The value can contain a maximum of 128 characters.
                                                                                          NOTE:

                                                                                          The elastic resource pool name is case-insensitive. Uppercase letters will be automatically converted to lowercase letters.

                                                                                          CU Range

                                                                                          +

                                                                                          Type

                                                                                          The maximum and minimum CUs allowed for the elastic resource pool.

                                                                                          +
                                                                                          • Basic edition: offers 16 CUs to 64 CUs
                                                                                            • This edition is suitable for testing scenarios with low resource consumption and low requirements for resource reliability and availability.
                                                                                            • High reliability and availability are not supported.
                                                                                            • Queue properties and job priorities cannot be set.
                                                                                            • Notebook instances cannot be interconnected with.
                                                                                            +
                                                                                          • Standard edition: offers at least 64 CUs

                                                                                            This edition offers powerful computing capabilities, high availability, and flexible resource management. It is suitable for large-scale computing tasks and business scenarios with long-term resource planning needs.

                                                                                            +

                                                                                          Description

                                                                                          +

                                                                                          CU Range

                                                                                          Description of the elastic resource pool

                                                                                          +

                                                                                          The maximum and minimum CUs allowed for the elastic resource pool.

                                                                                          +

                                                                                          CU settings are used to control the maximum and minimum CU ranges for elastic resource pools to avoid unlimited resource scaling.

                                                                                          +

                                                                                          In CU Range, set the minimum CUs on the left and the maximum CUs on the right.

                                                                                          +
                                                                                          • The total minimum CUs of all queues in an elastic resource pool must be no more than the minimum CUs of the pool.
                                                                                          • The maximum CUs of any queue in an elastic resource pool must be no more than the maximum CUs of the pool.
                                                                                          +

                                                                                          An elastic resource pool should at least ensure that all queues in it can run with the minimum CUs and should try to ensure that all queues in it can run with the maximum CUs.

                                                                                          +

                                                                                          The specifications (yearly/monthly CUs) of an elastic resource pool are equal to the minimum CUs allocated during creation. This means that when the elastic resource pool is first created, the actual CUs will be equal to the specifications, which is also the minimum CUs.

                                                                                          CIDR Block

                                                                                          +

                                                                                          Description

                                                                                          The CIDR block to which the elastic resource pool belongs. If you use an enhanced datasource connection, the CIDR block of the elastic resource pool cannot overlap that of the data source. The CIDR block of an elastic resource pool cannot be modified after being set.

                                                                                          +

                                                                                          Description of the elastic resource pool

                                                                                          +

                                                                                          CIDR Block

                                                                                          +

                                                                                          CIDR block the elastic resource pool belongs to. If you use an enhanced datasource connection, this CIDR block cannot overlap that of the data source. Once set, this CIDR block cannot be changed.

                                                                                          Recommended CIDR block:

                                                                                          -

                                                                                          10.0.0.0~10.255.0.0/16~19

                                                                                          -

                                                                                          172.16.0.0~172.31.0.0/16~19

                                                                                          -

                                                                                          192.168.0.0~192.168.0.0/16~19

                                                                                          +

                                                                                          10.0.0.0–10.255.0.0/16–19

                                                                                          +

                                                                                          172.16.0.0–172.31.0.0/16–19

                                                                                          +

                                                                                          192.168.0.0–192.168.0.0/16–19

                                                                                          Enterprise Project

                                                                                          +

                                                                                          Enterprise Project

                                                                                          If the created elastic resource pool belongs to an enterprise project, select the enterprise project.

                                                                                          -

                                                                                          An enterprise project facilitates project-level management and grouping of cloud resources and users.

                                                                                          +

                                                                                          If the created elastic resource pool belongs to an enterprise project, select the enterprise project.

                                                                                          +

                                                                                          Enterprise projects let you manage cloud resources and users by project.

                                                                                          NOTE:

                                                                                          This parameter is displayed only for users who have enabled the Enterprise Management Service.

                                                                                          Tags

                                                                                          +

                                                                                          Tags

                                                                                          Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                                                                          +

                                                                                          Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                                                                          NOTE:
                                                                                          • A maximum of 20 tags can be added.
                                                                                          • Only one tag value can be added to a tag key.
                                                                                          • The key name in each resource must be unique.
                                                                                          -
                                                                                          • Tag key: Enter a tag key name in the text box.
                                                                                            NOTE:

                                                                                            A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                            +
                                                                                            • Tag key: Enter a tag key name in the text box.
                                                                                              NOTE:

                                                                                              A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                            -
                                                                                            • Tag value: Enter a tag value in the text box.
                                                                                              NOTE:

                                                                                              A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                                                                              +
                                                                                              • Tag value: Enter a tag value in the text box.
                                                                                                NOTE:

                                                                                                A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                                                                                          -

                                                                                        4. Click Buy and confirm the configurations.
                                                                                        5. Click Pay. Wait until the status of the elastic resource pool changes to Available. The elastic resource pool is successfully created.
                                                                                        6. Refer to Creating an Elastic Resource Pool and Running a Job and Configuring Scaling Policies for Queues to perform subsequent operations as needed.
                                                                                        +

                                                                                      4. Click Buy and confirm the configurations.
                                                                                      5. Click Pay. Wait until the status of the elastic resource pool changes to Available. The elastic resource pool is successfully created.
                                                                                      6. Refer to Example Use Case: Creating an Elastic Resource Pool and Running Jobs and Example Use Case: Configuring Scaling Policies for Queues in an Elastic Resource Pool to perform subsequent operations as needed.
                                                                                        7. +
                                                                                        +

                                                                                        Creating Queues Within an Elastic Resource Pool

                                                                                        Create one or more queues within an elastic resource pool to run jobs. This section describes how to create a queue within an elastic resource pool.

                                                                                        +

                                                                                        Creating a queue within an elastic resource pool will trigger changes of elastic resource CUs.

                                                                                        +
                                                                                        1. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                        2. Locate the elastic resource pool in which you want to create queues and click Add Queue in the Operation column.
                                                                                        3. On the Add Queue page, set basic queue parameters based on the table below.

                                                                                          +

                                                                                          + + + + + + + + + + + + + + + + + + + + + + +
                                                                                          Table 3 Basic parameters for adding a queue

                                                                                          Parameter

                                                                                          +

                                                                                          Description

                                                                                          +

                                                                                          Name

                                                                                          +

                                                                                          Name of the queue to add

                                                                                          +

                                                                                          Type

                                                                                          +
                                                                                          • For SQL: The queue is used to run SQL jobs.
                                                                                          • For general purpose: The queue is used to run Spark and Flink jobs.
                                                                                          +

                                                                                          Engine

                                                                                          +

                                                                                          If Type is For SQL, the queue engine can be Spark or HetuEngine.

                                                                                          +

                                                                                          For a HetuEngine SQL queue, there must be at least 96 CUs.

                                                                                          +

                                                                                          Enterprise Project

                                                                                          +

                                                                                          Select the enterprise project the queue belongs to. Queues under different enterprise projects can be added to an elastic resource pool.

                                                                                          +

                                                                                          Enterprise projects let you manage cloud resources and users by project.

                                                                                          +
                                                                                          NOTE:

                                                                                          This parameter is available only for users who have subscribed to the Enterprise Management Service.

                                                                                          +
                                                                                          +

                                                                                          Description

                                                                                          +

                                                                                          Description about the queue.

                                                                                          +

                                                                                          Tags

                                                                                          +

                                                                                          Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                                                                          +

                                                                                          +
                                                                                          NOTE:
                                                                                          • A maximum of 20 tags can be added.
                                                                                          • Only one tag value can be added to a tag key.
                                                                                          • The key name in each resource must be unique.
                                                                                          +
                                                                                          +
                                                                                          • Tag key: Enter a tag key name in the text box.
                                                                                            NOTE:

                                                                                            A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                            +
                                                                                            +
                                                                                          +
                                                                                          • Tag value: Enter a tag value in the text box.
                                                                                            NOTE:

                                                                                            A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.

                                                                                            +
                                                                                            +
                                                                                          +
                                                                                          +
                                                                                          +

                                                                                        4. Click Next. On the displayed page, configure a scaling policy for the queue in the elastic resource pool.

                                                                                          Click Create to add a scaling policy with varying priority, period, minimum CUs, and maximum CUs. The parameters of each scaling policy are:

                                                                                          + +
                                                                                          + + + + + + + + + + + + + + + + +
                                                                                          Table 4 Scaling policy parameters

                                                                                          Parameter

                                                                                          +

                                                                                          Description

                                                                                          +

                                                                                          Priority

                                                                                          +

                                                                                          Priority of the scaling policy in the current elastic resource pool. A larger value indicates a higher priority. You can set a number ranging from 1 to 100.

                                                                                          +

                                                                                          Period

                                                                                          +

                                                                                          Time segment when the policy takes effect. It can be set only by hour. The start time is on the left, and the end time is on the right.

                                                                                          +
                                                                                          • The time range includes the start time but not the end time, that is, [start time, end time).

                                                                                            For example, if you set Period to 01 and 17, the scaling policy takes effect at 01:00 a.m. till 05:00 p.m.

                                                                                            +
                                                                                          +
                                                                                          • The periods of scaling policies with different priorities should not overlap.
                                                                                          +

                                                                                          Min CU

                                                                                          +

                                                                                          Minimum number of CUs allowed by the scaling policy.

                                                                                          +
                                                                                          • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                                                                                          • If the minimum CUs of the queue is less than 16 CUs, both Max. Spark Driver Instances and Max. Prestart Spark Driver Instances set in the queue properties do not apply. Refer to Setting Queue Properties.
                                                                                          +

                                                                                          For a HetuEngine SQL queue, there must be at least 96 CUs.

                                                                                          +

                                                                                          Max CU

                                                                                          +

                                                                                          Maximum number of CUs allowed by the scaling policy.

                                                                                          +

                                                                                          In any time segment of a day, the maximum CUs of any queue in an elastic resource pool cannot be more than the maximum CUs of the pool.

                                                                                          +
                                                                                          +
                                                                                          +
                                                                                          • The first scaling policy is the default policy, and its Period parameter configuration cannot be deleted or modified.
                                                                                          • Flink jobs cannot trigger automatic scaling of queues in an elastic resource pool.
                                                                                          +
                                                                                          +

                                                                                        5. Click OK. View all queues and scaling policies added to the elastic resource pool by referring to Adjusting Scaling Policies for Queues in an Elastic Resource Pool.
                                                                                        diff --git a/docs/dli/umn/dli_01_0506.html b/docs/dli/umn/dli_01_0506.html index 73af24159..ee547c2a7 100644 --- a/docs/dli/umn/dli_01_0506.html +++ b/docs/dli/umn/dli_01_0506.html @@ -1,39 +1,41 @@ -

                                                                                        Managing Queues

                                                                                        -

                                                                                        Multiple queues can be added to an elastic resource pool. For details about how to add a queue, see Adding a Queue. You can configure the number of CUs you want based on the compute resources used by DLI queues during peaks and troughs and set priorities for the scaling policies to ensure stable running of jobs.

                                                                                        -

                                                                                        Precautions

                                                                                        • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                                                                                        • In any time segment of a day, the maximum CUs of any queue in an elastic resource pool cannot be more than the maximum CUs of the pool.
                                                                                        • The periods of scaling policies cannot overlap.
                                                                                        • The period of a scaling policy can only be set by hour and specified by the start time and end time. For example, if you set the period to 00-09, the time range when the policy takes effect is [00:00, 09:00). The period of the default scaling policy cannot be modified.
                                                                                        • In any period, compute resources are preferentially allocated to meet the minimum number of CUs of all queues. The remaining CUs (total CUs of the elastic resource pool – total minimum CUs of all queues) are allocated in accordance with the scaling policy priorities.
                                                                                        • After the queue is scaled out, the system starts billing you for the added CUs. So, if you do not have sufficient requirements, scale in your queue to release unnecessary CUs to save cost. -
                                                                                          Table 1 CU allocation (without jobs)

                                                                                          Scenario

                                                                                          +

                                                                                          Adjusting Scaling Policies for Queues in an Elastic Resource Pool

                                                                                          +

                                                                                          Multiple queues can be added to an elastic resource pool. For how to add a queue, see Creating an Elastic Resource Pool and Creating Queues Within It. You can configure the number of CUs you want based on the compute resources used by DLI queues during peaks and troughs and set priorities for the scaling policies to ensure stable running of jobs.

                                                                                          +

                                                                                          Precautions

                                                                                          • You are advised to implement fine-grained management of resource pools for stream and batch processing jobs by placing Flink real-time stream jobs and SQL batch processing jobs in separate elastic resource pools.

                                                                                            Flink real-time stream jobs can run stably without forced scale-in, thus avoiding job interruption and system instability.

                                                                                            +

                                                                                            SQL batch processing jobs are placed in independent resource pools, which can scale out and in more flexibly, significantly enhancing the success rate and operational efficiency of scaling operations.

                                                                                            +
                                                                                          • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                                                                                          • In any time segment of a day, the maximum CUs of any queue in an elastic resource pool cannot be more than the maximum CUs of the pool.
                                                                                          • The periods of scaling policies cannot overlap.
                                                                                          • The period of a scaling policy can only be set by hour and specified by the start time and end time. For example, if you set the period to 00-09, the time range when the policy takes effect is [00:00, 09:00). The period of the default scaling policy cannot be modified.
                                                                                          • In any period, compute resources are preferentially allocated to meet the minimum number of CUs of all queues. The remaining CUs (total CUs of the elastic resource pool – total minimum CUs of all queues) are allocated in accordance with the scaling policy priorities.
                                                                                          • After the queue is scaled out, the system starts billing you for the added CUs. So, if you do not have sufficient requirements, scale in your queue to release unnecessary CUs to save cost. +
                                                                                            - - - - - - - - - @@ -41,48 +43,48 @@
                                                                                            Table 1 CU allocation (without jobs)

                                                                                            Scenario

                                                                                            CUs

                                                                                            +

                                                                                            CUs

                                                                                            An elastic resource pool has a maximum number of 256 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                            -
                                                                                            • Queue A: priority 5; period: 00:00–9:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: priority 10; time period: 00:00–9:00; minimum CU: 64; maximum CU: 128
                                                                                            +

                                                                                            An elastic resource pool has a maximum number of 256 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                            +
                                                                                            • Queue A: priority 5; period: 00:00–09:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: priority 10; time period: 00:00–09:00; minimum CU: 64; maximum CU: 128

                                                                                            From 00:00 a.m. to 09:00 a.m.:

                                                                                            -
                                                                                            1. The minimum CUs are allocated to the two queues. Queue A has 32 CUs, and queue B has 64 CUs. There are 160 CUs remaining.
                                                                                            2. The remaining CUs are allocated based on the priorities. Queue B is prior to queue A. Therefore, queue B gets 128 CUs, and queue A has 32 CUs.
                                                                                            +

                                                                                            From 00:00 a.m. to 09:00 a.m.:

                                                                                            +
                                                                                            1. The minimum CUs are allocated to the two queues. Queue A has 32 CUs, and queue B has 64 CUs. There are 160 CUs remaining.
                                                                                            2. The remaining CUs are allocated based on priority. Since queue B has a higher priority than queue A, 64 CUs will be allocated to queue B first, followed by the allocation of 32 CUs to queue A.

                                                                                            An elastic resource pool has a maximum number of 96 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                            -
                                                                                            • Queue A: priority 5; period: 00:00–9:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: priority 10; time period: 00:00–9:00; minimum CU: 64; maximum CU: 128
                                                                                            +

                                                                                            An elastic resource pool has a maximum number of 96 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                            +
                                                                                            • Queue A: priority 5; period: 00:00–09:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: priority 10; time period: 00:00–09:00; minimum CU: 64; maximum CU: 128

                                                                                            From 00:00 a.m. to 09:00 a.m.:

                                                                                            +

                                                                                            From 00:00 a.m. to 09:00 a.m.:

                                                                                            1. The minimum CUs are allocated to the two queues. Queue A has 32 CUs, and queue B has 64 CUs. There are no remaining CUs.
                                                                                            2. The allocation is complete.

                                                                                            An elastic resource pool has a maximum number of 128 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                            -
                                                                                            • Queue A: priority 5; period: 00:00–9:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: priority 10; time period: 00:00–9:00; minimum CU: 64; maximum CU: 128
                                                                                            +

                                                                                            An elastic resource pool has a maximum number of 128 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                            +
                                                                                            • Queue A: priority 5; period: 00:00–09:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: priority 10; time period: 00:00–09:00; minimum CU: 64; maximum CU: 128

                                                                                            From 00:00 a.m. to 09:00 a.m.:

                                                                                            +

                                                                                            From 00:00 a.m. to 09:00 a.m.:

                                                                                            1. The minimum CUs are allocated to the two queues. Queue A has 32 CUs, and queue B has 64 CUs. There are 32 CUs remaining.
                                                                                            2. The remaining 32 CUs are preferentially allocated to queue B.

                                                                                            An elastic resource pool has a maximum number of 128 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                            -
                                                                                            • Queue A: priority 5; period: 00:00–9:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: priority 5; time period: 00:00–9:00; minimum CU: 64; maximum CU: 128
                                                                                            +

                                                                                            An elastic resource pool has a maximum number of 128 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                            +
                                                                                            • Queue A: priority 5; period: 00:00–09:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: priority 5; time period: 00:00–09:00; minimum CU: 64; maximum CU: 128

                                                                                            From 00:00 a.m. to 09:00 a.m.:

                                                                                            +

                                                                                            From 00:00 a.m. to 09:00 a.m.:

                                                                                            1. The minimum CUs are allocated to the two queues. Queue A has 32 CUs, and queue B has 64 CUs. There are 32 CUs remaining.
                                                                                            2. The two queues have the same priority, the remaining 32 CUs are randomly allocated to the two queues.
                                                                                            -
                                                                                            Table 2 CU allocation (with jobs)

                                                                                            Scenario

                                                                                            +
                                                                                            - - - - - - - - - - - - - - - - - @@ -100,7 +102,7 @@ -
                                                                                            Table 2 CU allocation (with jobs)

                                                                                            Scenario

                                                                                            Actual CUs of Elastic Resource Pool

                                                                                            +

                                                                                            Actual CUs of Elastic Resource Pool

                                                                                            CUs Allocated to Queue A

                                                                                            +

                                                                                            CUs Allocated to Queue A

                                                                                            CUs Allocated to Queue B

                                                                                            +

                                                                                            CUs Allocated to Queue B

                                                                                            Allocation Description

                                                                                            +

                                                                                            Allocation Description

                                                                                            Queues A and B are added to the elastic resource pool. The scaling policies are as follows:

                                                                                            -
                                                                                            • Queue A: period: 00:00–9:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: period: 00:00–9:00; minimum CU: 64; maximum CU: 128
                                                                                            +

                                                                                            Queues A and B are added to the elastic resource pool. The scaling policies are as follows:

                                                                                            +
                                                                                            • Queue A: period: 00:00–09:00; minimum CU: 32; maximum CU: 64
                                                                                            • Queue B: period: 00:00–09:00; minimum CU: 64; maximum CU: 128

                                                                                            192 CUs

                                                                                            +

                                                                                            192 CUs

                                                                                            64 CUs

                                                                                            +

                                                                                            64 CUs

                                                                                            128 CUs

                                                                                            +

                                                                                            128 CUs

                                                                                            If the actual CUs of the elastic resource pool are greater than or equal to the sum of the maximum CUs of the two queues,

                                                                                            +

                                                                                            If the actual CUs of the elastic resource pool are greater than or equal to the sum of the maximum CUs of the two queues,

                                                                                            the maximum CUs are allocated to both queues.

                                                                                            96 CUs

                                                                                            +

                                                                                            96 CUs

                                                                                            32 CUs

                                                                                            +

                                                                                            32 CUs

                                                                                            64 CUs

                                                                                            +

                                                                                            64 CUs

                                                                                            The elastic resource pool preferentially meets the minimum CUs of the two queues.

                                                                                            +

                                                                                            The elastic resource pool preferentially meets the minimum CUs of the two queues.

                                                                                            After the minimum CUs are allocated to the two queues, no CUs are allocatable.

                                                                                            128 CUs

                                                                                            +

                                                                                            128 CUs

                                                                                            32 CUs to 64 CUs

                                                                                            +

                                                                                            32 CUs to 64 CUs

                                                                                            64 CUs to 96 CUs

                                                                                            +

                                                                                            64 CUs to 96 CUs

                                                                                            The elastic resource pool preferentially meets the minimum CUs of the two queues. That is, 32 CUs are allocated to queue A, 64 CUs are allocated to queue B, and the remaining 32 CUs are available.

                                                                                            +

                                                                                            The elastic resource pool preferentially meets the minimum CUs of the two queues. That is, 32 CUs are allocated to queue A, 64 CUs are allocated to queue B, and the remaining 32 CUs are available.

                                                                                            The remaining CUs are allocated based on the queue load and priority. The actual CUs of the queue change within the range listed.

                                                                                            Name

                                                                                            Queue name

                                                                                            +

                                                                                            Name of the queue to add

                                                                                            Type

                                                                                            @@ -160,7 +162,7 @@

                                                                                          • Locate the target queue and click Edit in the Operation column.
                                                                                          • In the displayed Queue Management pane, perform the following operations as needed:

                                                                                            • Add: Click Create to add a scaling policy. Set Priority, Period, Min CU, and Max CU, and click OK.
                                                                                            • Modify: Modify parameters of an existing scaling policy and click OK.
                                                                                            • Delete: Locate the row that contains the scaling policy you want, click Delete and click OK.

                                                                                              The Priority and Period parameters must meet the following requirements:

                                                                                              -
                                                                                              • Priority: The default value is 1. The value ranges from 1 to 100. A bigger value indicates a higher priority.
                                                                                              • Period:
                                                                                                • You can only set the period to hours in [start time,end time) format.
                                                                                                • For example, if the Period to 01 and 17, the scaling policy takes effect at 01:00 a.m. till 05:00 p.m.
                                                                                                • The periods of scaling policies with different priorities cannot overlap.
                                                                                                +
                                                                                                • Priority: The default value is 1. The value ranges from 1 to 100. A larger value indicates a higher priority.
                                                                                                • Period:
                                                                                                  • You can only set the period to hours in [start time,end time) format.
                                                                                                  • For example, if the Period to 01 and 17, the scaling policy takes effect at 01:00 a.m. till 05:00 p.m.
                                                                                                  • The periods of scaling policies with different priorities cannot overlap.
                                                                                                • Max CUs and Min CUs:
                                                                                                  • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                                                                                                  • In any time segment of a day, the maximum CUs of any queue in an elastic resource pool cannot be more than the maximum CUs of the pool.
                                                                                              @@ -171,7 +173,7 @@
                                                                                              -
                                                                                              Parent topic: Regular Operations
                                                                                              +
                                                                                              Parent topic: Managing Elastic Resource Pools
                                                                                              diff --git a/docs/dli/umn/dli_01_0507.html b/docs/dli/umn/dli_01_0507.html index 13395af9c..d761e5ef7 100644 --- a/docs/dli/umn/dli_01_0507.html +++ b/docs/dli/umn/dli_01_0507.html @@ -20,7 +20,7 @@
                                                                                              -
                                                                                              Parent topic: Regular Operations
                                                                                              +
                                                                                              Parent topic: Managing Elastic Resource Pools
                                                                                              diff --git a/docs/dli/umn/dli_01_0508.html b/docs/dli/umn/dli_01_0508.html index 9c6bb96c7..532045a9c 100644 --- a/docs/dli/umn/dli_01_0508.html +++ b/docs/dli/umn/dli_01_0508.html @@ -1,12 +1,22 @@ -

                                                                                              Elastic Resource Pool

                                                                                              +

                                                                                              Creating an Elastic Resource Pool and Queues Within It

                                                                                              diff --git a/docs/dli/umn/dli_01_0509.html b/docs/dli/umn/dli_01_0509.html index 863967446..a505e8896 100644 --- a/docs/dli/umn/dli_01_0509.html +++ b/docs/dli/umn/dli_01_0509.html @@ -1,104 +1,29 @@ -

                                                                                              Adding a Queue

                                                                                              -

                                                                                              You can add one or more queues to an elastic resource pool to run jobs. This section describes how to add a queue to an elastic resource pool.

                                                                                              -

                                                                                              Precautions

                                                                                              Automatic scaling of an elastic resource pool cannot be triggered for Flink jobs.

                                                                                              -
                                                                                              -

                                                                                              Adding a Queue

                                                                                              1. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                              2. Locate the target elastic resource pool and click Add Queue in the Operation column.
                                                                                              3. On the Add Queue page, configure basic queue information. The following table describes the parameters.

                                                                                                -

                                                                                                - - - - - - - - - - - - - - - - - - - - - - -
                                                                                                Table 1 Queue information

                                                                                                Parameter

                                                                                                -

                                                                                                Description

                                                                                                -

                                                                                                Name

                                                                                                -

                                                                                                Name of the queue to add

                                                                                                -

                                                                                                Type

                                                                                                -
                                                                                                • For SQL: The queue is used to run SQL jobs.
                                                                                                • General general purpose: The queue is used to run Spark and Flink jobs.
                                                                                                -

                                                                                                Engine

                                                                                                -

                                                                                                If Type is For SQL, the queue engine can be spark or trino.

                                                                                                -

                                                                                                Enterprise Project

                                                                                                -

                                                                                                Select the enterprise project the queue belongs to. Queues under different enterprise projects can be added to an elastic resource pool.

                                                                                                -

                                                                                                Enterprise projects let you manage cloud resources and users by project.

                                                                                                -
                                                                                                NOTE:

                                                                                                This parameter is displayed only for users who have enabled the Enterprise Management Service.

                                                                                                -
                                                                                                -

                                                                                                Description

                                                                                                -

                                                                                                Description about the queue.

                                                                                                -

                                                                                                Tags

                                                                                                -

                                                                                                Tags used to identify cloud resources. A tag includes the tag key and tag value. If you want to use the same tag to identify multiple cloud resources, that is, to select the same tag from the drop-down list box for all services, you are advised to create predefined tags on the Tag Management Service (TMS).

                                                                                                -
                                                                                                NOTE:
                                                                                                • A maximum of 20 tags can be added.
                                                                                                • Only one tag value can be added to a tag key.
                                                                                                • The key name in each resource must be unique.
                                                                                                -
                                                                                                -
                                                                                                • Tag key: Enter a tag key name in the text box.
                                                                                                  NOTE:

                                                                                                  A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                                  -
                                                                                                  -
                                                                                                -
                                                                                                • Tag value: Enter a tag value in the text box.
                                                                                                  NOTE:

                                                                                                  A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                                                                                  -
                                                                                                  -
                                                                                                -
                                                                                                -
                                                                                                -

                                                                                              4. Click Next. On the displayed page, configure a scaling policy for the queue in the elastic resource pool.

                                                                                                Click Create to add a scaling policy with specified priority, period, Minimum CUs, and Maximum CUs. The parameters of each scaling policy are as follows:

                                                                                                - -
                                                                                                - - - - - - - - - - - - - - - - -
                                                                                                Table 2 Auto scaling policy

                                                                                                Parameter

                                                                                                -

                                                                                                Description

                                                                                                -

                                                                                                Priority

                                                                                                -

                                                                                                A bigger priority value indicates a higher priority. You can set a number ranging from 1 to 100.

                                                                                                -

                                                                                                Period

                                                                                                -

                                                                                                Time segment when the policy takes effects. It can be set only by hour. The start time is on the left, and the end time is on the right.

                                                                                                -
                                                                                                • The time range includes the start time but not the end time, that is, [start time, end time).

                                                                                                  For example, if the Period to 01 and 17, the scaling policy takes effect at 01:00 a.m. till 05:00 p.m.

                                                                                                  -
                                                                                                -
                                                                                                • The periods of scaling policies with different priorities cannot overlap.
                                                                                                -

                                                                                                Min CU

                                                                                                -

                                                                                                Minimum number of CUs allowed by the scaling policy.

                                                                                                -
                                                                                                • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                                                                                                • If the minimum CUs of the queue is less than 16 CUs, both Max. Spark Driver Instances and Max. Prestart Spark Driver Instances set in the queue properties do not apply. Refer to Setting Queue Properties.
                                                                                                -

                                                                                                Max CU

                                                                                                -

                                                                                                Maximum number of CUs allowed by the scaling policy.

                                                                                                -

                                                                                                In any time segment of a day, the maximum CUs of any queue in an elastic resource pool cannot be more than the maximum CUs of the pool.

                                                                                                -
                                                                                                -
                                                                                                -

                                                                                                The first scaling policy is the default policy and cannot be deleted or modified.

                                                                                                -
                                                                                                -

                                                                                              5. Click OK. View all queues and scaling policies added to the elastic resource pool by referring to Managing Queues.
                                                                                              -
                                                                                              -
                                                                                              +

                                                                                              Managing Enhanced Datasource Connections

                                                                                              +
                                                                                              diff --git a/docs/dli/umn/dli_01_05110.html b/docs/dli/umn/dli_01_05110.html index b36ca0f89..79036c88d 100644 --- a/docs/dli/umn/dli_01_05110.html +++ b/docs/dli/umn/dli_01_05110.html @@ -1,19 +1,20 @@ - -

                                                                                              Appendix

                                                                                              - -

                                                                                              +

                                                                                              Creating and Managing SQL Job Templates

                                                                                              +

                                                                                              - diff --git a/docs/dli/umn/dli_01_05111.html b/docs/dli/umn/dli_01_05111.html index 58b7256e6..2771d140d 100644 --- a/docs/dli/umn/dli_01_05111.html +++ b/docs/dli/umn/dli_01_05111.html @@ -1,8 +1,8 @@ -

                                                                                              TPC-H Sample Data in the SQL Template

                                                                                              +

                                                                                              TPC-H Sample Data in the SQL Templates Preset on DLI

                                                                                              TPC-H Sample Data

                                                                                              TPC-H is a test set developed by the Transaction Processing Performance Council (TPC) to simulate decision-making support applications. It is widely used in academia and industry to evaluate the performance of decision-making support technology. This business test has higher requirements on vendors, because it can comprehensively evaluate the overall business computing capability. With universal business significance, is widely used in analysis of bank credit, credit card, telecom operation, tax, as well as tobacco industry decision-making analysis.

                                                                                              -

                                                                                              The TPC-H benchmark test is developed from TPC-D (a standard specified by TPC in 1994 and used as the test benchmark for decision-making support systems). TPC-H implements a 3NF data warehouse that contains eight basic relationships, with a data volume range from 1 GB to 3 TB. The TPC-H benchmark test includes 22 queries (Q1 to Q22). The main evaluation indicator is the response time of each query (from submission to result return). The unit of the TPC-H benchmark test is the query number per hour (QphH@size). H indicates the average number of complex queries per hour. size indicates the size of database, which reflects the query processing capability of the system. TPC-H can evaluate key performance parameters that other tests cannot evaluate, because it is modeled based on the actual production and operation environment. In a word, the TPC-H standard by TPC meets the test requirements of data warehouse and motivate vendors and research institutes to stretch the limit of this technology.

                                                                                              +

                                                                                              The TPC-H benchmark test is derived from the TPC-D standard, which was established by the TPC organization in 1994 for decision support system testing. TPC-H implements a 3NF data warehouse that contains eight basic relationships, with a data volume range from 1 GB to 3 TB. The TPC-H benchmark test includes 22 queries (Q1 to Q22). The main evaluation indicator is the response time of each query (from submission to result return). The unit of the TPC-H benchmark test is the query number per hour (QphH@size). H indicates the average number of complex queries per hour. size indicates the size of database, which reflects the query processing capability of the system. TPC-H can evaluate key performance parameters that other tests cannot evaluate, because it is modeled based on the actual production and operation environment. In a word, the TPC-H standard by TPC meets the test requirements of data warehouse and motivate vendors and research institutes to stretch the limit of this technology.

                                                                                              In this example, DLI directly queries the TPC-H dataset on OBS. DLI has generated a standard TPC-H-2.18 dataset of 100 MB which is uploaded to the tpch folder on OBS. The read-only permission is granted to you to facilitate query operations.

                                                                                              TPC-H Test and Metrics

                                                                                              TPC-H test is divided into three sub-tests: data loading test, Power test, and Throughput test. Data loading indicates the process of setting up a test database, and the loading test is to test the data loading ability of DBMS. The first test is data loading test that tests data loading time, which is time-consuming. The second test is Power test, also called raw query. After data loading test is complete, the database is in the initial state without any other operation, especially the data in the buffer is not tested. Power test requires that the 22 queries be executed once in sequence and a pair of RF1 and RF2 operations be executed at the same time. The third test is Throughput test, the core and most complex test, more similar to the actual application environment. With multiple query statement groups and a pair of RF1 and RF2 update flows, Throughput test pose greater pressure on the SUT system than Power test does.

                                                                                              @@ -19,12 +19,17 @@

                                                                                              Precautions

                                                                                              When a sub-account uses the TPC-H test suite, the main account needs to grant the sub-account the OBS access permission and the permission to view the main account table. If the master account has not logged in to DLI, the sub-account needs to have the permissions to create databases and tables in addition to the preceding permissions.

                                                                                              -

                                                                                              Operation Description

                                                                                              For details, see Managing SQL Templates.

                                                                                              +

                                                                                              Developing and Submitting a SQL Job Using the TPC-H Sample Template

                                                                                              DLI allows you to create custom templates or save currently used SQL statements as templates for quick and convenient SQL operations. Once a template is saved, you can execute SQL operations directly through the template without the need to write SQL statements.

                                                                                              +
                                                                                              1. Log in to the DLI management console.
                                                                                              2. On the DLI management console, choose Job Templates > SQL Templates in the navigation pane on the left. On the displayed Sample Templates tab, click the plus sign next to tpchQuery to find the Q1_Price_summary_report_query template. Then, click Execute in the Operation column.

                                                                                                +
                                                                                              3. In the upper part of the editing window, set Engine to Spark, Queues to default, and Databases to default, and click Execute.

                                                                                                +
                                                                                              4. Check the query result on the View Result tab below the editing window.

                                                                                                +
                                                                                              +

                                                                                              This example uses the default queue and database preset in the system as an example. You can also run the command in a self-created queue and database.

                                                                                              -
                                                                                              Parent topic: Appendix
                                                                                              +
                                                                                              Parent topic: Creating and Managing SQL Job Templates
                                                                                              diff --git a/docs/dli/umn/dli_01_0512.html b/docs/dli/umn/dli_01_0512.html index a15b965cb..a2578c7ec 100644 --- a/docs/dli/umn/dli_01_0512.html +++ b/docs/dli/umn/dli_01_0512.html @@ -1,15 +1,317 @@ -

                                                                                              Developing and Submitting a Spark SQL Job Using the TPC-H Sample Template

                                                                                              -

                                                                                              DLI allows you to customize query templates or save frequently used SQL statements as templates to facilitate SQL operations. After templates are saved, you do not need to write SQL statements. You can directly perform the SQL operations using the templates.

                                                                                              -

                                                                                              The current system provides various standard TPC-H query statement templates. You can select a template as needed. This example shows how to use a TPC-H template to develop and submit a Spark SQL job.

                                                                                              -

                                                                                              For details about the templates, see SQL Template Management.

                                                                                              -

                                                                                              Procedure

                                                                                              1. Log in to the DLI management console.
                                                                                              2. On the DLI management console, choose Job Templates > SQL Templates, and click the Sample Templates tab. Locate the Q1_Price_summary_report_query template under tpchQuery, and click Execute in the Operation column. The SQL Editor page is displayed.

                                                                                                -
                                                                                              3. In the upper part of the editing window, set Engine to spark, Queues to default, and Databases to default, and click Execute.

                                                                                                -
                                                                                              4. View the query result in the View Result tab in the lower part of the SQL Editor page.

                                                                                                +

                                                                                                Using DLI to Submit a Flink Jar Job

                                                                                                +

                                                                                                Scenario

                                                                                                Flink Jar jobs are suitable for data analysis scenarios that require custom stream processing logic, complex state management, or integration with specific libraries. You need to write and build a Jar job package. Before submitting a Flink Jar job, upload the Jar job package to OBS and submit it together with the data and job parameters to run the job.

                                                                                                +

                                                                                                This example introduces the basic process of submitting a Flink Jar job package through the DLI console. Due to different service requirements, the specific writing of the Jar package may vary. It is recommended that you refer to the sample code provided by DLI and edit and customize it according to your actual business scenario.

                                                                                                +
                                                                                                +

                                                                                                Procedure

                                                                                                Table 1 describes the procedure for submitting a Flink Jar job using DLI.

                                                                                                + +
                                                                                                + + + + + + + + + + + + + +
                                                                                                Table 1 Procedure for submitting a Flink Jar job using DLI

                                                                                                Procedure

                                                                                                +

                                                                                                Description

                                                                                                +

                                                                                                Step 1: Develop a JAR File and Upload It to OBS

                                                                                                +

                                                                                                Prepare a Flink Jar job package and upload it to OBS.

                                                                                                +

                                                                                                Step 2: Buy an Elastic Resource Pool and Add Queues to the Pool

                                                                                                +

                                                                                                Create compute resources required for submitting the Flink Jar job.

                                                                                                +

                                                                                                Step 3: Create a Flink Jar Job and Configure Job Information

                                                                                                +

                                                                                                Create a Flink Jar job to analyze data.

                                                                                                +
                                                                                                +
                                                                                                +
                                                                                                +

                                                                                                Step 1: Develop a JAR File and Upload It to OBS

                                                                                                Develop a JAR file offline as the DLI console does not support this capability. For development examples, refer to "Flink Jar Job Examples".

                                                                                                +

                                                                                                Develop a Flink Jar job program, compile it, and pack it into flink-examples.jar. Perform the following steps to upload the program:

                                                                                                +

                                                                                                Before submitting a Flink job, upload data files to OBS.

                                                                                                +
                                                                                                1. Log in to the DLI console.
                                                                                                2. In the service list, click Object Storage Service under Storage.
                                                                                                3. Create a bucket. In this example, name it dli-test-obs01.
                                                                                                  1. On the displayed Buckets page, click Create Bucket in the upper right corner.
                                                                                                  2. On the displayed Create Bucket page, set Bucket Name. Retain the default values for other parameters or modify them as needed.

                                                                                                    Select a region that matches the location of the DLI console.

                                                                                                    +
                                                                                                    +
                                                                                                  3. Click Create Now.
                                                                                                  +
                                                                                                4. In the bucket list, click the name of the dli-test-obs01 bucket you just created to access its Objects tab.
                                                                                                5. Click Upload Object. In the displayed dialog box, drag or add files or folders, for example, flink-examples.jar, to the upload area. Then, click Upload.

                                                                                                  In this example, the path after upload is obs://dli-test-obs01/flink-examples.jar.

                                                                                                  +

                                                                                                  For more operations on the OBS console, see the Object Storage Service User Guide.

                                                                                                -

                                                                                                This example uses the default queue and database preset in the system as an example. You can also run query statements on a self-created queue and database.

                                                                                                -

                                                                                                For details about how to create a queue, see "Creating a Queue" in Data Lake Insight User Guide. For details about how to create a database, see Creating a Database.

                                                                                                +
                                                                                                +

                                                                                                Step 2: Buy an Elastic Resource Pool and Add Queues to the Pool

                                                                                                To execute SQL jobs in datasource scenarios, you must use your own SQL queue as the existing default queue cannot be used. In this example, create an elastic resource pool named dli_resource_pool and a queue named dli_queue_01.

                                                                                                +
                                                                                                1. Log in to the DLI management console.
                                                                                                2. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                3. On the displayed page, click Buy Resource Pool in the upper right corner.
                                                                                                4. On the displayed page, set the parameters.
                                                                                                  Table 2 describes the parameters. +
                                                                                                  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                  Table 2 Parameters

                                                                                                  Parameter

                                                                                                  +

                                                                                                  Description

                                                                                                  +

                                                                                                  Example Value

                                                                                                  +

                                                                                                  Region

                                                                                                  +

                                                                                                  Select a region where you want to buy the elastic resource pool.

                                                                                                  +

                                                                                                  _

                                                                                                  +

                                                                                                  Project

                                                                                                  +

                                                                                                  Project uniquely preset by the system for each region

                                                                                                  +

                                                                                                  Default

                                                                                                  +

                                                                                                  Name

                                                                                                  +

                                                                                                  Name of the elastic resource pool

                                                                                                  +

                                                                                                  dli_resource_pool

                                                                                                  +

                                                                                                  Specifications

                                                                                                  +

                                                                                                  Specifications of the elastic resource pool

                                                                                                  +

                                                                                                  Standard

                                                                                                  +

                                                                                                  CU Range

                                                                                                  +

                                                                                                  The maximum and minimum CUs allowed for the elastic resource pool

                                                                                                  +

                                                                                                  64-64

                                                                                                  +

                                                                                                  CIDR Block

                                                                                                  +

                                                                                                  CIDR block the elastic resource pool belongs to. If you use an enhanced datasource connection, this CIDR block cannot overlap that of the data source. Once set, this CIDR block cannot be changed.

                                                                                                  +

                                                                                                  172.16.0.0/19

                                                                                                  +

                                                                                                  Enterprise Project

                                                                                                  +

                                                                                                  Select an enterprise project for the elastic resource pool.

                                                                                                  +

                                                                                                  default

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                5. Click Buy.
                                                                                                6. Click Submit.
                                                                                                7. In the elastic resource pool list, locate the pool you just created and click Add Queue in the Operation column.
                                                                                                8. Set the basic parameters listed below. +
                                                                                                  + + + + + + + + + + + + + + + + + +
                                                                                                  Table 3 Basic parameters for adding a queue

                                                                                                  Parameter

                                                                                                  +

                                                                                                  Description

                                                                                                  +

                                                                                                  Example Value

                                                                                                  +

                                                                                                  Name

                                                                                                  +

                                                                                                  Name of the queue to add

                                                                                                  +

                                                                                                  dli_queue_01

                                                                                                  +

                                                                                                  Type

                                                                                                  +

                                                                                                  Type of the queue

                                                                                                  +
                                                                                                  • To execute SQL jobs, select For SQL.
                                                                                                  • To execute Flink or Spark jobs, select For general purpose.
                                                                                                  +

                                                                                                  _

                                                                                                  +

                                                                                                  Enterprise Project

                                                                                                  +

                                                                                                  Select an enterprise project.

                                                                                                  +

                                                                                                  default

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                9. Click Next and configure scaling policies for the queue.

                                                                                                  Click Create to add a scaling policy with varying priority, period, minimum CUs, and maximum CUs.

                                                                                                  +

                                                                                                  + +
                                                                                                  + + + + + + + + + + + + + + + + + + + + + +
                                                                                                  Table 4 Scaling policy parameters

                                                                                                  Parameter

                                                                                                  +

                                                                                                  Description

                                                                                                  +

                                                                                                  Example Value

                                                                                                  +

                                                                                                  Priority

                                                                                                  +

                                                                                                  Priority of the scaling policy in the current elastic resource pool. A larger value indicates a higher priority. In this example, only one scaling policy is configured, so its priority is set to 1 by default.

                                                                                                  +

                                                                                                  1

                                                                                                  +

                                                                                                  Period

                                                                                                  +

                                                                                                  The first scaling policy is the default policy, and its Period parameter configuration cannot be deleted or modified.

                                                                                                  +

                                                                                                  The period for the scaling policy is from 00 to 24.

                                                                                                  +

                                                                                                  00–24

                                                                                                  +

                                                                                                  Min CU

                                                                                                  +

                                                                                                  Minimum number of CUs allowed by the scaling policy

                                                                                                  +

                                                                                                  16

                                                                                                  +

                                                                                                  Max CU

                                                                                                  +

                                                                                                  Maximum number of CUs allowed by the scaling policy

                                                                                                  +

                                                                                                  64

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                10. Click OK.
                                                                                                +
                                                                                                +

                                                                                                Step 3: Create a Flink Jar Job and Configure Job Information

                                                                                                1. Create a Flink Jar job.

                                                                                                  1. In the navigation pane on the left of the DLI management console, choose Job Management > Flink Jobs.
                                                                                                  2. On the displayed page, click Create Job in the upper right corner.

                                                                                                    In this example, set Type to Flink Jar and Name to Flink_Jar_for_test.

                                                                                                    +
                                                                                                  3. Click OK.
                                                                                                  +

                                                                                                2. Configure basic job information.

                                                                                                  Configure basic job information based on Table 5.

                                                                                                  + +
                                                                                                  + + + + + + + + + + + + + + + + + + + + + +
                                                                                                  Table 5 Parameters

                                                                                                  Parameter

                                                                                                  +

                                                                                                  Mandatory

                                                                                                  +

                                                                                                  Description

                                                                                                  +

                                                                                                  Queue

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  Select a queue where you want to run your job.

                                                                                                  +

                                                                                                  Application

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  Select the custom package in Step 1: Develop a JAR File and Upload It to OBS.

                                                                                                  +

                                                                                                  Main Class

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  Class name of the JAR file to load

                                                                                                  +

                                                                                                  This parameter specifies the entry for the Flink job, that is, the class that contains the main method. This is the class that is executed first when a Flink job is started.

                                                                                                  +

                                                                                                  If the application program is of type .jar, the main class name must be provided.

                                                                                                  +

                                                                                                  The main class name is case-sensitive and must be correct.

                                                                                                  +
                                                                                                  • Default: Specified based on the Manifest file in the JAR file.
                                                                                                  • Manually assign: You must enter the class name and confirm the class arguments (separated by spaces).
                                                                                                  +
                                                                                                  NOTE:

                                                                                                  When a class belongs to a package, the package path must be carried, for example, packagePath.KafkaMessageStreaming.

                                                                                                  +
                                                                                                  +

                                                                                                  Flink Version

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  Flink version used for job running

                                                                                                  +
                                                                                                  +
                                                                                                  +

                                                                                                3. Configure advanced settings for the Flink Jar job.

                                                                                                  Configure the Flink Jar job based on Table 6.

                                                                                                  + +
                                                                                                  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                  Table 6 Advanced settings for the Flink Jar job

                                                                                                  Parameter

                                                                                                  +

                                                                                                  Mandatory

                                                                                                  +

                                                                                                  Description

                                                                                                  +

                                                                                                  CUs

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  One CU consists of one vCPU and 4 GB of memory. The number of CUs ranges from 2 to 400.

                                                                                                  +

                                                                                                  Job Manager CUs

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  Number of CUs allowed for the job manager. The value ranges from 1 to 4. The default value is 1.

                                                                                                  +

                                                                                                  Parallelism

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  Maximum number of parallel operators in a job

                                                                                                  +
                                                                                                  NOTE:
                                                                                                  • The value cannot exceed four times the number of compute units (CUs – Job Manager CUs).
                                                                                                  • You are advised to set this parameter to a value greater than that configured in the code. Otherwise, job submission may fail.
                                                                                                  +
                                                                                                  +

                                                                                                  Task Manager Config

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  Whether Task Manager resource parameters are set

                                                                                                  +

                                                                                                  If this option is selected, you need to set the following parameters:

                                                                                                  +
                                                                                                  • CU(s) per TM: Number of resources occupied by each Task Manager.
                                                                                                  • Slot(s) per TM: Number of slots contained in each Task Manager.
                                                                                                  +

                                                                                                  Save Job Log

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  Whether job running logs are saved to OBS

                                                                                                  +

                                                                                                  If this option is selected, you need to set the following parameters:

                                                                                                  +

                                                                                                  OBS Bucket: Select an OBS bucket to store job logs. If the OBS bucket you selected is unauthorized, click Authorize.

                                                                                                  +

                                                                                                  Alarm on Job Exception

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  Whether to notify users of any job exceptions, such as running exceptions or arrears, via SMS or email.

                                                                                                  +

                                                                                                  If this option is selected, you need to set the following parameters:

                                                                                                  +

                                                                                                  SMN Topic

                                                                                                  +

                                                                                                  Select a custom SMN topic. For how to create a custom SMN topic, see "Creating a Topic" in the Simple Message Notification User Guide.

                                                                                                  +

                                                                                                  Auto Restart on Exception

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  Whether automatic restart is enabled. If enabled, jobs will be automatically restarted and restored when exceptions occur.

                                                                                                  +

                                                                                                  If this option is selected, you need to set the following parameters:

                                                                                                  +
                                                                                                  • Max. Retry Attempts: maximum number of retries upon an exception. The unit is times/hour.
                                                                                                    • Unlimited: The number of retries is unlimited.
                                                                                                    • Limited: The number of retries is user-defined.
                                                                                                    +
                                                                                                  • Restore Job from Checkpoint: Restore the job from the latest checkpoint.

                                                                                                    If you select this parameter, you also need to set Checkpoint Path.

                                                                                                    +

                                                                                                    Checkpoint Path: Select a path for storing checkpoints. This path must match that configured in the application package. Each job must have a unique checkpoint path, or, you will not be able to obtain the checkpoint.

                                                                                                    +
                                                                                                  +
                                                                                                  +
                                                                                                  +

                                                                                                4. Click Save in the upper right of the page.
                                                                                                5. Click Start in the upper right corner.
                                                                                                6. On the displayed Start Flink Job page, confirm the job specification and click Start Now to start the job.

                                                                                                  Once the job is started, the system automatically switches to the Flink Jobs page. Locate the job you created and check its status in the Status column.

                                                                                                  +

                                                                                                  Once a job is successfully submitted, its status changes from Submitting to Running. After the execution is complete, the status changes to Completed.

                                                                                                  +

                                                                                                  If the job status is Submission failed or Running exception, the job fails to submit or run. In this case, you can hover over the status icon in the Status column of the job list to view the error details. You can click to copy these details. Rectify the fault based on the error information and resubmit the job.

                                                                                                  +

                                                                                                diff --git a/docs/dli/umn/dli_01_0513.html b/docs/dli/umn/dli_01_0513.html deleted file mode 100644 index 309818762..000000000 --- a/docs/dli/umn/dli_01_0513.html +++ /dev/null @@ -1,21 +0,0 @@ - - - -

                                                                                                Other Common Operations

                                                                                                - -

                                                                                                -
                                                                                                - - - diff --git a/docs/dli/umn/dli_01_0514.html b/docs/dli/umn/dli_01_0514.html index 95de5686a..fd4a93f6d 100644 --- a/docs/dli/umn/dli_01_0514.html +++ b/docs/dli/umn/dli_01_0514.html @@ -1,7 +1,11 @@ -

                                                                                                Binding and Unbinding a Queue

                                                                                                -

                                                                                                Constraints

                                                                                                • The CIDR block of the DLI queue that is bound with a datasource connection cannot overlap with that of the data source.
                                                                                                • The default queue cannot be bound with a connection.
                                                                                                +

                                                                                                Binding or Unbinding an Enhanced Datasource Connection to a Queue

                                                                                                +

                                                                                                Bind an elastic resource pool to a queue to connect the queue to a target data source. When an enhanced datasource connection is no longer needed, unbind the queue that was bound to it to free up resources.

                                                                                                +

                                                                                                This section describes how to bind and unbind an enhanced datasource connection to a queue.

                                                                                                +

                                                                                                The operations described in this section only apply to standard queues.

                                                                                                +
                                                                                                +

                                                                                                Notes and Constraints

                                                                                                • The CIDR block of the DLI queue that is bound with a datasource connection cannot overlap with that of the data source.
                                                                                                • The default queue cannot be bound with a connection.

                                                                                                Binding a Queue

                                                                                                Before using an enhanced datasource connection, you must bind a queue and ensure that the VPC peering connection is in the Active state.

                                                                                                • Method 1

                                                                                                  In the Enhanced tab, select a connection and click Bind Queue in the Operation column. In the dialog box that is displayed, select the queues to be bound and click OK.

                                                                                                  @@ -10,40 +14,40 @@

                                                                                                Viewing Details about a Bound Queue

                                                                                                On the Enhanced tab page, select a connection and click the connection name to view the information about the bound queue. -
                                                                                                Table 1 Parameters in the details list of datasource connection queues

                                                                                                Parameter

                                                                                                +
                                                                                                - - - - - - - - - - - @@ -60,7 +64,7 @@ diff --git a/docs/dli/umn/dli_01_0515.html b/docs/dli/umn/dli_01_0515.html index 37e993a8a..8e7050394 100644 --- a/docs/dli/umn/dli_01_0515.html +++ b/docs/dli/umn/dli_01_0515.html @@ -1,6 +1,6 @@ -

                                                                                                Creating an Elastic Resource Pool and Running a Job

                                                                                                +

                                                                                                Example Use Case: Creating an Elastic Resource Pool and Running Jobs

                                                                                                This section walks you through the procedure of adding a queue to an elastic resource pool and binding an enhanced datasource connection to the elastic resource pool.
                                                                                                Figure 1 Process of creating an elastic resource pool
                                                                                                @@ -16,7 +16,7 @@
                                                                                                - - -
                                                                                                Table 1 Parameters in the details list of datasource connection queues

                                                                                                Parameter

                                                                                                Description

                                                                                                +

                                                                                                Description

                                                                                                VPC Peering ID

                                                                                                +

                                                                                                VPC Peering ID

                                                                                                ID of the VPC peering connection created in the cluster to which the queue belongs.

                                                                                                +

                                                                                                ID of the VPC peering connection created in the cluster to which the queue belongs.

                                                                                                NOTE:

                                                                                                A VPC peering connection is created for each queue bound to an enhanced datasource connection. The VPC peering connection is used for cross-VPC communication. Ensure that the security group used by the data source allows access from the DLI queue CIDR block, and do not delete the VPC peering connection during the datasource connection.

                                                                                                Name

                                                                                                +

                                                                                                Name

                                                                                                Name of a bound queue.

                                                                                                +

                                                                                                Name of a bound queue.

                                                                                                Connection Status

                                                                                                +

                                                                                                Connection Status

                                                                                                Datasource connection status. The following three statuses are available:

                                                                                                +

                                                                                                Datasource connection status. The following three statuses are available:

                                                                                                • Creating
                                                                                                • Active
                                                                                                • Failed
                                                                                                NOTE:

                                                                                                If the connection status is Failed, click on the left to view the detailed error information.

                                                                                                Updated

                                                                                                +

                                                                                                Updated

                                                                                                Time when a connection is updated. The connections in the connection list can be displayed according to the update time in ascending or descending order.

                                                                                                +

                                                                                                Time when a connection is updated. The connections in the connection list can be displayed according to the update time in ascending or descending order.

                                                                                                Operation

                                                                                                +

                                                                                                Operation

                                                                                                Unbind Queue: This operation is used to unbind a datasource connection from a queue.

                                                                                                +

                                                                                                Unbind Queue: This operation is used to unbind a datasource connection from a queue.

                                                                                                Create an elastic resource pool and configure basic information, such as the billing mode, CU range, and CIDR block.

                                                                                                Creating an Elastic Resource Pool

                                                                                                +

                                                                                                Creating an Elastic Resource Pool and Creating Queues Within It

                                                                                                Add a queue to the elastic resource pool

                                                                                                @@ -24,8 +24,8 @@

                                                                                                Add the queue where your jobs will run on to the elastic resource pool. The operations are as follows:

                                                                                                1. Set basic information about the queue, such as the name and type.
                                                                                                2. Configure the scaling policy of the queue, including the priority, period, and the maximum and minimum CUs allowed for scaling.

                                                                                                Adding a Queue

                                                                                                -

                                                                                                Managing Queues

                                                                                                +

                                                                                                Creating an Elastic Resource Pool and Creating Queues Within It

                                                                                                +

                                                                                                Adjusting Scaling Policies for Queues in an Elastic Resource Pool

                                                                                                (Optional) Create an enhanced datasource connection.

                                                                                                @@ -40,16 +40,16 @@

                                                                                                Create and submit the job as you need.

                                                                                                SQL Job Management

                                                                                                -

                                                                                                Overview

                                                                                                +

                                                                                                Managing SQL Jobs

                                                                                                +

                                                                                                Flink Job Overview

                                                                                                Creating a Spark Job
                                                                                                -

                                                                                                Step 1: Create an Elastic Resource Pool

                                                                                                1. Log in to the DLI management console. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                2. On the displayed Resource Pool page, click Buy Resource Pool in the upper right corner.
                                                                                                3. On the displayed page, set the following parameters:
                                                                                                  • Name: Enter the name of the elastic resource pool. For example, pool_test.
                                                                                                  • CU range: Minimum and maximum CUs of the elastic resource pool.
                                                                                                  • CIDR Block: Network segment of the elastic resource pool. For example, 172.16.0.0/18.
                                                                                                  • Set other parameters as required.
                                                                                                  -

                                                                                                  For details about how to create an elastic resource pool, see Creating an Elastic Resource Pool.

                                                                                                  +

                                                                                                  Step 1: Create an Elastic Resource Pool

                                                                                                  1. Log in to the DLI management console. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                  2. On the displayed page, click Buy Resource Pool in the upper right corner.
                                                                                                  3. On the displayed page, set the following parameters:
                                                                                                    • Name: Enter the name of the elastic resource pool. For example, pool_test.
                                                                                                    • CU range: Minimum and maximum CUs of the elastic resource pool.
                                                                                                    • CIDR Block: Network segment of the elastic resource pool. For example, 172.16.0.0/18.
                                                                                                    • Set other parameters as required.
                                                                                                    +

                                                                                                    For details about how to create an elastic resource pool, see Creating an Elastic Resource Pool and Creating Queues Within It.

                                                                                                  4. Click Buy. Confirm the configuration and click Pay.
                                                                                                  5. Go to the Resource Pool page to view the creation status. If the status is Available, the elastic resource pool is ready for use.

                                                                                                  Step 2: Add a Queue to the Elastic Resource Pool

                                                                                                  1. In the Operation column of the created elastic resource pool, click Add Queue.
                                                                                                  2. Specify the basic information about the queue. The configuration parameters are as follows:
                                                                                                    • Name: Queue name
                                                                                                    • Type: Queue type In this example, select For general purpose.

                                                                                                      For SQL: The queue is used to run Spark SQL jobs.

                                                                                                      @@ -58,19 +58,20 @@
                                                                                                    • Click Next. On the displayed page, set Min CU to 64 and Max CU to 64.
                                                                                                    • Click OK. The queue is added.

                                                                                                  (Optional) Step 3: Create an Enhanced Datasource Connection

                                                                                                  In this example, a datasource connection is required to connect to RDS. You need to create a datasource connection. If your job does not need to connect to an external data source, skip this step.

                                                                                                  -
                                                                                                  1. Log in to the RDS console and create an RDS DB instance. For details, see . Log in to the RDS instance, create a database and name it test2.
                                                                                                  2. Locate the row that contains the test2 database, click Query SQL Statements in the Operation column. On the displayed page, enter the following statement to create table tabletest2. Click Execute SQL. The table creation statement is as follows:
                                                                                                    CREATE TABLE `tabletest2` (
+
                                                                                                    1. Log in to the RDS console and create an RDS DB instance.
                                                                                                      For details, see "RDS for MySQL Getting Started" in the Relational Database Service Getting Started.
                                                                                                      
+
                                                                                                    2. Click Create Database. In the dialog box that appears, enter database name test2. Then, click OK.
                                                                                                    3. Locate the row that contains the test2 database, click Query SQL Statements in the Operation column. On the displayed page, enter the following statement to create table tabletest2. Click Execute SQL. The table creation statement is as follows:
                                                                                                      CREATE TABLE `tabletest2` (
 	`id` int(11) unsigned,
 	`name` VARCHAR(32)
 )	ENGINE = InnoDB	DEFAULT CHARACTER SET = utf8mb4;
                                                                                                      
-
                                                                                                    4. On the RDS console, choose Instances form the navigation pane. Click the name of a created RDS DB instance to view its basic information.
                                                                                                    5. In the Connection Information pane, obtain the floating IP address, database port, VPC, and subnet.
                                                                                                    6. Click the security group name. In the Inbound Rules tab, add a rule to allow access from the CIDR block of the elastic resource pool. For example, if the CIDR block of the elastic resource pool is 172.16.0.0/18 and the database port is 3306, set the rule Priority to 1, Action to Allow, Protocol to TCP and Port to 3306, Type to IPv4, and Source to 172.16.0.0/18.
                                                                                                      Click OK. The security group rule is added.
                                                                                                      
+
                                                                                                    7. On the RDS console, choose Instances form the navigation pane. Click the name of a created RDS DB instance to view its basic information.
                                                                                                    8. In the Connection Information pane, obtain the floating IP address, database port, VPC, and subnet.
                                                                                                    9. Click the security group name. In the Inbound Rules tab, add a rule to allow access from the CIDR block of the elastic resource pool. For example, if the CIDR block of the elastic resource pool is 172.16.0.0/18 and the database port is 3306, set the rule Priority to 1, Action to Allow, Protocol to TCP and Port to 3306, Type to IPv4, and Source to 172.16.0.0/18.
                                                                                                      Click OK. The security group rule is added.
                                                                                                      
 
                                                                                                    10. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
                                                                                                    11. In the displayed dialog box, set the following parameters:
                                                                                                      • Connection Name: Name of the enhanced datasource connection
                                                                                                      • Resource Pool: Select the elastic resource pool created in Step 1: Create an Elastic Resource Pool.
                                                                                                         
                                                                                                        If you cannot decide the elastic resource pool in this step, you can skip this parameter, go to the Enhanced tab, and click More > Bind Resource Pool in the Operation column of the row that contains this datasource connection after it is created.
                                                                                                        
 
                                                                                                        
-
                                                                                                      • VPC: Select the VPC of the RDS DB instance obtained in 4.
                                                                                                      • Subnet: Select the subnet of the RDS DB instance obtained in 4. 
                                                                                                      • Set other parameters as you need.
                                                                                                      
+
                                                                                                    12. VPC: Select the VPC of the RDS DB instance obtained in 5.
                                                                                                    13. Subnet: Select the subnet of the RDS DB instance obtained in 5. 
                                                                                                    14. Set other parameters as you need.
                                                                                                      15. 
 
                                                                                                      Click OK. Click the name of the created datasource connection to view its status. You can perform subsequent steps only after the connection status changes to Active.
                                                                                                      
 
                                                                                                    15. Click Resources > Queue Management, select the target queue, for example, general_test. In the Operation column, click More and select Test Address Connectivity.
                                                                                                    16. In the displayed dialog box, enter Floating IP address:Database port of the RDS database in the Address box and click Test to check whether the database is reachable.

                                                                                                  Step 4: Run a Job

                                                                                                  Run a Flink SQL jab on a queue in an elastic resource pool.

                                                                                                  -
                                                                                                  1. On the DLI management console, choose Job Management > Flink Jobs. On the Flink Jobs page, click Create Job.
                                                                                                  2. In the Create Job dialog box, set Type to Flink SQL and Name to testFlinkSqlJob. Click OK.
                                                                                                  3. On the job editing page, set the following parameters:
                                                                                                    • Queue: Select the general_test queue added to the elastic resource pool in Step 2: Add a Queue to the Elastic Resource Pool.
                                                                                                    • Save Job Log: Enable this function.
                                                                                                    • OBS Bucket: Select an OBS bucket for storing job logs and grant access permissions of the OBS bucket as prompted.
                                                                                                    • Enable Checkpointing: Enable.
                                                                                                    • Enter the SQL statement in the editing pane. The following is an example. Modify the parameters in bold as you need.
                                                                                                      CREATE SINK STREAM car_info (id INT, name STRING) WITH (
+
                                                                                                      1. On the DLI management console, choose Job Management > Flink Jobs. On the Flink Jobs page, click Create Job.
                                                                                                      2. In the Create Job dialog box, set Type to Flink SQL and Name to testFlinkSqlJob. Click OK.
                                                                                                      3. On the job editing page, set the following parameters:
                                                                                                        • Queue: Select the general_test queue added to the elastic resource pool in Step 2: Add a Queue to the Elastic Resource Pool.
                                                                                                        • Save Job Log: Enable this function.
                                                                                                        • OBS Bucket: Select an OBS bucket for storing job logs and grant access permissions of the OBS bucket as prompted.
                                                                                                        • Enable Checkpointing: Enable this function.
                                                                                                        • Enter the SQL statement in the editing pane. The following is an example. Modify the parameters in bold as you need.
                                                                                                          CREATE SINK STREAM car_info (id INT, name STRING) WITH (
   type = "rds",
   region = "", /* Change the value to the current region ID. */
    'pwd_auth_name'="xxxxx", // Name of the datasource authentication of the password type created on DLI. If datasource authentication is used, you do not need to set the username and password for the job.
@@ -88,7 +89,7 @@ SELECT
                                                                                                  diff --git a/docs/dli/umn/dli_01_0516.html b/docs/dli/umn/dli_01_0516.html index 195139dac..0579404fe 100644 --- a/docs/dli/umn/dli_01_0516.html +++ b/docs/dli/umn/dli_01_0516.html @@ -1,6 +1,6 @@ -

                                                                                                  Configuring Scaling Policies for Queues

                                                                                                  +

                                                                                                  Example Use Case: Configuring Scaling Policies for Queues in an Elastic Resource Pool

                                                                                                  Scenario

                                                                                                  A company has multiple departments that perform data analysis in different periods during a day.

                                                                                                  • Department A requires a large number of compute resources from 00:00 a.m. to 09:00 a.m. In other time segments, only small tasks are running.
                                                                                                  • Department B requires a large number of compute resources from 10:00 a.m. to 10:00 p.m. Some periodical tasks are running in other time segments during a day.

                                                                                                  In the preceding scenario, you can add two queues to an elastic resource pool: queue test_a for department A, and queue test_b for department B. You can add scaling policies for 00:00-09:00 and 10:00-23:00 respectively to the test_a and test_b queues. For jobs in other periods, you can modify the default scaling policy.

                                                                                                  @@ -65,38 +65,40 @@
                                                                                                -

                                                                                                Precautions

                                                                                                • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                                                                                                • In any time segment of a day, the maximum CUs of any queue in an elastic resource pool cannot be more than the maximum CUs of the pool.
                                                                                                • The periods of scaling policies cannot overlap.
                                                                                                • The period of a scaling policy can only be set by hour and specified by the start time and end time. For example, if you set the period to 00-09, the period when the policy takes effect is [00:00, 09:00). The period of the default scaling policy cannot be modified.
                                                                                                • In any period, compute resources are preferentially allocated to meet the minimum number of CUs of all queues. The remaining CUs (maximum CUs of the elastic resource pool – total minimum CUs of all queues) are allocated in accordance with the scaling policy priorities.
                                                                                                  • A scaling policy with a smaller priority value (for example, 1) is prior to those with a bigger priority value (for example, 100).
                                                                                                  • If the scaling policies of two queues have the same priority, resources are randomly allocated to a queue. If there are remaining resources, they are randomly allocated until there is no more left. -
                                                                                                    @@ -56,7 +56,7 @@
                                                                                                    • Click the text box for tag key and select a predefined tag key from the drop-down list.

                                                                                                      To add a predefined tag, you need to create one on TMS and then select it from the Tag key drop-down list. You can click View predefined tags to go to the Predefined Tags page of the TMS console. Then, click Create Tag in the upper corner of the page to create a predefined tag.

                                                                                                    -
                                                                                                    • Enter a tag key in the text box.
                                                                                                      NOTE:

                                                                                                      A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                                      +
                                                                                                      • Enter a tag key in the text box.
                                                                                                        NOTE:

                                                                                                        A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                                      @@ -64,7 +64,7 @@
                                                                                                    @@ -81,7 +81,7 @@ diff --git a/docs/dli/umn/dli_01_0553.html b/docs/dli/umn/dli_01_0553.html index 18bfae590..243bb5349 100644 --- a/docs/dli/umn/dli_01_0553.html +++ b/docs/dli/umn/dli_01_0553.html @@ -8,7 +8,7 @@ diff --git a/docs/dli/umn/dli_01_0556.html b/docs/dli/umn/dli_01_0556.html index 132c2c98a..22158a6fd 100644 --- a/docs/dli/umn/dli_01_0556.html +++ b/docs/dli/umn/dli_01_0556.html @@ -1,6 +1,6 @@ -

                                                                                                    Deleting a Route

                                                                                                    +

                                                                                                    Deleting the Route for an Enhanced Datasource Connection

                                                                                                    Scenario

                                                                                                    Delete a route that is no longer used.

                                                                                                    Constraints

                                                                                                    A custom route table cannot be deleted if it is associated with a subnet.

                                                                                                    @@ -13,7 +13,7 @@
                                                                                                    diff --git a/docs/dli/umn/dli_01_0558.html b/docs/dli/umn/dli_01_0558.html index f3ea894f6..b2b82a612 100644 --- a/docs/dli/umn/dli_01_0558.html +++ b/docs/dli/umn/dli_01_0558.html @@ -55,7 +55,7 @@
                                                                                                    diff --git a/docs/dli/umn/dli_01_0559.html b/docs/dli/umn/dli_01_0559.html index 9af8209c9..b3c584be8 100644 --- a/docs/dli/umn/dli_01_0559.html +++ b/docs/dli/umn/dli_01_0559.html @@ -45,7 +45,7 @@ diff --git a/docs/dli/umn/dli_01_0560.html b/docs/dli/umn/dli_01_0560.html index be7c831c7..8c3ac841d 100644 --- a/docs/dli/umn/dli_01_0560.html +++ b/docs/dli/umn/dli_01_0560.html @@ -68,7 +68,7 @@ diff --git a/docs/dli/umn/dli_01_0561.html b/docs/dli/umn/dli_01_0561.html index 2bd529013..68d69fcba 100644 --- a/docs/dli/umn/dli_01_0561.html +++ b/docs/dli/umn/dli_01_0561.html @@ -1,20 +1,41 @@

                                                                                                    Overview

                                                                                                    -

                                                                                                    What Is Datasource Authentication?

                                                                                                    When analyzing across multiple sources, it is not recommended to configure authentication information directly in a job as it can lead to password leakage. Instead, you are advised to use either Data Encryption Workshop (DEW) or datasource authentication provided by DLI to securely store data source authentication information.

                                                                                                    -
                                                                                                    • DEW is a comprehensive cloud-based encryption service that addresses data security, key security, and complex key management issues. You are advised to use DEW to store authentication information for data sources.
                                                                                                    • Datasource authentication is used to manage authentication information for accessing specified data sources. After datasource authentication is configured, you do not need to repeatedly configure data source authentication information in jobs, improving data source authentication security while enabling DLI to securely access data sources.
                                                                                                    +

                                                                                                    What Is Datasource Authentication?

                                                                                                    When analyzing across multiple sources, you are advised not to configure authentication information directly in a job as it can lead to password leakage. Instead, you are advised to use either Data Encryption Workshop (DEW) or datasource authentication provided by DLI to securely store data source authentication information.

                                                                                                    +
                                                                                                    • DEW is a comprehensive cloud-based encryption service that addresses data security, key security, and complex key management issues. You are advised to use DEW to store authentication information for data sources.

                                                                                                      You are advised to use DEW to store authentication information of data sources when Spark 3.3.1 or later and Flink 1.15 or later jobs access data sources using datasource connections. This will help you address issues related to data security, key security, and complex key management. For details, see Using DEW to Manage Access Credentials for Data Sources.

                                                                                                      +
                                                                                                    • Datasource authentication is used to manage authentication information for accessing specified data sources. After datasource authentication is configured, you do not need to repeatedly configure data source authentication information in jobs, improving data source authentication security while enabling DLI to securely access data sources.

                                                                                                      When SQL and Flink 1.12 jobs access data sources using datasource connections, use DLI's datasource authentication feature to manage data source access credentials.

                                                                                                      +

                                                                                                    This section describes how to use datasource authentication provided by DLI.

                                                                                                    -

                                                                                                    Notes and Constraints

                                                                                                    • Only Spark SQL and Flink OpenSource SQL 1.12 jobs support datasource authentication.
                                                                                                    • DLI supports four types of datasource authentication. Select an authentication type specific to each data source.
                                                                                                      • CSS: applies to 6.5.4 or later CSS clusters with the security mode enabled.
                                                                                                      • Kerberos: applies to MRS security clusters with Kerberos authentication enabled.
                                                                                                      • Kafka_SSL: applies to Kafka with SSL enabled.
                                                                                                      • Password: applies to GaussDB(DWS), RDS, DDS, and DCS.
                                                                                                      +

                                                                                                      Notes and Constraints

                                                                                                      +
                                                                                                    Table 2 CU allocation

                                                                                                    Scenario

                                                                                                    +

                                                                                                    Precautions

                                                                                                    • You are advised to implement fine-grained management of resource pools for stream and batch processing jobs by placing Flink real-time stream jobs and SQL batch processing jobs in separate elastic resource pools.

                                                                                                      Flink real-time stream jobs can run stably without forced scale-in, thus avoiding job interruption and system instability.

                                                                                                      +

                                                                                                      SQL batch processing jobs are placed in independent resource pools, which can scale out and in more flexibly, significantly enhancing the success rate and operational efficiency of scaling operations.

                                                                                                      +
                                                                                                    • In any time segment of a day, the total minimum CUs of all queues in an elastic resource pool cannot be more than the minimum CUs of the pool.
                                                                                                    • In any time segment of a day, the maximum CUs of any queue in an elastic resource pool cannot be more than the maximum CUs of the pool.
                                                                                                    • The periods of scaling policies cannot overlap.
                                                                                                    • The period of a scaling policy can only be set by hour and specified by the start time and end time. For example, if you set the period to 00-09, the period when the policy takes effect is [00:00, 09:00). The period of the default scaling policy cannot be modified.
                                                                                                    • In any period, compute resources are preferentially allocated to meet the minimum number of CUs of all queues. The remaining CUs (maximum CUs of the elastic resource pool – total minimum CUs of all queues) are allocated in accordance with the scaling policy priorities.
                                                                                                      • Scaling policies with smaller priority values are prior to over those with larger priority values.
                                                                                                      • If the scaling policies of two queues have the same priority, resources are randomly allocated to a queue. If there are remaining resources, they are randomly allocated until there is no more left. +
                                                                                                        - - - - - - - - - @@ -106,16 +108,16 @@ -

                                                                                                        Setting a Scaling Policy

                                                                                                        1. Log in to the DLI management console and create an elastic resource pool. Set the minimum and maximum number of CUs of the pool to 128 and 256 respectively. For details, see Creating an Elastic Resource Pool.
                                                                                                        2. Choose Resources > Resource Pool. Locate the row that contains the created elastic resource pool, and click Queue MGMT in the Operation column.
                                                                                                        3. Refer to Adding a Queue to create the test_a queue and set the scaling policy.

                                                                                                          1. Set the priority of the default scaling policy to 5, Min CU to 16, and Max CU to 32.
                                                                                                          2. Click create to add a scaling policy. Set the priority to 20, Period to 00--09, Min CU to 64, and Max CU to 128.
                                                                                                          +

                                                                                                          Setting a Scaling Policy

                                                                                                          1. Log in to the DLI management console and create an elastic resource pool. Set the minimum and maximum number of CUs of the pool to 128 and 256 respectively. For details, see Creating an Elastic Resource Pool and Creating Queues Within It.
                                                                                                          2. Choose Resources > Resource Pool. Locate the row that contains the created elastic resource pool, and click Queue MGMT in the Operation column.
                                                                                                          3. Refer to Creating an Elastic Resource Pool and Creating Queues Within It to create the test_a queue and set the scaling policy.

                                                                                                            1. Set the priority of the default scaling policy to 5, Min CU to 16, and Max CU to 32.
                                                                                                            2. Click create to add a scaling policy. Set the priority to 20, Period to 00--09, Min CU to 64, and Max CU to 128.

                                                                                                          4. View the scaling policy on the Queue Management page of the specific elastic resource pool.

                                                                                                            Click to view graphical statistics of priorities and CU settings for all time segments.

                                                                                                            -

                                                                                                          5. Refer to Adding a Queue to create the test_b queue and set the scaling policy.

                                                                                                            1. Set the priority of the default scaling policy to 5, Min CU to 32, and Max CU to 64.
                                                                                                            2. Click create to add a scaling policy. Set the priority to 20, Period to 10--23, Min CU to 64, and Max CU to 128.
                                                                                                            +

                                                                                                          6. Refer to Creating an Elastic Resource Pool and Creating Queues Within It to create the test_b queue and set the scaling policy.

                                                                                                            1. Set the priority of the default scaling policy to 5, Min CU to 32, and Max CU to 64.
                                                                                                            2. Click create to add a scaling policy. Set the priority to 20, Period to 10--23, Min CU to 64, and Max CU to 128.

                                                                                                          7. View the scaling policy on the Queue Management page of the specific elastic resource pool.

                                                                                                            Click to view graphical statistics on priorities and CU settings of the two queues for all time segments.

                                                                                                        diff --git a/docs/dli/umn/dli_01_0524.html b/docs/dli/umn/dli_01_0524.html index 0b2db1586..2675ec686 100644 --- a/docs/dli/umn/dli_01_0524.html +++ b/docs/dli/umn/dli_01_0524.html @@ -1,20 +1,20 @@

                                                                                                        Modifying Specifications

                                                                                                        -

                                                                                                        Scenario

                                                                                                        If the current specifications of your elastic resource pool do not meet your service needs, you can modify them using the change specifications function.

                                                                                                        +

                                                                                                        Scenario

                                                                                                        If the current specifications of your elastic resource pool do not meet your service needs, you can modify them using the change yearly/monthly CUs function.

                                                                                                        -

                                                                                                        Scaling Up

                                                                                                        1. In the navigation pane on the left of the console, choose Resources > Resource Pool.
                                                                                                        2. Select the elastic resource pool you want and choose More > Modify Specifications in the Operation column.
                                                                                                        3. In the Modify Specifications dialog page, set Operation to Scale-out and specify the number of CUs you want to add.
                                                                                                        4. Confirm the changes and click OK.
                                                                                                        5. Choose Job Management > SQL Jobs to view the status of the SCALE_POOL SQL job.

                                                                                                          If the job status is Scaling, the elastic resource pool is scaling up. Wait until the job status changes to Finished.

                                                                                                          +

                                                                                                          Scaling Out

                                                                                                          1. In the navigation pane on the left of the console, choose Resources > Resource Pool.
                                                                                                          2. Locate the elastic resource pool you want to scale out, click More in the Operation column, and select Modify Yearly/Monthly CU.
                                                                                                          3. On the Modify Yearly/Monthly CU page, set Operation to Scale-out and specify the number of CUs you want to add.
                                                                                                          4. Confirm the changes and click OK.
                                                                                                          5. Choose Job Management > SQL Jobs to view the status of the SCALE_POOL SQL job.

                                                                                                            If the job status is Scaling, the elastic resource pool is scaling out. Wait until the job status changes to Finished.

                                                                                                          -

                                                                                                          Scaling Down

                                                                                                          By default, the minimum number of CUs is 16. That is, when the specifications of an elastic resource pool are 16 CUs, you cannot scale the pool down.

                                                                                                          +

                                                                                                          Scaling In

                                                                                                          By default, the minimum number of CUs is 16. That is, when the specifications of an elastic resource pool are 16 CUs, you cannot scale the pool down.

                                                                                                          -
                                                                                                          1. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                          2. Select the elastic resource pool you want and choose More > Modify Specifications in the Operation column.
                                                                                                          3. In the Modify Specifications dialog page, set Operation to Scale-in and specify the number of CUs you want to add.
                                                                                                          4. Confirm the changes and click OK.
                                                                                                          5. Choose Job Management > SQL Jobs to view the status of the SCALE_POOL SQL job.

                                                                                                            If the job status is Scaling, the elastic resource pool is scaling down. Wait until the job status changes to Finished.

                                                                                                            +
                                                                                                            1. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                            2. Locate the elastic resource pool you want to scale in, click More in the Operation column, and select Modify Yearly/Monthly CU.
                                                                                                            3. On the Modify Yearly/Monthly CU page, set Operation to Scale-in and specify the number of CUs you want to decrease.
                                                                                                            4. Confirm the changes and click OK.
                                                                                                            5. Choose Job Management > SQL Jobs to view the status of the SCALE_POOL SQL job.

                                                                                                              If the job status is Scaling, the elastic resource pool is scaling in. Wait until the job status changes to Finished.

                                                                                                          -
                                                                                                          Parent topic: Regular Operations
                                                                                                          +
                                                                                                          Parent topic: Managing Elastic Resource Pools
                                                                                                          diff --git a/docs/dli/umn/dli_01_0525.html b/docs/dli/umn/dli_01_0525.html index f185f6514..616edbc1a 100644 --- a/docs/dli/umn/dli_01_0525.html +++ b/docs/dli/umn/dli_01_0525.html @@ -21,7 +21,7 @@
                                                                                                          • Click the text box and select a predefined tag key from the drop-down list.

                                                                                                            To add a predefined tag, you need to create one on TMS and then select it from the Tag key drop-down list. You can click View predefined tags to go to the Predefined Tags page of the TMS console. Then, click Create Tag in the upper corner of the page to create a predefined tag.

                                                                                                          -
                                                                                                          • Enter a tag key in the text box.
                                                                                                            NOTE:

                                                                                                            A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                                            +
                                                                                                            • Enter a tag key in the text box.
                                                                                                              NOTE:

                                                                                                              A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                                            @@ -29,7 +29,7 @@
                                                                                                        @@ -42,7 +42,7 @@
                                                                                                        -
                                                                                                        Parent topic: Regular Operations
                                                                                                        +
                                                                                                        Parent topic: Managing Elastic Resource Pools
                                                                                                        diff --git a/docs/dli/umn/dli_01_0526.html b/docs/dli/umn/dli_01_0526.html index 6716b3c05..e21007af0 100644 --- a/docs/dli/umn/dli_01_0526.html +++ b/docs/dli/umn/dli_01_0526.html @@ -39,7 +39,7 @@
                                                                                                        -
                                                                                                        Parent topic: Regular Operations
                                                                                                        +
                                                                                                        Parent topic: Managing Elastic Resource Pools
                                                                                                        diff --git a/docs/dli/umn/dli_01_0528.html b/docs/dli/umn/dli_01_0528.html index c6775822e..e86963f9b 100644 --- a/docs/dli/umn/dli_01_0528.html +++ b/docs/dli/umn/dli_01_0528.html @@ -1,19 +1,35 @@ -

                                                                                                        Before You Start

                                                                                                        +

                                                                                                        Managing Queues

                                                                                                        diff --git a/docs/dli/umn/dli_01_0529.html b/docs/dli/umn/dli_01_0529.html index 2357a99fa..58cee0404 100644 --- a/docs/dli/umn/dli_01_0529.html +++ b/docs/dli/umn/dli_01_0529.html @@ -1,25 +1,23 @@ -

                                                                                                        Regular Operations

                                                                                                        +

                                                                                                        Managing Elastic Resource Pools

                                                                                                        diff --git a/docs/dli/umn/dli_01_0530.html b/docs/dli/umn/dli_01_0530.html index 6cab58e6d..1e800bacd 100644 --- a/docs/dli/umn/dli_01_0530.html +++ b/docs/dli/umn/dli_01_0530.html @@ -15,7 +15,7 @@
                                                                                                        -
                                                                                                        Parent topic: Regular Operations
                                                                                                        +
                                                                                                        Parent topic: Managing Elastic Resource Pools
                                                                                                        diff --git a/docs/dli/umn/dli_01_0531.html b/docs/dli/umn/dli_01_0531.html index e31296813..1cc1499c2 100644 --- a/docs/dli/umn/dli_01_0531.html +++ b/docs/dli/umn/dli_01_0531.html @@ -1,34 +1,331 @@ -

                                                                                                        Creating and Submitting a Flink OpenSource SQL Job

                                                                                                        -

                                                                                                        Scenario

                                                                                                        DLI Flink jobs can use other cloud services as data sources and sink streams for real-time compute. This example describes how to create and submit a Flink Opensource SQL job that uses Kafka as the input stream and RDS as the output stream.

                                                                                                        +

                                                                                                        Using DLI to Submit a Flink OpenSource SQL Job to Query RDS for MySQL Data

                                                                                                        +

                                                                                                        Scenario

                                                                                                        DLI Flink jobs can use other cloud services as data sources and sink streams for real-time compute.

                                                                                                        +

                                                                                                        This example describes how to create and submit a Flink OpenSource SQL job that uses Kafka as the source stream and RDS as the sink stream.

                                                                                                        -

                                                                                                        Procedure

                                                                                                        You need to create a Flink OpenSource SQL job that has an input stream and an output stream. The input stream reads data from Kafka, and the output stream writes data into RDS. The procedure is as follows:

                                                                                                        -

                                                                                                        Step 1: Prepare a Data Source

                                                                                                        -

                                                                                                        Step 2: Prepare a Data Output Channel

                                                                                                        -

                                                                                                        Step 3: Create an OBS Bucket to Store Output Data

                                                                                                        -

                                                                                                        Step 4: Create a Queue

                                                                                                        -

                                                                                                        Step 5: Create an Enhanced Datasource Connection Between DLI and Kafka

                                                                                                        -

                                                                                                        Step 6: Create an Enhanced Datasource Connection Between DLI and RDS

                                                                                                        -

                                                                                                        Step 7: Create a Flink OpenSource SQL Job

                                                                                                        +

                                                                                                        Procedure

                                                                                                        You need to create a Flink OpenSource SQL job that has a source stream and a sink stream. The source stream reads data from Kafka, and the sink stream writes data to RDS for MySQL. Procedure shows the process.

                                                                                                        + +
                                                                                                        Table 2 CU allocation

                                                                                                        Scenario

                                                                                                        CUs

                                                                                                        +

                                                                                                        CUs

                                                                                                        An elastic resource pool has a maximum number of 256 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                                        -
                                                                                                        • Queue A: priority 5; period: 00:00–9:00; minimum CU: 32; maximum CU: 128
                                                                                                        • Queue B: priority 10; time period: 00:00–9:00; minimum CU: 64; maximum CU: 128
                                                                                                        +

                                                                                                        An elastic resource pool has a maximum number of 256 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                                        +
                                                                                                        • Queue A: priority 5; period: 00:00–09:00; minimum CU: 32; maximum CU: 128
                                                                                                        • Queue B: priority 10; time period: 00:00–09:00; minimum CU: 64; maximum CU: 128

                                                                                                        From 00:00 a.m. to 09:00 a.m.:

                                                                                                        +

                                                                                                        From 00:00 a.m. to 09:00 a.m.:

                                                                                                        1. The minimum CUs are allocated to the two queues. Queue A has 32 CUs, and queue B has 64 CUs. There are 160 CUs remaining.
                                                                                                        2. The remaining CUs are allocated based on the priorities. Queue B is prior to queue A. Therefore, queue B gets 64 CUs, and queue A has 96 CUs.

                                                                                                        An elastic resource pool has a maximum number of 96 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                                        -
                                                                                                        • Queue A: priority 5; period: 00:00–9:00; minimum CU: 32; maximum CU: 64
                                                                                                        • Queue B: priority 10; time period: 00:00–9:00; minimum CU: 64; maximum CU: 128
                                                                                                        +

                                                                                                        An elastic resource pool has a maximum number of 96 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                                        +
                                                                                                        • Queue A: priority 5; period: 00:00–09:00; minimum CU: 32; maximum CU: 64
                                                                                                        • Queue B: priority 10; time period: 00:00–09:00; minimum CU: 64; maximum CU: 128

                                                                                                        From 00:00 a.m. to 09:00 a.m.:

                                                                                                        +

                                                                                                        From 00:00 a.m. to 09:00 a.m.:

                                                                                                        1. The minimum CUs are allocated to the two queues. Queue A has 32 CUs, and queue B has 64 CUs. There are no remaining CUs.
                                                                                                        2. The allocation is complete.

                                                                                                        An elastic resource pool has a maximum number of 128 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                                        -
                                                                                                        • Queue A: priority 5; period: 00:00–9:00; minimum CU: 32; maximum CU: 64
                                                                                                        • Queue B: priority 10; time period: 00:00–9:00; minimum CU: 64; maximum CU: 128
                                                                                                        +

                                                                                                        An elastic resource pool has a maximum number of 128 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                                        +
                                                                                                        • Queue A: priority 5; period: 00:00–09:00; minimum CU: 32; maximum CU: 64
                                                                                                        • Queue B: priority 10; time period: 00:00–09:00; minimum CU: 64; maximum CU: 128

                                                                                                        From 00:00 a.m. to 09:00 a.m.:

                                                                                                        +

                                                                                                        From 00:00 a.m. to 09:00 a.m.:

                                                                                                        1. The minimum CUs are allocated to the two queues. Queue A has 32 CUs, and queue B has 64 CUs. There are 32 CUs remaining.
                                                                                                        2. The remaining 32 CUs are all preferentially allocated to queue B.

                                                                                                        An elastic resource pool has a maximum number of 128 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                                        -
                                                                                                        • Queue A: priority 5; period: 00:00–9:00; minimum CU: 32; maximum CU: 64
                                                                                                        • Queue B: priority 5; time period: 00:00–9:00; minimum CU: 64; maximum CU: 128
                                                                                                        +

                                                                                                        An elastic resource pool has a maximum number of 128 CUs for queue A and queue B. The scaling policies are as follows:

                                                                                                        +
                                                                                                        • Queue A: priority 5; period: 00:00–09:00; minimum CU: 32; maximum CU: 64
                                                                                                        • Queue B: priority 5; time period: 00:00–09:00; minimum CU: 64; maximum CU: 128

                                                                                                        From 00:00 a.m. to 09:00 a.m.:

                                                                                                        +

                                                                                                        From 00:00 a.m. to 09:00 a.m.:

                                                                                                        1. The minimum CUs are allocated to the two queues. Queue A has 32 CUs, and queue B has 64 CUs. There are 32 CUs remaining.
                                                                                                        2. The two queues have the same priority, the remaining 32 CUs are randomly allocated to the two queues.

                                                                                                        Tag value

                                                                                                        You can specify the tag value in either of the following ways:

                                                                                                        -
                                                                                                        • Click the text box and select a predefined tag value from the drop-down list.
                                                                                                        • Enter a tag value in the text box.
                                                                                                          NOTE:

                                                                                                          A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                                                                                          +
                                                                                                          • Click the text box and select a predefined tag value from the drop-down list.
                                                                                                          • Enter a tag value in the text box.
                                                                                                            NOTE:

                                                                                                            A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                                                                                                        + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                        Table 1 Procedure for using DLI to submit a Flink OpenSource SQL job to query RDS for MySQL data

                                                                                                        Procedure

                                                                                                        +

                                                                                                        Description

                                                                                                        +

                                                                                                        Step 1: Prepare a Source Stream

                                                                                                        +

                                                                                                        In this example, a Kafka instance is created as the data source.

                                                                                                        +

                                                                                                        Step 2: Prepare a Sink Stream

                                                                                                        +

                                                                                                        In this example, an RDS for MySQL DB instance is created as the data destination.

                                                                                                        +

                                                                                                        Step 3: Create an OBS Bucket to Store Output Data

                                                                                                        +

                                                                                                        Create an OBS bucket to store checkpoints, job logs, and debugging test data for the Flink OpenSource SQL job.

                                                                                                        +

                                                                                                        Step 4: Create an Elastic Resource Pool and Add Queues to the Pool

                                                                                                        +

                                                                                                        Create compute resources required for submitting the Flink OpenSource SQL job.

                                                                                                        +

                                                                                                        Step 5: Create an Enhanced Datasource Connection Between DLI and Kafka

                                                                                                        +

                                                                                                        Create an enhanced datasource connection to connect the DLI elastic resource pool and the Kafka instance.

                                                                                                        +

                                                                                                        Step 6: Create an Enhanced Datasource Connection Between DLI and RDS

                                                                                                        +

                                                                                                        Create an enhanced datasource connection to connect the DLI elastic resource pool and the RDS for MySQL DB instance.

                                                                                                        +

                                                                                                        Step 7: Create a Flink OpenSource SQL Job

                                                                                                        +

                                                                                                        Once you have prepared a source stream and a sink stream, you can create a Flink OpenSource SQL job to analyze the data.

                                                                                                        +
                                                                                                        -

                                                                                                        Step 1: Prepare a Data Source

                                                                                                        In this example, Kafka is the data source.

                                                                                                        -

                                                                                                        For more information about Flink job data, see Preparing Flink Job Data.

                                                                                                        -

                                                                                                        Enable DIS to import Kafka data to DLI. For details, see "Buying a Kafka Instance" in Distributed Message Service Kafka User Guide.

                                                                                                        -
                                                                                                        1. Create the dependent Kafka resources.
                                                                                                          Before creating a Kafka instance, ensure the availability of resources, including a virtual private cloud (VPC), subnet, security group, and security group rules.
                                                                                                          • For details about how to create a VPC and subnet, see "Creating a VPC and Subnet" in Virtual Private Cloud User Guide. For details about how to create and use a subnet in an existing VPC, see "Create a Subnet for the VPC" in Virtual Private Cloud User Guide.
                                                                                                            • The created VPC and the Kafka instance you will create must be in the same region.
                                                                                                            • Retain the default settings unless otherwise specified.
                                                                                                            +
                                                                                                            +

                                                                                                            Step 1: Prepare a Source Stream

                                                                                                            In this example, Kafka is the source stream.

                                                                                                            +

                                                                                                            Enable DIS to import Kafka data to DLI. For details, see "Buying a Kafka Instance" in the Distributed Message Service Kafka User Guide.

                                                                                                            +
                                                                                                            1. Create the dependent Kafka resources.
                                                                                                              Before creating a Kafka instance, ensure the availability of resources, including a virtual private cloud (VPC), subnet, security group, and security group rules.
                                                                                                              • For how to create a VPC and subnet, see "Creating a VPC and Subnet" in the Virtual Private Cloud User Guide. For how to create and use a subnet in an existing VPC, see "Create a Subnet for the VPC" in the Virtual Private Cloud User Guide.
                                                                                                                • The created VPC and the Kafka instance you will create must be in the same region.
                                                                                                                • Retain the default settings unless otherwise specified.
                                                                                                                -
                                                                                                              • For details about how to create a security group, see "Creating a Security Group" in the Virtual Private Cloud User Guide. For details about how to add rules to a security group, see "Creating a Subnet for the VPC" in the Virtual Private Cloud User Guide.
                                                                                                              +
                                                                                                            2. For how to create a security group, see "Creating a Security Group" in the Virtual Private Cloud User Guide. For how to add rules to a security group, see "Creating a Subnet for the VPC" in the Virtual Private Cloud User Guide.
                                                                                                          -

                                                                                                          For more information, see in Distributed Message Service for Kafka User Guide.

                                                                                                          -
                                                                                                        2. Create a DMS for Kafka Instance for job input streams.
                                                                                                          1. Log in to the DMS for Kafka console.
                                                                                                          2. Select a region in the upper left corner.
                                                                                                          3. On the DMS for Kafka page, click Buy Instance in the upper right corner and set related parameters. The required instance information is as follows:
                                                                                                            • Region: Select the region where DLI is located.
                                                                                                            • Project: Keep the default value.
                                                                                                            • AZ: Keep the default value.
                                                                                                            • Instance Name: kafka-dliflink
                                                                                                            • Specifications: Default
                                                                                                            • Enterprise Project: default
                                                                                                            • Version: Keep the default value.
                                                                                                            • CPU Architecture: Keep the default value.
                                                                                                            • Broker Flavor: Select a flavor as needed.
                                                                                                            • Brokers: Retain the default value.
                                                                                                            • Storage Space: Keep the default value.
                                                                                                            • Capacity Threshold Policy: Keep the default value.
                                                                                                            • VPC and Subnet: Select the VPC and subnet created in 1.
                                                                                                            • Security Group: Select the security group created in 1.
                                                                                                            • Manager Username: Enter dliflink (used to log in to the instance management page).
                                                                                                            • Password: **** (The system cannot detect your password.)
                                                                                                            • Confirm Password: ****
                                                                                                            • More Settings: Do not configure this parameter.
                                                                                                            -
                                                                                                          4. Click Buy. The confirmation page is displayed.
                                                                                                          5. Confirm that the instance information is correct, read and agree to the , and click Submit. It takes about 10 to 15 minutes to create an instance.
                                                                                                          +
                                                                                                        3. Create a Kafka premium instance as the job source stream.
                                                                                                          1. Log in to the DMS for Kafka management console.
                                                                                                          2. Select a region in the upper left corner.
                                                                                                          3. On the DMS for Kafka page, click Buy Instance in the upper right corner and set related parameters. The required instance information is as follows: +
                                                                                                            + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                            Table 2 Parameters for buying a DMS for Kafka instance

                                                                                                            Parameter

                                                                                                            +

                                                                                                            Description

                                                                                                            +

                                                                                                            Example Value

                                                                                                            +

                                                                                                            Region

                                                                                                            +

                                                                                                            DMS for Kafka instances in different regions cannot communicate with each other over an intranet. For lower network latency and quick resource access, select the region nearest to you.

                                                                                                            +

                                                                                                            _

                                                                                                            +

                                                                                                            AZ

                                                                                                            +

                                                                                                            An AZ is a physical location that uses an independent power supply and network. AZs within the same region can communicate with each other over an intranet.

                                                                                                            +

                                                                                                            AZ 1, AZ 2, and AZ 3

                                                                                                            +

                                                                                                            Version

                                                                                                            +

                                                                                                            Kafka version, which cannot be changed after the instance is created.

                                                                                                            +

                                                                                                            3.x

                                                                                                            +

                                                                                                            Specifications

                                                                                                            +

                                                                                                            The options are Cluster and Single-node.

                                                                                                            +

                                                                                                            Cluster

                                                                                                            +

                                                                                                            Broker Flavor

                                                                                                            +

                                                                                                            Select a broker flavor based on service requirements.

                                                                                                            +

                                                                                                            kafka.2u4g.cluster.small

                                                                                                            +

                                                                                                            Brokers

                                                                                                            +

                                                                                                            Specify the broker quantity.

                                                                                                            +

                                                                                                            3

                                                                                                            +

                                                                                                            Storage Space per Broker

                                                                                                            +

                                                                                                            Select a disk type and specify the disk size. You cannot change the disk type once the instance is created.

                                                                                                            +

                                                                                                            High I/O | 100 GB

                                                                                                            +

                                                                                                            VPC

                                                                                                            +

                                                                                                            Select a created or shared VPC.

                                                                                                            +

                                                                                                            Select the VPC created in 1.

                                                                                                            +

                                                                                                            Subnet

                                                                                                            +

                                                                                                            Select a created or shared subnet.

                                                                                                            +

                                                                                                            Once the instance is created, its subnet cannot be changed.

                                                                                                            +

                                                                                                            Select the subnet created in 1.

                                                                                                            +

                                                                                                            Security Group

                                                                                                            +

                                                                                                            Select a created security group.

                                                                                                            +

                                                                                                            Select the security group created in 1.

                                                                                                            +

                                                                                                            Access Mode

                                                                                                            +

                                                                                                            There are two modes:

                                                                                                            +
                                                                                                            • Plaintext Access: SASL authentication is not conducted when a client connects to the Kafka instance.
                                                                                                            • Ciphertext Access: SASL authentication is conducted when a client connects to the Kafka instance.
                                                                                                            +

                                                                                                            Plaintext Access

                                                                                                            +

                                                                                                            Instance Name

                                                                                                            +

                                                                                                            Enter an instance name based on the naming rule.

                                                                                                            +

                                                                                                            kafka-dliflink

                                                                                                            +

                                                                                                            Enterprise Project

                                                                                                            +

                                                                                                            An enterprise project organizes cloud resources into groups, allowing you to manage both resources and members by project. The default project is default.

                                                                                                            +

                                                                                                            default

                                                                                                            +
                                                                                                            +
                                                                                                            +
                                                                                                          4. Click Buy.
                                                                                                          5. Confirm that the instance information is correct, read and agree to the , and click Submit. It takes about 10 to 15 minutes to create an instance.
                                                                                                        4. Create a Kafka topic.
                                                                                                          1. Click the name of the created Kafka instance. The basic information page of the instance is displayed.
                                                                                                          2. Choose Topics in the navigation pane on the left. On the displayed page, click Create Topic. Configure the following parameters:
                                                                                                            • Topic Name: For this example, enter testkafkatopic.
                                                                                                            • Partitions: Set the value to 1.
                                                                                                            • Replicas: Set the value to 1.

                                                                                                            Retain the default values for other parameters.

                                                                                                        -

                                                                                                        Step 2: Prepare a Data Output Channel

                                                                                                        To use RDS as the data output channel, create an RDS MySQL instance. For details, see "Getting Started with RDS for MySQL" in Getting Started with Relational Database Service.

                                                                                                        -
                                                                                                        1. Log in to the RDS management console.
                                                                                                        2. Select a region in the upper left corner.
                                                                                                        3. Click Buy DB Instance in the upper right corner of the page and set related parameters. Retain the default values for other parameters.
                                                                                                          • Region: Select the region where DLI is located.
                                                                                                          • DB Instance Name: Enter rds-dliflink.
                                                                                                          • DB Engine: Select MySQL.
                                                                                                          • DB Engine Version: Select 8.0.
                                                                                                          • DB Instance Type: Select Primary/Standby.
                                                                                                          • Storage Type: Cloud SSD may be selected by default.
                                                                                                          • Primary AZ: Select a custom AZ.
                                                                                                          • Standby AZ: Select a custom AZ.
                                                                                                          • Instance Class: Select a class as needed and choose 2 vCPUs | 8 GB.
                                                                                                          • Storage Space (GB): Set it to 40.
                                                                                                          • VPC: Select the VPC and subnet created in 1.
                                                                                                          • Database Port: Enter 3306.
                                                                                                          • Security Group: Select the security group created in 1.
                                                                                                          • Administrator Password: **** (Keep the password secure. The system cannot retrieve your password.)
                                                                                                          • Confirm Password: ****
                                                                                                          • Parameter Template: Choose Default-MySQL-8.0.
                                                                                                          +

                                                                                                          Step 2: Prepare a Sink Stream

                                                                                                          Use RDS for MySQL as the data sink stream and create an RDS for MySQL DB instance.

                                                                                                          +
                                                                                                          1. Log in to the RDS management console.
                                                                                                          2. Select a region in the upper left corner.
                                                                                                          3. Click Buy DB Instance in the upper right corner of the page and set related parameters. Retain the default values for other parameters.

                                                                                                            For the parameters, see "RDS for MySQL Getting Started" in the Relational Database Service Getting Started.

                                                                                                            + +
                                                                                                            + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                            Table 3 RDS for MySQL instance parameters

                                                                                                            Parameter

                                                                                                            +

                                                                                                            Description

                                                                                                            +

                                                                                                            Example Value

                                                                                                            +

                                                                                                            Region

                                                                                                            +

                                                                                                            Select the region where DLI is.

                                                                                                            +

                                                                                                            _

                                                                                                            +

                                                                                                            Instance Name

                                                                                                            +

                                                                                                            Instance name

                                                                                                            +

                                                                                                            rds-dliflink

                                                                                                            +

                                                                                                            DB Engine

                                                                                                            +

                                                                                                            MySQL

                                                                                                            +

                                                                                                            MySQL

                                                                                                            +

                                                                                                            DB Engine Version

                                                                                                            +

                                                                                                            If you select MySQL for DB Engine, select an engine version that best suits your service needs. You are advised to select the latest available version for more stable performance, higher security, and greater reliability.

                                                                                                            +

                                                                                                            8.0

                                                                                                            +

                                                                                                            DB Instance Type

                                                                                                            +

                                                                                                            Primary/standby mode of the DB instance

                                                                                                            +

                                                                                                            Primary/Standby

                                                                                                            +

                                                                                                            Storage Type

                                                                                                            +

                                                                                                            Determines the instance read/write speed. The higher the maximum throughput, the faster the read and write speeds.

                                                                                                            +

                                                                                                            Cloud SSD

                                                                                                            +

                                                                                                            AZ

                                                                                                            +

                                                                                                            For a single DB instance, you only need to select a single AZ.

                                                                                                            +

                                                                                                            Custom

                                                                                                            +

                                                                                                            Time Zone

                                                                                                            +

                                                                                                            Select a time zone based on the region you selected. You can change it after the DB instance is created.

                                                                                                            +

                                                                                                            Retain the default value.

                                                                                                            +

                                                                                                            Instance Class

                                                                                                            +

                                                                                                            vCPUs and memory. These instance classes support varying number of connections and maximum IOPS.

                                                                                                            +

                                                                                                            2 vCPUs | 4 GB

                                                                                                            +

                                                                                                            Storage Space

                                                                                                            +

                                                                                                            If the storage type is cloud SSD or extreme SSD, you can enable storage autoscaling. If the available storage drops to a specified threshold, autoscaling is triggered.

                                                                                                            +

                                                                                                            40 GB

                                                                                                            +

                                                                                                            Disk Encryption

                                                                                                            +

                                                                                                            Determine whether to enable disk encryption.

                                                                                                            +

                                                                                                            Disabled

                                                                                                            +

                                                                                                            VPC and Subnet

                                                                                                            +

                                                                                                            Select an existing VPC and subnet.

                                                                                                            +

                                                                                                            For how to recreate a VPC and subnet, refer to "Creating a VPC and Subnet" in the Virtual Private Cloud User Guide.

                                                                                                            +
                                                                                                            NOTE:

                                                                                                            In datasource scenarios, the CIDR block of the data source cannot overlap that of the elastic resource pool.

                                                                                                            +
                                                                                                            +

                                                                                                            Select the VPC and subnet created in 1.

                                                                                                            +

                                                                                                            Database Port

                                                                                                            +

                                                                                                            Port 3306 is used by default.

                                                                                                            +

                                                                                                            3306

                                                                                                            +

                                                                                                            Security Group

                                                                                                            +

                                                                                                            Enhances security by providing rules that control access to RDS from other services.

                                                                                                            +

                                                                                                            The security group where the data source is must allow access from the CIDR block of the DLI elastic resource pool.

                                                                                                            +

                                                                                                            Select the security group created in 1.

                                                                                                            +

                                                                                                            Password

                                                                                                            +

                                                                                                            Set a password for logging in to the DB instance.

                                                                                                            +

                                                                                                            -

                                                                                                            +

                                                                                                            Administrator

                                                                                                            +

                                                                                                            root

                                                                                                            +

                                                                                                            root

                                                                                                            +

                                                                                                            Administrator Password

                                                                                                            +

                                                                                                            Administrator password

                                                                                                            +

                                                                                                            -

                                                                                                            +

                                                                                                            Parameter Template

                                                                                                            +

                                                                                                            A template of parameters for creating an instance. The template contains engine configuration values that are applied to one or more instances.

                                                                                                            +

                                                                                                            Default-MySQL-8.0

                                                                                                            +

                                                                                                            Enterprise Project

                                                                                                            +

                                                                                                            If the instance has been associated with an enterprise project, select the target project from the Enterprise Project drop-down list.

                                                                                                            +

                                                                                                            default

                                                                                                            +

                                                                                                            Quantity

                                                                                                            +

                                                                                                            Number of instances to buy

                                                                                                            +

                                                                                                            1

                                                                                                            +
                                                                                                            +
                                                                                                          4. Click Next and confirm the specifications.
                                                                                                          5. Click Submit. The RDS DB instance is created.
                                                                                                          6. Log in to the MySQL database and create table orders in database flink.
                                                                                                            Log in to the MySQL instance, click the flink database. On the displayed page, click SQL Window. Enter the following table creation statement in the SQL editing pane to create a table.
                                                                                                            CREATE TABLE `flink`.`orders` (
 	`order_id` VARCHAR(32) NOT NULL,
 	`order_channel` VARCHAR(32) NULL,
@@ -46,39 +343,222 @@
                                                                                                          -

                                                                                                          Step 3: Create an OBS Bucket to Store Output Data

                                                                                                          In this step, you need to enable OBS for JobSample and enable checkpointing for the DLI Flink job to save job logs and test data storage.

                                                                                                          -

                                                                                                          For details about how to create a bucket, see "Creating a Bucket" in the Object Storage Service Console Operation Guide.

                                                                                                          -
                                                                                                          1. In the navigation pane on the OBS management console, choose Object Storage.
                                                                                                          2. In the upper right corner of the page, click Create Bucket and set bucket parameters.
                                                                                                            • Region: Select the region where DLI is located.
                                                                                                            • Bucket Name: Enter a bucket name. For this example, enter obstest.
                                                                                                            • Default Storage Class: Standard
                                                                                                            • Bucket Policy: Private
                                                                                                            • Default Encryption: Do not enable
                                                                                                            • Direct Reading: Do not enable
                                                                                                            • Enterprise Project: default
                                                                                                            • Tags: Leave it blank.
                                                                                                            +

                                                                                                            Step 3: Create an OBS Bucket to Store Output Data

                                                                                                            In this example, you need to enable OBS for job JobSample to provide DLI Flink jobs with the functions of checkpointing, saving job logs, and commissioning test data.

                                                                                                            +

                                                                                                            For how to create a bucket, see "Creating a Bucket" in the Object Storage Service Console Operation Guide.

                                                                                                            +
                                                                                                            1. In the navigation pane on the OBS management console, choose Object Storage.
                                                                                                            2. In the upper right corner of the page, click Create Bucket and set bucket parameters. +
                                                                                                              + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                              Table 4 OBS bucket parameters

                                                                                                              Parameter

                                                                                                              +

                                                                                                              Description

                                                                                                              +

                                                                                                              Example Value

                                                                                                              +

                                                                                                              Region

                                                                                                              +

                                                                                                              Geographic area where a bucket resides. Select the region where DLI is.

                                                                                                              +

                                                                                                              _

                                                                                                              +

                                                                                                              Bucket Name

                                                                                                              +

                                                                                                              Name of the bucket. The name must be unique across all regions and accounts. Once a bucket is created, its name cannot be changed.

                                                                                                              +

                                                                                                              obstest

                                                                                                              +

                                                                                                              Storage Class

                                                                                                              +

                                                                                                              Storage class of the bucket. You can choose a storage class that meets your needs for storage performance and costs.

                                                                                                              +

                                                                                                              Standard

                                                                                                              +

                                                                                                              Bucket Policies

                                                                                                              +

                                                                                                              Controls read and write permissions for a bucket.

                                                                                                              +

                                                                                                              If you select Private, only users granted permissions by the bucket ACL can access the bucket.

                                                                                                              +

                                                                                                              Private

                                                                                                              +

                                                                                                              Enterprise Project

                                                                                                              +

                                                                                                              You can add the bucket to an enterprise project for unified management.

                                                                                                              +

                                                                                                              default

                                                                                                              +
                                                                                                              +
                                                                                                            3. Click Create Now.
                                                                                                            -

                                                                                                            Step 4: Create a Queue

                                                                                                            Flink OpenSource SQL jobs cannot run on the default queue. You need to create a queue, for example, Flinktest. For details, see "Creating a Queue".

                                                                                                            -
                                                                                                            1. Log in to the DLI management console. On the Overview page, click Buy Queue in the upper right corner.

                                                                                                              If this is your first time logging in to the DLI management console, you need to be authorized to access OBS.

                                                                                                              -
                                                                                                            2. Configure the following parameters:
                                                                                                              • Name: Flinktest
                                                                                                              • Type: For general purpose. Select Dedicated Resource Mode.
                                                                                                              • Specifications: 16 CUs
                                                                                                              • Enterprise Project: default
                                                                                                              • Description: Leave it blank.
                                                                                                              • Advanced Settings: Custom
                                                                                                              • CIDR Block: Set a CIDR block that does not conflict with the Kafka instance's CIDR block.
                                                                                                              -
                                                                                                            3. Click Buy and confirm the configuration.
                                                                                                            4. Submit the request.

                                                                                                              It takes 10 to 15 minutes to bind the queue to a cluster after the queue is created.

                                                                                                              -
                                                                                                            +

                                                                                                            Step 4: Create an Elastic Resource Pool and Add Queues to the Pool

                                                                                                            To create a Flink OpenSource SQL job, you must use your own queue as the existing default queue cannot be used. In this example, create an elastic resource pool named dli_resource_pool and a queue named dli_queue_01.

                                                                                                            +
                                                                                                            1. Log in to the DLI management console.
                                                                                                            2. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                            3. On the displayed page, click Buy Resource Pool in the upper right corner.
                                                                                                            4. On the displayed page, set the parameters.
                                                                                                              Table 5 describes the parameters. +
                                                                                                              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                              Table 5 Parameters

                                                                                                              Parameter

                                                                                                              +

                                                                                                              Description

                                                                                                              +

                                                                                                              Example Value

                                                                                                              +

                                                                                                              Region

                                                                                                              +

                                                                                                              Select a region where you want to buy the elastic resource pool.

                                                                                                              +

                                                                                                              _

                                                                                                              +

                                                                                                              Project

                                                                                                              +

                                                                                                              Project uniquely preset by the system for each region

                                                                                                              +

                                                                                                              Default

                                                                                                              +

                                                                                                              Name

                                                                                                              +

                                                                                                              Name of the elastic resource pool

                                                                                                              +

                                                                                                              dli_resource_pool

                                                                                                              +

                                                                                                              Specifications

                                                                                                              +

                                                                                                              Specifications of the elastic resource pool

                                                                                                              +

                                                                                                              Standard

                                                                                                              +

                                                                                                              CU Range

                                                                                                              +

                                                                                                              The maximum and minimum CUs allowed for the elastic resource pool

                                                                                                              +

                                                                                                              64-64

                                                                                                              +

                                                                                                              CIDR Block

                                                                                                              +

                                                                                                              CIDR block the elastic resource pool belongs to. If you use an enhanced datasource connection, this CIDR block cannot overlap that of the data source. Once set, this CIDR block cannot be changed.

                                                                                                              +

                                                                                                              172.16.0.0/19

                                                                                                              +

                                                                                                              Enterprise Project

                                                                                                              +

                                                                                                              Select an enterprise project for the elastic resource pool.

                                                                                                              +

                                                                                                              default

                                                                                                              +
                                                                                                              -

                                                                                                              Step 5: Create an Enhanced Datasource Connection Between DLI and Kafka

                                                                                                              You need to create an enhanced datasource connection for the Flink OpenSource SQL job. For details, see "Creating an Enhanced Datasource Connection".

                                                                                                              +
                                                                                                              +
                                                                                                            5. Click Buy.
                                                                                                            6. Click Submit.
                                                                                                            7. In the elastic resource pool list, locate the pool you just created and click Add Queue in the Operation column.
                                                                                                            8. Set the basic parameters listed below. +
                                                                                                              + + + + + + + + + + + + + + + + + +
                                                                                                              Table 6 Basic parameters for adding a queue

                                                                                                              Parameter

                                                                                                              +

                                                                                                              Description

                                                                                                              +

                                                                                                              Example Value

                                                                                                              +

                                                                                                              Name

                                                                                                              +

                                                                                                              Name of the queue to add

                                                                                                              +

                                                                                                              dli_queue_01

                                                                                                              +

                                                                                                              Type

                                                                                                              +

                                                                                                              Type of the queue

                                                                                                              +
                                                                                                              • To execute SQL jobs, select For SQL.
                                                                                                              • To execute Flink or Spark jobs, select For general purpose.
                                                                                                              +

                                                                                                              _

                                                                                                              +

                                                                                                              Enterprise Project

                                                                                                              +

                                                                                                              Select an enterprise project.

                                                                                                              +

                                                                                                              default

                                                                                                              +
                                                                                                              +
                                                                                                              +
                                                                                                            9. Click Next and configure scaling policies for the queue.

                                                                                                              Click Create to add a scaling policy with varying priority, period, minimum CUs, and maximum CUs.

                                                                                                              +

                                                                                                              + +
                                                                                                              + + + + + + + + + + + + + + + + + + + + + +
                                                                                                              Table 7 Scaling policy parameters

                                                                                                              Parameter

                                                                                                              +

                                                                                                              Description

                                                                                                              +

                                                                                                              Example Value

                                                                                                              +

                                                                                                              Priority

                                                                                                              +

                                                                                                              Priority of the scaling policy in the current elastic resource pool. A larger value indicates a higher priority. In this example, only one scaling policy is configured, so its priority is set to 1 by default.

                                                                                                              +

                                                                                                              1

                                                                                                              +

                                                                                                              Period

                                                                                                              +

                                                                                                              The first scaling policy is the default policy, and its Period parameter configuration cannot be deleted or modified.

                                                                                                              +

                                                                                                              The period for the scaling policy is from 00 to 24.

                                                                                                              +

                                                                                                              00–24

                                                                                                              +

                                                                                                              Min CU

                                                                                                              +

                                                                                                              Minimum number of CUs allowed by the scaling policy

                                                                                                              +

                                                                                                              16

                                                                                                              +

                                                                                                              Max CU

                                                                                                              +

                                                                                                              Maximum number of CUs allowed by the scaling policy

                                                                                                              +

                                                                                                              64

                                                                                                              +
                                                                                                              +
                                                                                                              +
                                                                                                            10. Click OK.
                                                                                                            +
                                                                                                            +

                                                                                                            Step 5: Create an Enhanced Datasource Connection Between DLI and Kafka

                                                                                                            You need to create an enhanced datasource connection for the Flink OpenSource SQL job. For details, see "Datasource Connections" > "Creating an Enhanced Datasource Connection" in the Data Lake Insight User Guide.

                                                                                                            • The CIDR block of the DLI queue bound with a datasource connection cannot overlap with the CIDR block of the data source.
                                                                                                            • Datasource connections cannot be created for the default queue.
                                                                                                            • To access a table across data sources, you need to use a queue bound to a datasource connection.
                                                                                                            -
                                                                                                            1. Create a Kafka security group rule to allow access from the CIDR block of the DLI queue.

                                                                                                              1. On the Kafka management console, click an instance name on the DMS for Kafka page. Basic information of the Kafka instance is displayed.
                                                                                                              2. In the Connection pane, obtain the Instance Address (Private Network). In the Network pane, obtain the VPC and subnet of the instance.
                                                                                                              3. Click the security group name in the Network pane. On the displayed page, click the Inbound Rules tab and add a rule to allow access from the DLI queue.

                                                                                                                For example, if the CIDR block of the queue is 10.0.0.0/16, set Protocol to TCP, Type to IPv4, Source to 10.0.0.0/16, and click OK.

                                                                                                                +
                                                                                                                1. Create a Kafka security group rule to allow access from the CIDR block of the DLI queue.

                                                                                                                  1. On the Kafka management console, click an instance name on the DMS for Kafka page. Basic information of the Kafka instance is displayed.
                                                                                                                  2. In the Connection pane, obtain the Instance Address (Private Network). In the Network pane, obtain the VPC and subnet of the instance.
                                                                                                                  3. Click the security group name in the Network pane. On the displayed page, click the Inbound Rules tab and add a rule to allow access from the DLI queue.

                                                                                                                    For example, if the CIDR block of the queue is 10.0.0.0/16, set Protocol to TCP, Type to IPv4, Source to 10.0.0.0/16, and click OK.

                                                                                                                  -

                                                                                                                2. Create an enhanced datasource connection to Kafka.

                                                                                                                  1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
                                                                                                                  2. In the displayed dialog box, set the following parameters: For details, see the following section:
                                                                                                                    • Connection Name: Name of the enhanced datasource connection For this example, enter dli_kafka.
                                                                                                                    • Resource Pool: Select the name of the queue created in Step 4: Create a Queue.
                                                                                                                    • VPC: Select the VPC of the Kafka instance.
                                                                                                                    • Subnet: Select the subnet of Kafka instance.
                                                                                                                    • Set other parameters as you need.
                                                                                                                    +

                                                                                                                  3. Create an enhanced datasource connection to Kafka.

                                                                                                                    1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
                                                                                                                    2. In the displayed dialog box, set the following parameters: For details, see the following section:
                                                                                                                      • Connection Name: Name of the enhanced datasource connection For this example, enter dli_kafka.
                                                                                                                      • Resource Pool: Select the elastic resource pool created in Step 4: Create an Elastic Resource Pool and Add Queues to the Pool.
                                                                                                                      • VPC: Select the VPC of the Kafka instance.
                                                                                                                      • Subnet: Select the subnet of Kafka instance.
                                                                                                                      • Set other parameters as you need.

                                                                                                                      Click OK. Click the name of the created datasource connection to view its status. You can perform subsequent steps only after the connection status changes to Active.

                                                                                                                      -
                                                                                                                    3. Choose Resources > Queue Management and locate the queue created in Step 4: Create a Queue. In the Operation column, click More and select Test Address Connectivity.
                                                                                                                    4. In the displayed dialog box, enter Kafka instance address (private network):port in the Address box and click Test to check whether the instance is reachable. Note that multiple addresses must be tested separately.
                                                                                                                    +
                                                                                                                  4. Choose Resources > Queue Management and locate the queue created in Step 4: Create an Elastic Resource Pool and Add Queues to the Pool. In the Operation column, click More and select Test Address Connectivity.
                                                                                                                  5. In the displayed dialog box, enter Kafka instance address (private network):port in the Address box and click Test to check whether the instance is reachable. Note that multiple addresses must be tested separately.

                                                                                                            -

                                                                                                            Step 6: Create an Enhanced Datasource Connection Between DLI and RDS

                                                                                                            1. Create an RDS security group rule to allow access from CIDR block of the DLI queue.

                                                                                                              If the RDS DB instance and Kafka instance are in the same security group of the same VPC, skip this step. Access from the DLI queue has been allowed in 1.
                                                                                                              1. Go to the RDS console, click the name of the target RDS for MySQL DB instance on the Instances page. Basic information of the instance is displayed.
                                                                                                              2. In the Connection Information pane, obtain the floating IP address, database port, VPC, and subnet.
                                                                                                              3. Click the security group name. On the displayed page, click the Inbound Rules tab and add a rule to allow access from the DLI queue. For example, if the CIDR block of the queue is 10.0.0.0/16, set Priority to 1, Action to Allow, Protocol to TCP, Type to IPv4, Source to 10.0.0.0/16, and click OK.
                                                                                                              +

                                                                                                              Step 6: Create an Enhanced Datasource Connection Between DLI and RDS

                                                                                                              1. Create an RDS security group rule to allow access from CIDR block of the DLI queue.

                                                                                                                If the RDS DB instance and Kafka instance are in the same security group of the same VPC, skip this step. Access from the DLI queue has been allowed in 1.
                                                                                                                1. Go to the RDS console, click the name of the target RDS for MySQL DB instance on the Instances page. Basic information of the instance is displayed.
                                                                                                                2. In the Connection Information pane, obtain the floating IP address, database port, VPC, and subnet.
                                                                                                                3. Click the security group name. On the displayed page, click the Inbound Rules tab and add a rule to allow access from the DLI queue. For example, if the CIDR block of the queue is 10.0.0.0/16, set Priority to 1, Action to Allow, Protocol to TCP, Type to IPv4, Source to 10.0.0.0/16, and click OK.

                                                                                                              2. Create an enhanced datasource connection to RDS.

                                                                                                                If the RDS DB instance and Kafka instance are in the same VPC and subnet, skip this step. The enhanced datasource connection created in 2 has connected the subnet.

                                                                                                                -
                                                                                                                If the two instances are in different VPCs or subnets, perform the following steps to create an enhanced datasource connection:
                                                                                                                1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
                                                                                                                2. In the displayed dialog box, set the following parameters: For details, see the following section:
                                                                                                                  • Connection Name: Name of the enhanced datasource connection For this example, enter dli_rds.
                                                                                                                  • Resource Pool: Select the name of the queue created in Step 4: Create a Queue.
                                                                                                                  • VPC: Select the VPC of the RDS DB instance.
                                                                                                                  • Subnet: Select the subnet of RDS DB instance.
                                                                                                                  • Set other parameters as you need.
                                                                                                                  +
                                                                                                                  If the two instances are in different VPCs or subnets, perform the following steps to create an enhanced datasource connection:
                                                                                                                  1. Log in to the DLI management console. In the navigation pane on the left, choose Datasource Connections. On the displayed page, click Create in the Enhanced tab.
                                                                                                                  2. In the displayed dialog box, set the following parameters: For details, see the following section:
                                                                                                                    • Connection Name: Name of the enhanced datasource connection For this example, enter dli_rds.
                                                                                                                    • Resource Pool: Select the name of the queue created in Step 4: Create an Elastic Resource Pool and Add Queues to the Pool.
                                                                                                                    • VPC: Select the VPC of the RDS DB instance.
                                                                                                                    • Subnet: Select the subnet of RDS DB instance.
                                                                                                                    • Set other parameters as you need.

                                                                                                                    Click OK. Click the name of the created datasource connection to view its status. You can perform subsequent steps only after the connection status changes to Active.

                                                                                                                    -
                                                                                                                  3. Choose Resources > Queue Management and locate the queue created in Step 4: Create a Queue. In the Operation column, click More and select Test Address Connectivity.
                                                                                                                  4. In the displayed dialog box, enter floating IP address:database port of the RDS DB instance in the Address box and click Test to check whether the database is reachable.
                                                                                                                  +
                                                                                                                3. Choose Resources > Queue Management and locate the queue created in Step 4: Create an Elastic Resource Pool and Add Queues to the Pool. In the Operation column, click More and select Test Address Connectivity.
                                                                                                                4. In the displayed dialog box, enter Floating IP address:Database port of the RDS for MySQL DB instance in the Address box and click Test to check if the instance is reachable.

                                                                                                              -

                                                                                                              Step 7: Create a Flink OpenSource SQL Job

                                                                                                              After the data source and data output channel are prepared, you can create a Flink OpenSource SQL job.

                                                                                                              +

                                                                                                              Step 7: Create a Flink OpenSource SQL Job

                                                                                                              After the source and sink streams are prepared, you can create a Flink OpenSource SQL job.

                                                                                                              1. In the left navigation pane of the DLI management console, choose Job Management > Flink Jobs. The Flink Jobs page is displayed.
                                                                                                              2. In the upper right corner of the Flink Jobs page, click Create Job. Set the following parameters:
                                                                                                                • Type: Flink OpenSource SQL
                                                                                                                • Name: JobSample
                                                                                                                • Description: Leave it blank.
                                                                                                                • Template Name: Do not select any template.
                                                                                                                -
                                                                                                              3. Click OK to enter the editing page.
                                                                                                              4. Set job running parameters. The mandatory parameters are as follows:
                                                                                                                • Queue: Flinktest
                                                                                                                • Flink Version: Select 1.12.
                                                                                                                • Save Job Log: Enable this function.
                                                                                                                • OBS Bucket: Select an OBS bucket for storing job logs and grant access permissions of the OBS bucket as prompted.
                                                                                                                • Enable Checkpointing: Enable this function.
                                                                                                                +
                                                                                                              5. Click OK to enter the editing page.
                                                                                                              6. Set job running parameters. The mandatory parameters are as follows:
                                                                                                                • Queue: dli_queue_01
                                                                                                                • Flink Version: Select 1.12.
                                                                                                                • Save Job Log: Enable this function.
                                                                                                                • OBS Bucket: Select an OBS bucket for storing job logs and grant access permissions of the OBS bucket as prompted.
                                                                                                                • Enable Checkpointing: Enable this function.

                                                                                                                You do not need to set other parameters.

                                                                                                              7. Click Save.
                                                                                                              8. Edit the Flink OpenSource SQL job.
                                                                                                                In the SQL statement editing area, enter query and analysis statements as you need. The example statements are as follows. Note that the values of the parameters in bold must be changed according to the comments.
                                                                                                                CREATE TABLE kafkaSource (
   order_id string,
@@ -120,7 +600,7 @@ CREATE TABLE jdbcSink (
 insert into jdbcSink select * from kafkaSource;
                                                                                                              9. Click Check Semantics.
                                                                                                              10. Click Start. On the displayed Start Flink Job page, confirm the job specifications and the price, and click Start Now to start the job.

                                                                                                                After the job is started, the system automatically switches to the Flink Jobs page, and the created job is displayed in the job list. You can view the job status in the Status column. After a job is successfully submitted, Status of the job will change from Submitting to Running.

                                                                                                                -

                                                                                                                If Status of a job is Submission failed or Running exception, the job fails to be submitted or fails to run. In this case, you can hover over the status icon to view the error details. You can click to copy these details. After handling the fault based on the provided information, resubmit the job.

                                                                                                                +

                                                                                                                If Status of a job is Submission failed or Running exception, the job fails to be submitted or fails to run. In this case, you can hover over the status icon to view the error details. You can click to copy these details. Rectify the fault based on the error information and resubmit the job.

                                                                                                              11. Connect to the Kafka cluster and send the following test data to the Kafka topics:
                                                                                                                {"order_id":"202103241000000001", "order_channel":"webShop", "order_time":"2021-03-24 10:00:00", "pay_amount":"100.00", "real_pay":"100.00", "pay_time":"2021-03-24 10:02:03", "user_id":"0001", "user_name":"Alice", "area_id":"330106"} 
 
 {"order_id":"202103241606060001", "order_channel":"appShop", "order_time":"2021-03-24 16:06:06", "pay_amount":"200.00", "real_pay":"180.00", "pay_time":"2021-03-24 16:10:06", "user_id":"0001", "user_name":"Alice", "area_id":"330106"}
                                                                                                                diff --git a/docs/dli/umn/dli_01_0532.html b/docs/dli/umn/dli_01_0532.html index 128c06feb..b3c7fafda 100644 --- a/docs/dli/umn/dli_01_0532.html +++ b/docs/dli/umn/dli_01_0532.html @@ -2,8 +2,6 @@

                                                                                                                Viewing Scaling History

                                                                                                                Scenario

                                                                                                                If you added a queue to or deleted one from an elastic resource pool, or you scaled an added queue, the CU quantity of the elastic resource pool may be changed. You can view historical CU changes of an elastic resource pool on the console.

                                                                                                                -

                                                                                                                When scaling in an elastic resource pool, Spark and SQL jobs may automatically retry. However, if the number of retries exceeds the limit, the jobs will fail and need to be manually executed again.

                                                                                                                -

                                                                                                                Prerequisites

                                                                                                                Currently, you can only view the historical records generated within the last 30 days on the console.

                                                                                                                @@ -14,7 +12,7 @@
                                                                                                                -
                                                                                                                Parent topic: Regular Operations
                                                                                                                +
                                                                                                                Parent topic: Managing Elastic Resource Pools
                                                                                                                diff --git a/docs/dli/umn/dli_01_0533.html b/docs/dli/umn/dli_01_0533.html deleted file mode 100644 index e07095570..000000000 --- a/docs/dli/umn/dli_01_0533.html +++ /dev/null @@ -1,41 +0,0 @@ - - -

                                                                                                                Permission Management for Global Variables

                                                                                                                -

                                                                                                                Scenario

                                                                                                                • You can grant permissions on a global variable to users.
                                                                                                                • The administrator and the global variable owner have all permissions. You do not need to set permissions for them, and their global variable permissions cannot be modified by other users.
                                                                                                                • When setting global variables for a new user, ensure that the user group the user belongs to has the Tenant Guest permission. For details about the Tenant Guest permission and how to apply for the permission, see .
                                                                                                                -
                                                                                                                -

                                                                                                                Granting Permissions on a Global Variable to a User

                                                                                                                1. Choose Global Configuration > Global Variables, locate the row containing the desired global variable, and click Set Permission in the Operation column. On the User Permissions page displayed, you can grant permissions for the global variable, set and revoke user permissions.
                                                                                                                2. Click Grant Permission in the upper right corner of the page. -
                                                                                                                  - - - - - - - - - - -
                                                                                                                  Table 1 Global variable parameters

                                                                                                                  Parameter

                                                                                                                  -

                                                                                                                  Description

                                                                                                                  -

                                                                                                                  Username

                                                                                                                  -

                                                                                                                  Name of the IAM user to whom permissions on the datasource connection are to be granted.

                                                                                                                  -
                                                                                                                  NOTE:

                                                                                                                  The username is the name of an existing IAM user.

                                                                                                                  -
                                                                                                                  -

                                                                                                                  Permissions to be granted to the user

                                                                                                                  -
                                                                                                                  • Update: This permission allows you to update the global variable.
                                                                                                                  • Delete: This permission allows you to delete the global variable.
                                                                                                                  • Grant Permission: This permission allows you to grant permissions of the global variable to other users.
                                                                                                                  • Revoke Permission: This permission allows you to revoke the global variable permissions that other users have but cannot revoke the global variable owner's permissions.
                                                                                                                  • View Other User's Permissions: This permission allows you to view the global variable permissions of other users.
                                                                                                                  -
                                                                                                                  -
                                                                                                                  -
                                                                                                                -
                                                                                                                -

                                                                                                                Granting Global Variable Permissions

                                                                                                                Click Set Permission in the Operation column of the sub-user to modify their permissions. Table 1 lists the permission parameters.

                                                                                                                -

                                                                                                                If all permission options are grayed out, you are not allowed to change permissions on this global variable. You can apply to the administrator, group owner, or other authorized users for required permissions on the global variable.

                                                                                                                -
                                                                                                                -

                                                                                                                Revoking Global Variable Permissions

                                                                                                                Click Revoke Permission in the Operation column of a sub-user to revoke their permissions. After this operation, the sub-user does not have any permission on the global variable.

                                                                                                                -
                                                                                                                -
                                                                                                                -
                                                                                                                -
                                                                                                                -
                                                                                                                Parent topic: Global Configuration
                                                                                                                -
                                                                                                                -
                                                                                                                - diff --git a/docs/dli/umn/dli_01_0534.html b/docs/dli/umn/dli_01_0534.html index 9792555a9..e15b2ac56 100644 --- a/docs/dli/umn/dli_01_0534.html +++ b/docs/dli/umn/dli_01_0534.html @@ -3,10 +3,11 @@

                                                                                                                Enabling Dynamic Scaling for Flink Jobs

                                                                                                                Scenario

                                                                                                                In actual job operations, the compute resources required by a job vary depending on the data volume. As a result, compute resources are wasted when the volume is small and are insufficient when the volume is large.

                                                                                                                DLI provides dynamic scaling to dynamically adjust the compute resources used by a job based on the job load, such as the data input and output volume, data input and output rate, and backpressure, to improve resource utilization.

                                                                                                                -

                                                                                                                Currently, dynamic scaling can be enabled only for jobs of Flink 1.12 or later.

                                                                                                                +

                                                                                                                After enabling dynamic scaling for Flink jobs, the system will adjust resource allocation based on the actual resource requirements of the Flink jobs. When the remaining pod resources in the elastic resource pool are sufficient to meet the minimum resource requirements of the job, the system will automatically reduce the number of nodes where the job is running. This ensures efficient job running while improving resource utilization.

                                                                                                                +

                                                                                                                Currently, dynamic scaling can only be enabled for Flink 1.12 jobs.

                                                                                                                -

                                                                                                                Notes

                                                                                                                • During dynamic scaling of a Flink job, if queue resources are occupied and the remaining resources are insufficient for starting the job, the job may fail to be restored.
                                                                                                                • When the resources that can be used by a Flink job are dynamically scaled in or out, the background job needs to be stopped and then restored from the savepoint. So, the job cannot process data before the restoration is successful.
                                                                                                                • Savepoints need to be triggered during scaling. So, you must configure an OBS bucket, save logs, and enable checkpointing.
                                                                                                                • Do not set the scaling detection period to a small value to avoid frequent job start and stop.
                                                                                                                • The restoration duration of a scaling job is affected by the savepoint size. If the savepoint size is large, the restoration may take a long time.
                                                                                                                • To adjust the configuration items of dynamic scaling, you need to stop the job, edit the job, and submit the job for the modification to take effect.
                                                                                                                +

                                                                                                                Notes

                                                                                                                • During dynamic scaling of a Flink job, if queue resources are preempted and the remaining resources are insufficient for starting the job, the job may fail to be restored.
                                                                                                                • When the resources that can be used by a Flink job are dynamically scaled in or out, the background job needs to be stopped and then restored from the savepoint. So, the job cannot process data before the restoration is successful.
                                                                                                                • Savepoints need to be triggered during scaling. So, you must configure an OBS bucket, save logs, and enable checkpointing.
                                                                                                                • Do not set the scaling detection period to a small value to avoid frequent job start and stop.
                                                                                                                • The restoration duration of a scaling job is affected by the savepoint size. If the savepoint size is large, the restoration may take a long time.
                                                                                                                • To adjust the configuration items of dynamic scaling, you need to stop the job, edit the job, and submit the job for the modification to take effect.

                                                                                                                Procedure

                                                                                                                Dynamic scaling applies to Flink OpenSource SQL and Flink Jar jobs.

                                                                                                                1. Log in to the DLI management console.
                                                                                                                2. In the navigation pane on the left, choose Job Management > Flink Jobs.
                                                                                                                3. Select the job for which you want to enable dynamic scaling and click Edit in the Operation column.
                                                                                                                  • For a Flink OpenSource SQL job, click Runtime Configuration on the right to configure dynamic scaling parameters.
                                                                                                                  • For a Flink Jar job, click the Runtime Configuration box to configure dynamic scaling parameters.
                                                                                                                  @@ -61,7 +62,7 @@
                                                                                                                -
                                                                                                                Parent topic: Flink Job Management
                                                                                                                +
                                                                                                                Parent topic: Managing Flink Jobs
                                                                                                                diff --git a/docs/dli/umn/dli_01_0535.html b/docs/dli/umn/dli_01_0535.html index 1cd296ebc..27305fabc 100644 --- a/docs/dli/umn/dli_01_0535.html +++ b/docs/dli/umn/dli_01_0535.html @@ -1,41 +1,19 @@ -

                                                                                                                Setting the Priority for a Job

                                                                                                                -

                                                                                                                Scenario

                                                                                                                In actual job operations, jobs have different importance and urgency levels. So, the compute resources required for normal operations of important and urgent jobs need to be guaranteed.

                                                                                                                -

                                                                                                                DLI allows you to set the priority of each Spark job, Spark SQL job, and Flink job. When compute resources are insufficient, the resources are preferentially used for jobs with higher priorities.

                                                                                                                -
                                                                                                                • Priorities can only be set for jobs running in elastic resource pools.
                                                                                                                • SQL jobs in elastic resource pools support priorities.
                                                                                                                • Priorities can be set for jobs of Spark 2.4.5 or later.
                                                                                                                • Priorities can be set for jobs of Flink 1.12 or later.
                                                                                                                -
                                                                                                                +

                                                                                                                Setting the Priority for a SQL Job

                                                                                                                +

                                                                                                                Scenario

                                                                                                                In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. This requires providing the necessary compute resources for their normal operations.

                                                                                                                +

                                                                                                                DLI offers a feature to set job priorities for each SQL job, which prioritizes the allocation of compute resources to higher priority jobs when resources are limited.

                                                                                                                -

                                                                                                                Notes

                                                                                                                • You can set the priority for each job. The value ranges from 1 to 10. A larger value indicates a higher priority. Compute resources are preferentially allocated to high-priority jobs. That is, if compute resources required for high-priority jobs are insufficient, compute resources for low-priority jobs are reduced.
                                                                                                                • The default priority of Flink jobs running on a general-purpose queue is 5.
                                                                                                                • The default priority of Spark jobs running on a general-purpose queue is 3.
                                                                                                                • The default priority of jobs running on a SQL queue is 3.
                                                                                                                • The change to the job priority takes effect only after the job is stopped, edited, and submitted.
                                                                                                                • For Flink jobs, enable dynamic scaling by setting flink.dli.job.scale.enable to true by referring to Enabling Dynamic Scaling for Flink Jobs, and then set the job priority.
                                                                                                                • The change to the priority for a Flink job takes effect only after the job is stopped, edited, and submitted.
                                                                                                                +

                                                                                                                Notes

                                                                                                                • You can assign a priority level of 1 to 10 for each job, with a larger value indicating a higher priority. Compute resources are preferentially allocated to high-priority jobs. That is, if compute resources required for high-priority jobs are insufficient, compute resources for low-priority jobs are reduced.
                                                                                                                • Jobs running on a SQL queue have a default priority level of 3.
                                                                                                                • To change the priority for a job, you must first stop the job, change the priority level, and then submit the job for the modification to take effect.
                                                                                                                -

                                                                                                                Procedure for Setting the Priorities for Flink OpenSource SQL Jobs

                                                                                                                1. Log in to the DLI management console.
                                                                                                                2. In the navigation pane on the left, choose Job Management > Flink Jobs.
                                                                                                                3. Locate the row containing the job for which you want to set the priority and click Edit in the Operation column.
                                                                                                                4. On the right of the page displayed, click Runtime Configuration.
                                                                                                                5. In the text box, enter the following statements to enable dynamic scaling and then set the job priority:

                                                                                                                  For Flink jobs, you must set flink.dli.job.scale.enable to true to enable dynamic scaling, and then set the job priority.

                                                                                                                  -

                                                                                                                  For more parameter settings for enabling dynamic scaling, see Enabling Dynamic Scaling for Flink Jobs.

                                                                                                                  -
                                                                                                                  -
                                                                                                                  flink.dli.job.scale.enable=true
-flink.dli.job.priority=x
                                                                                                                  -
                                                                                                                -
                                                                                                                -

                                                                                                                Procedure for Setting the Priorities for Flink Jar Jobs

                                                                                                                In the Runtime Configuration text box, enter the following statement. x indicates the priority value.

                                                                                                                -
                                                                                                                flink.dli.job.priority=x
                                                                                                                -
                                                                                                                1. Log in to the DLI management console.
                                                                                                                2. In the navigation pane on the left, choose Job Management > Flink Jobs.
                                                                                                                3. Locate the row containing the job for which you want to set the priority and click Edit in the Operation column.
                                                                                                                4. In the Runtime Configuration text box, enter the following statements to enable dynamic scaling and set the job priority:

                                                                                                                  For Flink jobs, you must set flink.dli.job.scale.enable to true to enable dynamic scaling, and then set the job priority.

                                                                                                                  -

                                                                                                                  For more parameter settings for enabling dynamic scaling, see Enabling Dynamic Scaling for Flink Jobs.

                                                                                                                  -
                                                                                                                  -
                                                                                                                  flink.dli.job.scale.enable=true
-flink.dli.job.priority=x
                                                                                                                  -
                                                                                                                -
                                                                                                                -

                                                                                                                Procedure for Setting the Priorities for Spark Jobs

                                                                                                                In the Spark Arguments(--conf) text box, enter the following statement. x indicates the priority value.

                                                                                                                -
                                                                                                                spark.dli.job.priority=x
                                                                                                                -
                                                                                                                1. Log in to the DLI management console.
                                                                                                                2. In the navigation pane on the left, choose Job Management > Spark Jobs.
                                                                                                                3. Locate the row containing the job for which you want to set the priority and click Edit in the Operation column.
                                                                                                                4. In the Spark Arguments(--conf) text box, configure the spark.dli.job.priority parameter.
                                                                                                                -
                                                                                                                -

                                                                                                                Procedure for Setting the Priorities for Spark SQL Jobs

                                                                                                                Click Settings. In the Parameter Settings area, configure the following parameter. x indicates the priority value.
                                                                                                                spark.sql.dli.job.priority=x
                                                                                                                +

                                                                                                                Setting the Priority for a SQL Job

                                                                                                                Click Settings. In the Parameter Settings area, configure the following parameter. x indicates the priority value.
                                                                                                                spark.sql.dli.job.priority=x
                                                                                                                1. Log in to the DLI management console.
                                                                                                                2. In the navigation pane on the left, choose Job Management > SQL Jobs.
                                                                                                                3. Locate the row containing the job for which you want to set the priority and click Edit in the Operation column.
                                                                                                                4. Click Settings. In the Parameter Settings area, configure the spark.sql.dli.job.priority parameter.
                                                                                                                -
                                                                                                                Parent topic: Job Management
                                                                                                                +
                                                                                                                Parent topic: Submitting a SQL Job Using DLI
                                                                                                                diff --git a/docs/dli/umn/dli_01_0536.html b/docs/dli/umn/dli_01_0536.html new file mode 100644 index 000000000..51f93ec3a --- /dev/null +++ b/docs/dli/umn/dli_01_0536.html @@ -0,0 +1,20 @@ + + +

                                                                                                                Configuring a DLI Job Bucket

                                                                                                                +

                                                                                                                Before using DLI, you need to configure a DLI job bucket. The bucket is used to store temporary data generated during DLI job running, such as job logs and results.

                                                                                                                +

                                                                                                                Configure a DLI job bucket on the Global Configuration > Project page of the DLI management console.

                                                                                                                +

                                                                                                                Preparations

                                                                                                                Before the configuration, create an OBS bucket or parallel file system (PFS).

                                                                                                                +

                                                                                                                In big data scenarios, you are advised to create a PFS. PFS is a high-performance file system provided by OBS, with access latency in milliseconds. PFS can achieve a bandwidth performance of up to TB/s and millions of IOPS, which makes it ideal for processing high-performance computing (HPC) workloads.

                                                                                                                +
                                                                                                                +

                                                                                                                Notes

                                                                                                                • Do not use the OBS bucket for other purposes.
                                                                                                                • The OBS bucket must be set and modified by the main account. Member users do not have the permission.
                                                                                                                • If the bucket is not configured, you will not be able to view job logs.
                                                                                                                • You can create lifecycle rules to automatically delete objects or change storage classes for objects that meet specified conditions.
                                                                                                                • Inappropriate modifications of the job bucket may lead to loss of historical data.
                                                                                                                +
                                                                                                                +

                                                                                                                Procedure

                                                                                                                1. In the navigation pane of the DLI console, choose Global Configuration > Project.
                                                                                                                2. On the Project page, click next to Job Bucket to configure bucket information.
                                                                                                                3. Click to view available buckets.
                                                                                                                4. Select the bucket for storing the temporary data of the DLI job and click OK.

                                                                                                                  Temporary data generated during DLI job running will be stored in the OBS bucket.

                                                                                                                  +
                                                                                                                +
                                                                                                                +
                                                                                                                +
                                                                                                                +
                                                                                                                +
                                                                                                                Parent topic: Preparations
                                                                                                                +
                                                                                                                +
                                                                                                                + diff --git a/docs/dli/umn/dli_01_0538.html b/docs/dli/umn/dli_01_0538.html index 95c382a34..7f178e144 100644 --- a/docs/dli/umn/dli_01_0538.html +++ b/docs/dli/umn/dli_01_0538.html @@ -8,7 +8,7 @@
                                                                                                                  -
                                                                                                                • What Is Data Lake Insight?
                                                                                                                  +
                                                                                                                • What Is Data Lake Insight
                                                                                                                • Advantages
                                                                                                                  • @@ -16,6 +16,8 @@
                                                                                                                • Notes and Constraints
                                                                                                                  • +
                                                                                                                • Specifications
                                                                                                                  +
                                                                                                                • Permissions Management
                                                                                                                • Quotas
                                                                                                                  diff --git a/docs/dli/umn/dli_01_0539.html b/docs/dli/umn/dli_01_0539.html index b87234d72..2d25d55ff 100644 --- a/docs/dli/umn/dli_01_0539.html +++ b/docs/dli/umn/dli_01_0539.html @@ -8,25 +8,25 @@ diff --git a/docs/dli/umn/dli_01_0550.html b/docs/dli/umn/dli_01_0550.html index 00230a3ba..b2db12109 100644 --- a/docs/dli/umn/dli_01_0550.html +++ b/docs/dli/umn/dli_01_0550.html @@ -1,6 +1,6 @@ -

                                                                                                                  Quota Management

                                                                                                                  +

                                                                                                                  Managing DLI Resource Quotas

                                                                                                                  What Is a Quota?

                                                                                                                  A quota limits the quantity of a resource available to users, thereby preventing spikes in the usage of the resource.

                                                                                                                  You can also request for an increased quota if your existing quota cannot meet your service requirements.

                                                                                                                  @@ -18,7 +18,7 @@
                                                                                                                  diff --git a/docs/dli/umn/dli_01_0551.html b/docs/dli/umn/dli_01_0551.html index eabdc2b32..8ef448aaa 100644 --- a/docs/dli/umn/dli_01_0551.html +++ b/docs/dli/umn/dli_01_0551.html @@ -1,6 +1,6 @@ -

                                                                                                                  Managing Spark SQL Templates

                                                                                                                  +

                                                                                                                  Managing Spark Job Templates

                                                                                                                  Scenario

                                                                                                                  You can modify a sample template to meet the Spark job requirements, saving time for editing SQL statements.

                                                                                                                  Currently, the cloud platform does not provide preset Spark templates. You can customize Spark job templates. This section describes how to create a Spark job template.

                                                                                                                  @@ -10,7 +10,7 @@
                                                                                                                  -
                                                                                                                  Parent topic: Job Templates
                                                                                                                  +
                                                                                                                  Parent topic: Submitting a Spark Job Using DLI
                                                                                                                  diff --git a/docs/dli/umn/dli_01_0552.html b/docs/dli/umn/dli_01_0552.html index 27082aaa1..3322ce424 100644 --- a/docs/dli/umn/dli_01_0552.html +++ b/docs/dli/umn/dli_01_0552.html @@ -21,7 +21,7 @@
                                                                                                                  • Click the text box for tag key and select a predefined tag key from the drop-down list.

                                                                                                                    To add a predefined tag, you need to create one on TMS and then select it from the Tag key drop-down list. You can click View predefined tags to go to the Predefined Tags page of the TMS console. Then, click Create Tag in the upper corner of the page to create a predefined tag.

                                                                                                                  -
                                                                                                                  • Enter a tag key in the text box.
                                                                                                                    NOTE:

                                                                                                                    A tag key can contain a maximum of 128 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                                                    +
                                                                                                                    • Enter a tag key in the text box.
                                                                                                                      NOTE:

                                                                                                                      A tag key can contain a maximum of 128 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed, but the value cannot start or end with a space or start with _sys_.

                                                                                                                    @@ -29,7 +29,7 @@

                                                                                                    Tag value

                                                                                                    You can specify the tag value in either of the following ways:

                                                                                                    -
                                                                                                    • Click the tag value text box and select a predefined tag value from the drop-down list.
                                                                                                    • Enter a tag value in the text box.
                                                                                                      NOTE:

                                                                                                      A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                                                                                      +
                                                                                                      • Click the tag value text box and select a predefined tag value from the drop-down list.
                                                                                                      • Enter a tag value in the text box.
                                                                                                        NOTE:

                                                                                                        A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.

                                                                                                    Tag value

                                                                                                    You can specify the tag value in either of the following ways:

                                                                                                    -
                                                                                                    • Click the tag value text box and select a predefined tag value from the drop-down list.
                                                                                                    • Enter a tag value in the text box.
                                                                                                      NOTE:

                                                                                                      A tag value can contain a maximum of 255 characters. Only letters, digits, spaces, and special characters (_.:=+-@) are allowed. The value cannot start or end with a space.

                                                                                                      +
                                                                                                      • Click the tag value text box and select a predefined tag value from the drop-down list.
                                                                                                      • Enter a tag value in the text box.
                                                                                                        NOTE:

                                                                                                        A tag value can contain a maximum of 255 characters. Only letters, numbers, spaces, and special characters (_.:+-@) are allowed.
                                                                                                    + + + + + + + + + + +
                                                                                                    Table 1 Notes and constraints on datasource authentication

                                                                                                    Item

                                                                                                    +

                                                                                                    Description

                                                                                                    +

                                                                                                    Use case

                                                                                                    +
                                                                                                    • Only Spark SQL and Flink OpenSource SQL 1.12 jobs support datasource authentication.
                                                                                                    +

                                                                                                    Datasource authentication type

                                                                                                    +
                                                                                                    • DLI supports four types of datasource authentication. Select an authentication type specific to each data source.
                                                                                                      • CSS: applies to 6.5.4 or later CSS clusters with the security mode enabled.
                                                                                                      • Kerberos: applies to MRS security clusters with Kerberos authentication enabled.
                                                                                                      • Kafka_SSL: applies to Kafka with SSL enabled.
                                                                                                      • Password: applies to GaussDB(DWS), RDS, DDS, and DCS.
                                                                                                    +
                                                                                                    +

                                                                                                Datasource Authentication Types

                                                                                                DLI supports four types of datasource authentication. Select an authentication type specific to each data source.

                                                                                                • CSS: applies to 6.5.4 or later CSS clusters with the security mode enabled. During the configuration, you need to specify the username, password, and authentication certificate of the cluster and store the information in DLI through datasource authentication so that DLI can securely access CSS data sources. For details, see Creating a CSS Datasource Authentication.
                                                                                                • Kerberos: applies to MRS security clusters with Kerberos authentication enabled. During the configuration, you need to specify MRS cluster authentication credentials, including the krb5.conf and user.keytab files. For details, see Creating a Kerberos Datasource Authentication.
                                                                                                • Kafka_SSL: applies to Kafka with SSL enabled. During the configuration, you need to specify the KafkaTruststore path and password. For details, see Creating a Kafka_SSL Datasource Authentication.
                                                                                                • Password: applies to GaussDB(DWS), RDS, DDS, and DCS data sources. During the configuration, you need to store the passwords of the data sources in DLI. For details, see Creating a Password Datasource Authentication.

                                                                                                Jobs That Can Connect to Data Sources Through Datasource Authentication

                                                                                                Different types of jobs can connect to data sources through different types of datasource authentication.

                                                                                                -
                                                                                                • For details about the data sources that Spark SQL jobs can connect to through datasource authentication and their constraints, see Table 1.
                                                                                                • For details about the data sources that Flink SQL jobs can connect to through datasource authentication and their constraints, see Table 2.
                                                                                                +
                                                                                                • For details about the data sources that Spark SQL jobs can connect to through datasource authentication and their constraints, see Table 2.
                                                                                                • For details about the data sources that Flink SQL jobs can connect to through datasource authentication and their constraints, see Table 3.
                                                                                                -
                                                                                                Table 1 Data sources that Spark SQL jobs can connect to through datasource authentication

                                                                                                Datasource Authentication Type

                                                                                                +
                                                                                                @@ -41,7 +62,7 @@
                                                                                                Table 2 Data sources that Spark SQL jobs can connect to through datasource authentication

                                                                                                Datasource Authentication Type

                                                                                                Data Source

                                                                                                -
                                                                                                Table 2 Data sources that Flink SQL jobs can connect to through datasource authentication

                                                                                                Table Type

                                                                                                +
                                                                                                @@ -115,7 +136,7 @@ diff --git a/docs/dli/umn/dli_01_0563.html b/docs/dli/umn/dli_01_0563.html index f0a8216f8..b3876d59a 100644 --- a/docs/dli/umn/dli_01_0563.html +++ b/docs/dli/umn/dli_01_0563.html @@ -7,7 +7,7 @@

                                                                                                Notes and Constraints

                                                                                                • Only SQL queues of the Spark engine support configuring queue properties.
                                                                                                • Setting queue properties is only supported after the queue has been created.
                                                                                                • Currently, only queue properties related to the Spark driver can be set.
                                                                                                • Queue properties cannot be set in batches.
                                                                                                • For a queue in an elastic resource pool, if the minimum CUs of the queue is less than 16 CUs, both Max. Spark Driver Instances and Max. Prestart Spark Driver Instances set in the queue properties do not apply.
                                                                                                -

                                                                                                Procedure

                                                                                                1. In the navigation pane of the DLI management console, choose Resources > Queue Management.
                                                                                                2. In the Operation column of the queue, choose More > Set Property.
                                                                                                3. Go to the queue property setting page and set property parameters. For details about the property parameters, see Table 1. +

                                                                                                  Procedure

                                                                                                  1. In the navigation pane of the DLI management console, choose Resources > Queue Management.
                                                                                                  2. Locate the queue for which you want to set properties, click More in the Operation column, and select Set Property.
                                                                                                  3. Go to the queue property setting page and set property parameters. Table 1 describes the property parameters.
                                                                                                Table 3 Data sources that Flink SQL jobs can connect to through datasource authentication

                                                                                                Table Type

                                                                                                Datasource Authentication Type
                                                                                                Table 1 Queue properties

                                                                                                Property

                                                                                                Description

                                                                                                @@ -45,7 +45,7 @@
                                                                                                -
                                                                                                Parent topic: Queue Management
                                                                                                +
                                                                                                Parent topic: Managing Queues
                                                                                                diff --git a/docs/dli/umn/dli_01_0565.html b/docs/dli/umn/dli_01_0565.html index 2ae1b24a1..1f2fa07e7 100644 --- a/docs/dli/umn/dli_01_0565.html +++ b/docs/dli/umn/dli_01_0565.html @@ -7,12 +7,8 @@

                                                                                                Prerequisites

                                                                                                You have logged in to the Enterprise Project Management Service console and created an enterprise project.

                                                                                                -

                                                                                                Binding an Enterprise Project

                                                                                                When creating a DLI queue, you can select a created enterprise project for Enterprise Project.

                                                                                                -

                                                                                                Alternatively, you can click Create Enterprise Project to go to the Enterprise Project Management Service console to create an enterprise project and view existing ones.

                                                                                                -

                                                                                                For details about how to create a queue, see Creating a Queue.

                                                                                                -
                                                                                                -

                                                                                                Modifying an Enterprise Project

                                                                                                You can modify the enterprise project bound to a created cluster as needed.

                                                                                                -
                                                                                                1. Log in to the DLI management console.
                                                                                                2. In the navigation pane on the left, choose Resources > Queue Management.
                                                                                                3. In the queue list, locate the queue for which you want to modify the enterprise project, click More in the Operation column, and select Modify Enterprise Project.
                                                                                                4. In the Modify Enterprise Project dialog box displayed, select an enterprise project.

                                                                                                  Alternatively, you can click Create Enterprise Project to go to the Enterprise Project Management Service console to create an enterprise project and view existing ones.

                                                                                                  +

                                                                                                  Modifying an Enterprise Project

                                                                                                  You can change the enterprise project associated with a queue that has been created.

                                                                                                  +
                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Resources > Queue Management.
                                                                                                  3. In the queue list, locate the queue for which you want to modify the enterprise project, click More in the Operation column, and select Modify Enterprise Project.
                                                                                                  4. In the Modify Enterprise Project dialog box displayed, select an enterprise project.

                                                                                                    Alternatively, you can click Create Enterprise Project to go to the Enterprise Project Management Service console to create an enterprise project and check existing ones.

                                                                                                  5. After the modification, click OK to save the enterprise project information of the queue.

                                                                                                  Related Operations

                                                                                                  For details about how to modify the enterprise project of an elastic resource pool, see Allocating to an Enterprise Project.

                                                                                                  @@ -20,7 +16,7 @@
                                                                                                  -
                                                                                                  Parent topic: Queue Management
                                                                                                  +
                                                                                                  Parent topic: Managing Queues
                                                                                                  diff --git a/docs/dli/umn/dli_01_0566.html b/docs/dli/umn/dli_01_0566.html index 19726f5e0..7003ab8ff 100644 --- a/docs/dli/umn/dli_01_0566.html +++ b/docs/dli/umn/dli_01_0566.html @@ -9,11 +9,11 @@

                                                                                                  Prerequisites

                                                                                                  You have logged in to the Enterprise Project Management Service console and created an enterprise project.

                                                                                                  Binding an Enterprise Project

                                                                                                  When creating an elastic resource pool, you can select a created enterprise project for Enterprise Project.

                                                                                                  -

                                                                                                  Alternatively, you can click Create Enterprise Project to go to the Enterprise Project Management Service console to create an enterprise project and view existing ones.

                                                                                                  -

                                                                                                  For details about how to create a queue, see Creating an Elastic Resource Pool.

                                                                                                  +

                                                                                                  Alternatively, you can click Create Enterprise Project to go to the Enterprise Project Management Service console to create an enterprise project and check existing ones.

                                                                                                  +

                                                                                                  For how to create a queue, see Creating an Elastic Resource Pool and Creating Queues Within It.

                                                                                                  Modifying an Enterprise Project

                                                                                                  You can modify the enterprise project bound to a created cluster as needed.

                                                                                                  -
                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                  3. In the elastic resource pool list, locate the elastic resource pool for which you want to modify the enterprise project, click More in the Operation column, and select Allocate to Enterprise Project.
                                                                                                  4. In the Modify Enterprise Project dialog box displayed, select an enterprise project.

                                                                                                    Alternatively, you can click Create Enterprise Project to go to the Enterprise Project Management Service console to create an enterprise project and view existing ones.

                                                                                                    +
                                                                                                    1. Log in to the DLI management console.
                                                                                                    2. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                    3. In the elastic resource pool list, locate the elastic resource pool for which you want to modify the enterprise project, click More in the Operation column, and select Allocate to Enterprise Project.
                                                                                                    4. In the Modify Enterprise Project dialog box displayed, select an enterprise project.

                                                                                                      Alternatively, you can click Create Enterprise Project to go to the Enterprise Project Management Service console to create an enterprise project and check existing ones.

                                                                                                    5. After the modification, click OK to save the enterprise project information of the elastic resource pool.

                                                                                                  Related Operations

                                                                                                  For details about how to modify the enterprise project of a queue, see Allocating a Queue to an Enterprise Project.

                                                                                                  @@ -21,7 +21,7 @@
                                                                                                  -
                                                                                                  Parent topic: Regular Operations
                                                                                                  +
                                                                                                  Parent topic: Managing Elastic Resource Pools
                                                                                                  diff --git a/docs/dli/umn/dli_01_0567.html b/docs/dli/umn/dli_01_0567.html index fb35606f8..fb2502bce 100644 --- a/docs/dli/umn/dli_01_0567.html +++ b/docs/dli/umn/dli_01_0567.html @@ -1,15 +1,17 @@ -

                                                                                                  Overview

                                                                                                  -

                                                                                                  DLI Job Type

                                                                                                  DLI provides the following job types:

                                                                                                  -
                                                                                                  • SQL job: SQL jobs provide you with standard SQL statements and are compatible with Spark SQL and Presto SQL (based on Presto). You can query and analyze heterogeneous data sources on the cloud through visualized APIs, JDBC, ODBC, or Beeline. SQL jobs are compatible with mainstream data formats such as CSV, JSON, Parquet, Carbon, and ORC.
                                                                                                  • Flink job: Flink jobs are real-time streaming big data analysis service jobs running on the public cloud. In full hosting mode, you only need to focus on Stream SQL services and execute jobs instantly without being aware of compute clusters. Flink jobs are fully compatible with Apache Flink APIs.
                                                                                                  • Spark job: Spark jobs provide fully-managed Spark compute services. You can submit jobs through the GUI or RESTful APIs. Full-stack Spark jobs, such as Spark Core, DataSet, Streaming, MLlib, and GraphX jobs, are supported.
                                                                                                  -
                                                                                                  -

                                                                                                  Constraints

                                                                                                  • Only the latest 100 jobs are displayed on DLI's SparkUI.
                                                                                                  • A maximum of 1,000 job results can be displayed on the console. To view more or all jobs, export the job data to OBS.
                                                                                                  • To export job run logs, you must have the permission to access OBS buckets. You need to configure a DLI job bucket on the Global Configuration > Project page in advance.
                                                                                                  • The View Log button is not available for synchronization jobs and jobs running on the default queue.
                                                                                                  • Only Spark jobs support custom images.
                                                                                                  • An elastic resource pool supports a maximum of 32,000 CUs.
                                                                                                  -
                                                                                                  -
                                                                                                  +

                                                                                                  Common DLI Management Operations

                                                                                                  +
                                                                                                  diff --git a/docs/dli/umn/dli_01_0611.html b/docs/dli/umn/dli_01_0611.html new file mode 100644 index 000000000..832bb2294 --- /dev/null +++ b/docs/dli/umn/dli_01_0611.html @@ -0,0 +1,15 @@ + + +

                                                                                                  Preparations

                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0616.html b/docs/dli/umn/dli_01_0616.html new file mode 100644 index 000000000..db4eb5845 --- /dev/null +++ b/docs/dli/umn/dli_01_0616.html @@ -0,0 +1,132 @@ + + +

                                                                                                  Creating a Custom DLI Agency

                                                                                                  +

                                                                                                  When Flink 1.15, Spark 3.3, or a later version is used to execute jobs and the required agency is not included in the DLI system agency dli_management_agency, you need to create an agency on the IAM console and add information about the new agency to the job configuration. dli_management_agency contains the permissions required for datasource operations, message notifications, and user authorization operations. For other agency permission requirements, you need to create custom DLI agencies. For details about dli_management_agency, see DLI Agency Overview.

                                                                                                  +

                                                                                                  This section walks you through on how to create a custom agency, complete service authorization, and add information about the new agency to the job configuration.

                                                                                                  +

                                                                                                  DLI Custom Agency Scenarios

                                                                                                  +
                                                                                                  + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                  Table 1 DLI custom agency scenarios

                                                                                                  Scenario

                                                                                                  +

                                                                                                  Agency Name

                                                                                                  +

                                                                                                  Description

                                                                                                  +

                                                                                                  Permission Policy

                                                                                                  +

                                                                                                  Allowing DLI to clear data according to the lifecycle of a table

                                                                                                  +

                                                                                                  dli_data_clean_agency

                                                                                                  +

                                                                                                  Data cleanup agency, which is used to clean up data according to the lifecycle of a table and clean up lakehouse table data.

                                                                                                  +

                                                                                                  You need to create an agency and customize permissions for it. However, the agency name is fixed to dli_data_clean_agency.

                                                                                                  +

                                                                                                  Data Cleanup Agency Permission Configuration

                                                                                                  +

                                                                                                  Allowing DLI to read and write data from and to OBS to transfer logs

                                                                                                  +

                                                                                                  Custom

                                                                                                  +

                                                                                                  For DLI Flink jobs, the permissions include downloading OBS objects, obtaining OBS/GaussDB(DWS) data sources (foreign tables), transferring logs, using savepoints, and enabling checkpointing. For DLI Spark jobs, the permissions allow downloading OBS objects and reading/writing OBS foreign tables.

                                                                                                  +

                                                                                                  Permission Policies for Accessing and Using OBS

                                                                                                  +

                                                                                                  Allowing DLI to obtain data access credentials by accessing DEW

                                                                                                  +

                                                                                                  Custom

                                                                                                  +

                                                                                                  DLI jobs use DEW-CSMS' secret management.

                                                                                                  +

                                                                                                  Permission to Use DEW's Encryption Function

                                                                                                  +

                                                                                                  Allowing DLI to access DLI catalogs to retrieve metadata

                                                                                                  +

                                                                                                  Custom

                                                                                                  +

                                                                                                  DLI accesses catalogs to retrieve metadata.

                                                                                                  +

                                                                                                  Permission to Access DLI Catalog Metadata

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +

                                                                                                  Procedure

                                                                                                  Figure 1 Process of creating a custom agency
                                                                                                  +
                                                                                                  +

                                                                                                  Notes and Constraints

                                                                                                  • The custom agency name cannot be the default agency names dli_admin_agency, dli_management_agency, or dli_data_clean_agency. It must be unique.
                                                                                                  • The name of the agency that allows DLI to clear data according to the lifecycle of a table must be dli_data_clean_agency.
                                                                                                  • Custom agencies can be configured only for jobs executed by engines like Flink 1.15, Spark 3.3.1 (Spark general queue scenario), or later versions.
                                                                                                  • Once the agency permissions are updated, your dli_admin_agency will be upgraded to dli_management_agency. This new agency will have the necessary permissions for datasource operations, notifications, and user authorization operations. If you have other agency permission needs, you will need to create custom agencies. For details about dli_management_agency, see DLI Agency Overview.
                                                                                                  • Common scenarios for creating an agency: DLI is allowed to read and write data from and to OBS, dump logs, and read and write Flink checkpoints. DLI is allowed to access DEW to obtain data access credentials and access catalogs to obtain metadata. For details about agency permissions in these scenarios, see Agency Permission Policies in Common Scenarios.
                                                                                                  +
                                                                                                  +

                                                                                                  Step 1: Create a Cloud Service Agency on the IAM Console and Grant Permissions

                                                                                                  1. Log in to the management console.
                                                                                                  2. In the upper right corner of the page, hover over the username and select Identity and Access Management.
                                                                                                  3. In the navigation pane of the IAM console, choose Agencies.
                                                                                                  4. On the displayed page, click Create Agency.
                                                                                                  5. On the Create Agency page, set the following parameters:
                                                                                                    • Agency Name: Enter an agency name, for example, dli_obs_agency_access.
                                                                                                    • Agency Type: Select Cloud service.
                                                                                                    • Cloud Service: This parameter is available only when you select Cloud service for Agency Type. Select Data Lake Insight (DLI) from the drop-down list.
                                                                                                    • Validity Period: Select Unlimited.
                                                                                                    • Description: You can enter Agency with OBS OperateAccess permissions. This parameter is optional.
                                                                                                    +
                                                                                                  6. Click Next.
                                                                                                  7. Click the agency name. On the displayed page, click the Permissions tab. Click Authorize. On the displayed page, click Create Policy.
                                                                                                  8. Configure policy information.
                                                                                                    1. Enter a policy name, for example, dli-obs-agency.
                                                                                                    2. Select JSON.
                                                                                                    3. In the Policy Content area, paste a custom policy.

                                                                                                      In this example, the permissions allow access and usage of OBS in various scenarios. For DLI Flink jobs, this includes downloading OBS objects, obtaining OBS/GaussDB(DWS) data sources (foreign tables), transferring logs, using savepoints, and enabling checkpointing. For DLI Spark jobs, the permissions allow downloading OBS objects and reading/writing OBS foreign tables.

                                                                                                      +
                                                                                                      For how to configure common agency permissions for Flink jobs, see Agency Permission Policies in Common Scenarios.
                                                                                                      {
+    "Version": "1.1",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "obs:bucket:GetBucketPolicy",
+                "obs:bucket:GetLifecycleConfiguration",
+                "obs:bucket:GetBucketLocation",
+                "obs:bucket:ListBucketMultipartUploads",
+                "obs:bucket:GetBucketLogging",
+                "obs:object:GetObjectVersion",
+                "obs:bucket:GetBucketStorage",
+                "obs:bucket:GetBucketVersioning",
+                "obs:object:GetObject",
+                "obs:object:GetObjectVersionAcl",
+                "obs:object:DeleteObject",
+                "obs:object:ListMultipartUploadParts",
+                "obs:bucket:HeadBucket",
+                "obs:bucket:GetBucketAcl",
+                "obs:bucket:GetBucketStoragePolicy",
+                "obs:object:AbortMultipartUpload",
+                "obs:object:DeleteObjectVersion",
+                "obs:object:GetObjectAcl",
+                "obs:bucket:ListBucketVersions",
+                "obs:bucket:ListBucket",
+                "obs:object:PutObject"
+            ],
+            "Resource": [
+                "OBS:*:*:bucket:bucketName",// Replace bucketName with the actual bucket name.
+                "OBS:*:*:object:*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "obs:bucket:ListAllMyBuckets"
+            ]
+        }
+    ]
+}
                                                                                                      +
                                                                                                      +
                                                                                                    4. Enter a policy description as required.
                                                                                                    +
                                                                                                  9. Click Next.
                                                                                                  10. On the Select Policy/Role page, select Custom policy from the first drop-down list and select the custom policy created in 8.
                                                                                                  11. Click Next. On the Select Scope page, set the authorization scope.

                                                                                                    In this example, the custom policy is an OBS agency. So, select Global services. If a DLI agency is used, you are advised to select Region-specific projects.

                                                                                                    +
                                                                                                  12. Click OK.

                                                                                                    It takes 15 to 30 minutes for the authorization to be in effect.

                                                                                                    +
                                                                                                  +
                                                                                                  +

                                                                                                  Step 2: Set Agency Permissions for a Job

                                                                                                  When Flink 1.15, Spark 3.3, or a later version is used to execute jobs, you need to add information about the new agency to the job configuration.

                                                                                                  +

                                                                                                  Otherwise, If you do not specify an agency for Spark 3.3.1 jobs, the jobs cannot use OBS. If you do not specify an agency for a Flink 1.15 job, checkpointing cannot be enabled, savepoints cannot be used, logs cannot be transferred, and data sources such as OBS and GaussDB(DWS) cannot be used.

                                                                                                  +
                                                                                                  • You can only specify an agency for Flink 1.15 and Spark 3.3.1 jobs running on queues in an elastic resource pool.
                                                                                                  • After specifying an agency for a job, be careful when modifying the permissions granted to the agency. Any changes made may impact the job's normal operation.
                                                                                                  +
                                                                                                  +
                                                                                                  • Specifying an agency for a Flink Jar job
                                                                                                    1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                                                                                                    2. Select a desired job and click Edit in the Operation column.
                                                                                                    3. In the job configuration area on the right, configure agency information.
                                                                                                      • Flink Version: Select 1.15.
                                                                                                      • Runtime Configuration: Configure the key-value information of the new agency. The key is fixed to flink.dli.job.agency.name, and the value is the custom agency name.

                                                                                                        In this example, set the key-value information to flink.dli.job.agency.name=dli_obs_agency_access.

                                                                                                        +
                                                                                                      +
                                                                                                    +
                                                                                                  • Specifying an agency for a Flink OpenSource SQL job
                                                                                                    1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                                                                                                    2. Select a desired job and click Edit in the Operation column.
                                                                                                    3. In the job configuration area on the right, configure agency information.
                                                                                                      • On the Running Parameters tab, ensure that the selected Flink version is 1.15.
                                                                                                      • Click Runtime Configuration. Configure the key-value information of the new agency. The key is fixed to flink.dli.job.agency.name, and the value is the custom agency name.

                                                                                                        In this example, set the key-value information to flink.dli.job.agency.name=dli_obs_agency_access.

                                                                                                        +
                                                                                                      +
                                                                                                    +
                                                                                                  • Specifying an agency for a Spark job
                                                                                                    1. Log in to the DLI console. In the navigation pane, choose Job Management > Spark Jobs.
                                                                                                    2. Select the target job and click Edit in the Operation column.
                                                                                                    3. In the job configuration area on the right, configure agency information.
                                                                                                      • Ensure that the selected Spark version is 3.3.1.
                                                                                                      • In the Spark Arguments(--conf) area, configure the key-value information of the new agency. The key is fixed to spark.dli.job.agency.name, and the value is the custom agency name.

                                                                                                        In this example, set the key-value information to spark.dli.job.agency.name=dli_obs_agency_access.

                                                                                                        +
                                                                                                      +
                                                                                                    +
                                                                                                  +
                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0617.html b/docs/dli/umn/dli_01_0617.html new file mode 100644 index 000000000..3f8d3b83c --- /dev/null +++ b/docs/dli/umn/dli_01_0617.html @@ -0,0 +1,152 @@ + + +

                                                                                                  Agency Permission Policies in Common Scenarios

                                                                                                  +

                                                                                                  This section provides agency permission policies for common scenarios, which can be used to configure agency permission policies when you customize your permissions. The "Resource" in the agency policy should be replaced according to specific needs.

                                                                                                  +

                                                                                                  Data Cleanup Agency Permission Configuration

                                                                                                  Application scenario: Data cleanup agency, which is used to clean up data according to the lifecycle of a table and clean up lakehouse table data. You need to create an agency and customize permissions for it. However, the agency name is fixed to dli_data_clean_agency.

                                                                                                  +

                                                                                                  Set the authorization scope of an agency as follows:

                                                                                                  +
                                                                                                  • For an OBS agency, select Global services.
                                                                                                  • For a DLI agency, select Region-specific projects.
                                                                                                  +
                                                                                                  +
                                                                                                  {
+    "Version": "1.1",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "obs:object:GetObject",
+                "obs:object:DeleteObject",
+                "obs:bucket:HeadBucket",
+                "obs:bucket:ListBucket",
+                "obs:object:PutObject"
+            ]
+        }
+    ]
+}
                                                                                                  +

                                                                                                  +
                                                                                                  {
+    "Version": "1.1",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "dli:table:showPartitions",
+                "dli:table:select",
+                "dli:table:dropTable",
+                "dli:table:alterTableDropPartition"
+            ]
+        }
+    ]
+}
                                                                                                  +
                                                                                                  +

                                                                                                  Permission Policies for Accessing and Using OBS

                                                                                                  Application scenario: For DLI Flink jobs, the permissions include downloading OBS objects, obtaining OBS/GaussDB(DWS) data sources (foreign tables), transferring logs, using savepoints, and enabling checkpointing. For DLI Spark jobs, the permissions allow downloading OBS objects and reading/writing OBS foreign tables.

                                                                                                  +
                                                                                                  {
+    "Version": "1.1",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "obs:bucket:GetBucketPolicy",
+                "obs:bucket:GetLifecycleConfiguration",
+                "obs:bucket:GetBucketLocation",
+                "obs:bucket:ListBucketMultipartUploads",
+                "obs:bucket:GetBucketLogging",
+                "obs:object:GetObjectVersion",
+                "obs:bucket:GetBucketStorage",
+                "obs:bucket:GetBucketVersioning",
+                "obs:object:GetObject",
+                "obs:object:GetObjectVersionAcl",
+                "obs:object:DeleteObject",
+                "obs:object:ListMultipartUploadParts",
+                "obs:bucket:HeadBucket",
+                "obs:bucket:GetBucketAcl",
+                "obs:bucket:GetBucketStoragePolicy",
+                "obs:object:AbortMultipartUpload",
+                "obs:object:DeleteObjectVersion",
+                "obs:object:GetObjectAcl",
+                "obs:bucket:ListBucketVersions",
+                "obs:bucket:ListBucket",
+                "obs:object:PutObject"
+            ],
+            "Resource": [
+                "OBS:*:*:bucket:bucketName",// Replace bucketName with the actual bucket name.
+                "OBS:*:*:object:*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "obs:bucket:ListAllMyBuckets"
+            ]
+        }
+    ]
+}
                                                                                                  +
                                                                                                  +

                                                                                                  Permission to Use DEW's Encryption Function

                                                                                                  Application scenario: DLI Flink and Spark jobs use DEW-CSMS' secret management.

                                                                                                  +
                                                                                                  {
+    "Version": "1.1",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "csms:secretVersion:get",
+                "csms:secretVersion:list",
+                "kms:dek:decrypt"
+            ]
+        }
+    ]
+}
                                                                                                  +
                                                                                                  +

                                                                                                  Permission to Access DLI Catalog Metadata

                                                                                                  Application scenario: DLI Flink and Spark jobs are authorized to access DLI metadata.

                                                                                                  +
                                                                                                  {
+    "Version": "1.1",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "dli:table:showPartitions",
+                "dli:table:alterTableAddPartition",
+                "dli:table:alterTableAddColumns",
+                "dli:table:alterTableRenamePartition",
+                "dli:table:delete",
+                "dli:column:select",
+                "dli:database:dropFunction",
+                "dli:table:insertOverwriteTable",
+                "dli:table:describeTable",
+                "dli:database:explain",
+                "dli:table:insertIntoTable",
+                "dli:database:createDatabase",
+                "dli:table:alterView",
+                "dli:table:showCreateTable",
+                "dli:table:alterTableRename",
+                "dli:table:compaction",
+                "dli:database:displayAllDatabases",
+                "dli:database:dropDatabase",
+                "dli:table:truncateTable",
+                "dli:table:select",
+                "dli:table:alterTableDropColumns",
+                "dli:table:alterTableSetProperties",
+                "dli:database:displayAllTables",
+                "dli:database:createFunction",
+                "dli:table:alterTableChangeColumn",
+                "dli:database:describeFunction",
+                "dli:table:showSegments",
+                "dli:database:createView",
+                "dli:database:createTable",
+                "dli:table:showTableProperties",
+                "dli:database:showFunctions",
+                "dli:database:displayDatabase",
+                "dli:table:alterTableRecoverPartition",
+                "dli:table:dropTable",
+                "dli:table:update",
+                "dli:table:alterTableDropPartition"
+            ]
+        }
+    ]
+}
                                                                                                  +
                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0618.html b/docs/dli/umn/dli_01_0618.html new file mode 100644 index 000000000..c571e90d3 --- /dev/null +++ b/docs/dli/umn/dli_01_0618.html @@ -0,0 +1,70 @@ + + +

                                                                                                  Configuring DLI Agency Permissions

                                                                                                  +

                                                                                                  To use DLI, you must first configure permissions.

                                                                                                  +

                                                                                                  This section applies to the following scenarios:

                                                                                                  +
                                                                                                  • If you use DLI for the first time, configure DLI agency permissions by referring to this section.

                                                                                                    DLI needs to work with other cloud services. You must grant DLI basic operation permissions of these services so that DLI can access them and perform resource O&M operations on your behalf.

                                                                                                    +
                                                                                                  • If you are still using the previous-generation agency dli_admin_agency, update it by referring to this section.

                                                                                                    To balance practical business needs with the risk of excessive delegation, DLI upgraded its system agency to achieve more granular control over permissions. The previous dli_admin_agency was upgraded to dli_management_agency, which includes permissions for accessing IAM user information, datasource operations, and message notifications. This effectively prevents uncontrolled permission issues related to the services associated with DLI. After the upgrade, the DLI agency is more flexible and more suitable for scenario-based agency customization for medium- and large-sized enterprises.

                                                                                                    +
                                                                                                  +

                                                                                                  After agency permissions are configured, the dli_management_agency agency is generated on the Agencies page of the IAM console. Do not delete this default system agency. Otherwise, the permissions included in the agency will be automatically revoked. The system cannot obtain IAM user information, access network resources required by datasource connections, or access SMN to send notifications.

                                                                                                  +

                                                                                                  Notes and Constraints

                                                                                                  • Only the tenant account or a member account of user group admin can authorize the service.
                                                                                                  • DLI authorization needs to be conducted by project. The permissions of required agencies must be updated separately in each project. This means you need to switch to the corresponding project and then update the agency by following the instructions provided in this section.
                                                                                                  +
                                                                                                  +

                                                                                                  Updating DLI Agency Permissions (dli_management_agency)

                                                                                                  1. In the navigation pane of the DLI console, choose Global Configuration > Service Authorization.
                                                                                                  2. On the displayed page, select permissions for scenarios.

                                                                                                    Click on a permission card to view its detailed permission policies.

                                                                                                    +
                                                                                                    Table 1 describes these agencies. +
                                                                                                    + + + + + + + + + + + + + + + + + +
                                                                                                    Table 1 Permissions contained in the dli_management_agency agency

                                                                                                    Use Case

                                                                                                    +

                                                                                                    Agency

                                                                                                    +

                                                                                                    Description

                                                                                                    +

                                                                                                    Basic usage

                                                                                                    +

                                                                                                    IAM ReadOnlyAccess

                                                                                                    +

                                                                                                    To authorize IAM users who have not logged in to DLI, you need to obtain their information. So, the permissions contained in the IAM ReadOnlyAccess policy are required.

                                                                                                    +

                                                                                                    IAM ReadOnlyAccess is a global policy. Make sure you select this policy. If you do not select it, all its permissions will become invalid in all regions, and the system cannot obtain IAM user information.

                                                                                                    +

                                                                                                    Datasource

                                                                                                    +

                                                                                                    DLI Datasource Connections Agency Access

                                                                                                    +

                                                                                                    Permissions to access and use VPCs, subnets, routes, and VPC peering connections

                                                                                                    +

                                                                                                    O&M

                                                                                                    +

                                                                                                    DLI Notification Agency Access

                                                                                                    +

                                                                                                    Permissions to send notifications through SMN when a job fails to be executed

                                                                                                    +
                                                                                                    +
                                                                                                    +
                                                                                                    Among the permissions contained in dli_management_agency:
                                                                                                    • The authorization scope of the IAM ReadOnlyAccess policy covers all global service resources in all regions.
                                                                                                      • If you select this policy when updating a DLI agency in any region, this policy's permissions apply to the projects in all regions.
                                                                                                      • If you do not select this policy when updating an agency in any project, this policy's permissions will be revoked from all regions. This means that all projects cannot obtain IAM user information.
                                                                                                      +
                                                                                                    • The authorization scope of the DLI Datasource Connections Agency Access and DLI Notification Agency Access policies covers the project resources in specified regions.

                                                                                                      These policies' permissions only apply to projects for which these policies are selected and the DLI agency permissions are updated. Projects for which these policies are not selected do not have the permissions required in datasource scenarios and the permission to send notifications using SMN.

                                                                                                      +
                                                                                                    +
                                                                                                    +
                                                                                                    +
                                                                                                    +
                                                                                                  3. Select the policies to be included in dli_management_agency and click Update.
                                                                                                  4. View and understand the notes for updating the agency and click OK. The DLI agency permissions are updated.
                                                                                                    • The system upgrades your dli_admin_agency to dli_management_agency.
                                                                                                    • To maintain compatibility with existing job agency permission requirements, dli_admin_agency will still be listed in the IAM agency list even after the update.
                                                                                                    • Do not delete the agency created by the system by default.
                                                                                                    +
                                                                                                  +
                                                                                                  +

                                                                                                  Follow-Up Operations

                                                                                                  In addition to the permissions provided by dli_management_agency, you need to create an agency on the IAM console and add information about the new agency to the job configuration for scenarios like allowing DLI to read and write data from and to OBS to transfer logs, or allowing DLI to access DEW to obtain data access credentials. For details, see Creating a Custom DLI Agency and Agency Permission Policies in Common Scenarios.

                                                                                                  +
                                                                                                  • When Flink 1.15, Spark 3.3.1 (Spark general queue scenario), or a later version is used to execute jobs, you need to create an agency on the IAM console.
                                                                                                  • If the engine version is earlier than Flink 1.15, dli_admin_agency is used by default during job execution. If the engine version is earlier than Spark 3.3.1, user authentication information (AK/SK and security token) is used during job execution.

                                                                                                    This means that jobs whose engine versions are earlier than Flink 1.15 or Spark 3.3.1 are not affected by the update of agency permissions and do not require custom agencies.

                                                                                                    +
                                                                                                  +

                                                                                                  Common service scenarios where you need to create an agency:

                                                                                                  +
                                                                                                  • Data cleanup agency required for clearing data according to the lifecycle of a table and clearing lakehouse table data. You need to create a DLI agency named dli_data_clean_agency on IAM and grant permissions to it. You need to create an agency and customize permissions for it. However, the agency name is fixed to dli_data_clean_agency.
                                                                                                  • Tenant Administrator permissions are required to access data from OBS to execute Flink jobs on DLI, for example, obtaining OBS data sources, log dump (including bucket authorization), checkpointing enabling, and job import and export.
                                                                                                  • The AK/SK required by DLI Flink jobs is stored in DEW. To allow DLI to access DEW data during job execution, you need to create an agency to delegate the permissions to operate on DEW data to DLI.
                                                                                                  • To allow DLI to access DLI catalogs to retrieve metadata when executing jobs, you need to create a new agency that grants DLI catalog data operation permissions to DLI. This will enable DLI to access DLI catalogs on your behalf.
                                                                                                  • Cloud data required by DLI Flink jobs is stored in LakeFormation. To allow DLI to access catalogs to retrieve metadata during job execution, you need to create an agency to delegate the permissions to operate on catalog data to DLI.
                                                                                                  +

                                                                                                  When creating an agency, you cannot use the default agency names dli_admin_agency, dli_management_agency, or dli_data_clean_agency. It must be unique.

                                                                                                  +

                                                                                                  For more information about custom agency operations, see Creating a Custom DLI Agency and Agency Permission Policies in Common Scenarios.

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Preparations
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0621.html b/docs/dli/umn/dli_01_0621.html new file mode 100644 index 000000000..4095bf484 --- /dev/null +++ b/docs/dli/umn/dli_01_0621.html @@ -0,0 +1,102 @@ + + +

                                                                                                  Exporting SQL Job Results

                                                                                                  +

                                                                                                  Store the data results of analyzed SQL jobs in a specified location in the desired format.

                                                                                                  +

                                                                                                  By default, DLI stores SQL job results in its job bucket. You can also download job results to a local host or export job results to a specified OBS bucket.

                                                                                                  +

                                                                                                  Exporting Job Results to the DLI Job Bucket

                                                                                                  DLI specifies a default OBS bucket for storing job results. You can configure the bucket information on the Global Configuration > Project page of the DLI management console. Once a job is complete, the system automatically stores its results to this bucket.

                                                                                                  +
                                                                                                  The following conditions must be met if you want to read job results from the DLI job bucket:
                                                                                                  • You have configured the job bucket on the Global Configuration > Project page of the DLI management console by referring to Configuring a DLI Job Bucket.
                                                                                                  • You have submitted a service ticket to request the whitelisting of the feature that allows writing job results to buckets.
                                                                                                  • The user who executes jobs has been granted read and write permissions either on the job bucket or on the jobs/result path of the job bucket.
                                                                                                  +
                                                                                                  +
                                                                                                  +

                                                                                                  Exporting Job Results to a Specified Location in Another Bucket

                                                                                                  In addition to storing job results in the default bucket, you can also export them to a specified location in another bucket, increasing the flexibility of job result management and making it easier to organize and manage them.

                                                                                                  +

                                                                                                  On the console, you can only view a maximum of 1,000 job results. To view additional results, you can export them to an OBS path. The procedure is as follows:

                                                                                                  +

                                                                                                  You can export job results on either the SQL Jobs or the SQL Editor page.

                                                                                                  +
                                                                                                  • SQL Jobs page: In the navigation pane on the left, choose Job Management > SQL Jobs. On the displayed page, locate the row containing a desired job, click More in the Operation column, and select Export Result.
                                                                                                  • SQL Editor page: In the navigation pane on the left, choose SQL Editor. On the displayed page, once query statements are successfully executed, click next to the View Result tab to export job results.
                                                                                                  +
                                                                                                  • If there are no numerical columns in the query results, job results cannot be exported.
                                                                                                  • Ensure that the user who exports job results has the read and write permissions on the OBS bucket.
                                                                                                  +
                                                                                                  + +
                                                                                                  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                  Table 1 Parameters

                                                                                                  Parameter

                                                                                                  +

                                                                                                  Mandatory

                                                                                                  +

                                                                                                  Description

                                                                                                  +

                                                                                                  Data Format

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  Choose a data format for the job results you want to export. The options include json and csv.

                                                                                                  +

                                                                                                  Queues

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  Select the queue where the job is executed. SQL jobs can only be executed on SQL queues.

                                                                                                  +

                                                                                                  Compression Format

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  Compression format of the data to be exported. The options are:

                                                                                                  +
                                                                                                  • none
                                                                                                  • bzip2
                                                                                                  • deflate
                                                                                                  • gzip
                                                                                                  +

                                                                                                  Storage Path

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  Path in an OBS bucket where the job results are exported

                                                                                                  +
                                                                                                  • If Export Mode is set to New OBS directory, then

                                                                                                    You need to manually enter a directory name and ensure that the directory name does not exist. Otherwise, the system returns an error message and the export operation cannot be performed.

                                                                                                    +
                                                                                                    NOTE:

                                                                                                    The folder name cannot contain special characters (\/:*?"<>|) and cannot start or end with a period (.).

                                                                                                    +
                                                                                                    +

                                                                                                    For example, after selecting the storage path obs://bucket/src1/, you need to manually enter a directory name to change the path to obs://bucket/src1/src2/ and ensure that the src2 directory name does not exist under src1.

                                                                                                    +

                                                                                                    The job result export path is obs://bucket/src1/src2/test.csv.

                                                                                                    +
                                                                                                  • If Export Mode is set to Existing OBS directory (Overwritten), then

                                                                                                    After selecting a bucket path, the job results are exported to that path. If there are files with the same name, they will be automatically overwritten.

                                                                                                    +

                                                                                                    For example, if you select obs://bucket/src1/ as the bucket path, then

                                                                                                    +

                                                                                                    The job result export path is obs://bucket/src1/test.csv.

                                                                                                    +
                                                                                                  +

                                                                                                  Export Mode

                                                                                                  +

                                                                                                  Yes

                                                                                                  +
                                                                                                  • New OBS directory

                                                                                                    If you select this mode, a new folder path is created and the job results are saved to this path. This mode is used when you need to save exported results to a new location, making it easier to manage and track job results.

                                                                                                    +

                                                                                                    If you select this option, you must manually enter an export directory in Storage Path and ensure that the directory must not exist. If the directory already exists, the system displays an error message and the export operation cannot be performed.

                                                                                                    +
                                                                                                  • Existing OBS directory (Overwritten): When exporting job results, you can choose an existing file path as the output directory. If there is a file with the same name in that path, it will be automatically overwritten by the new exported job result file.

                                                                                                    This mode is used when you only need to save a single job result file in the same path, and you do not need to keep old job results.

                                                                                                    +
                                                                                                  +

                                                                                                  Number of Results

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  Number of results to be exported

                                                                                                  +

                                                                                                  If you do not specify or set it to 0, all results will be exported.

                                                                                                  +

                                                                                                  Table Header

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  Whether the job results to be exported contain table headers

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +

                                                                                                  Exporting Job Results to a Local Host

                                                                                                  You can download the results of asynchronous DDL and QUERY statements to a local directory. By default, you can download a maximum of 1,000 data records to a local host.

                                                                                                  +

                                                                                                  The procedure is as follows:

                                                                                                  +
                                                                                                  1. Locate the row containing a desired job whose asynchronous DDL or QUERY statement has been successfully executed, click More in the Operation column, and select Submit Download Request. In the displayed dialog box, click OK. After a few seconds, the Submit Download Request button would change to Download.
                                                                                                  2. Click Download to download the results to your local host.
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Submitting a SQL Job Using DLI
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0622.html b/docs/dli/umn/dli_01_0622.html new file mode 100644 index 000000000..a2a474d5d --- /dev/null +++ b/docs/dli/umn/dli_01_0622.html @@ -0,0 +1,55 @@ + + +

                                                                                                  Viewing Basic Information

                                                                                                  +

                                                                                                  After creating an elastic resource pool, you can check and manage it on the management console.

                                                                                                  +

                                                                                                  This section describes how to check basic information about an elastic resource pool on the management console, including the VPC CIDR block, IPv6 CIDR block, and creation time.

                                                                                                  +

                                                                                                  Procedure

                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                  3. On the displayed page, select the elastic resource pool you want to check.
                                                                                                    • In the upper right corner of the list page, click to customize the columns to display and set the rules for displaying the table content and the Operation column.
                                                                                                    • In the search box above the list, you can filter the required elastic resource pool by name or tag.
                                                                                                    +
                                                                                                  4. Click to expand the basic information card of the elastic resource pool and view detailed information about the pool.

                                                                                                    These details will include the name of the pool, the user who created it, the date it was created, the VPC CIDR block, and whether IPv6 is enabled. If IPv6 is enabled, the subnet's IPv6 CIDR block will also be displayed.

                                                                                                    +

                                                                                                    For the definitions of actual CUs, used CUs, CU range, and specifications of the elastic resource pool, refer to Actual CUs, Used CUs, CU Range, and Specifications of an Elastic Resource Pool.

                                                                                                    +
                                                                                                  +
                                                                                                  +

                                                                                                  Actual CUs, Used CUs, CU Range, and Specifications of an Elastic Resource Pool

                                                                                                  • Actual CUs: number of CUs that can be allocated in the elastic resource pool.
                                                                                                    • Formula for calculating actual CUs:
                                                                                                      • Actual CUs = min{sum(maximum CUs of the queue), maximum CUs of the elastic resource pool}.
                                                                                                      • The calculation result must be a multiple of 16 CUs. If it is not exactly divisible by 16, round up to the nearest multiple.
                                                                                                      +
                                                                                                    • Example of actual CU allocation:

                                                                                                      In Table 1, the calculation process for the actual allocation of CUs in an elastic resource pool is as follows:

                                                                                                      +
                                                                                                      1. Calculate the sum of maximum CUs of the queues: sum(maximum CUs) = 32 + 56 = 88 CUs.
                                                                                                      2. Compare the sum of maximum CUs of the queues with the maximum CUs of the elastic resource pool and take the smaller value: min{88 CUs, 112 CUs} = 88 CUs.
                                                                                                      3. Check if 88 CUs is a multiple of 16 CUs. Since 88 is not divisible by 16, round up to 96 CUs. +
                                                                                                        + + + + + + + + + + + + + + + +
                                                                                                        Table 1 Example of actual CU allocation of an elastic resource pool

                                                                                                        Scenario

                                                                                                        +

                                                                                                        Resource

                                                                                                        +

                                                                                                        CU Range

                                                                                                        +

                                                                                                        New elastic resource pool: 64–112 CUs

                                                                                                        +

                                                                                                        Queues A and B are created within the elastic resource pool. The CU ranges of the two queues are:

                                                                                                        +
                                                                                                        • CU range of queue A: 16–32 CUs
                                                                                                        • CU range of queue B: 16–56 CUs
                                                                                                        +

                                                                                                        Elastic resource pool

                                                                                                        +

                                                                                                        64–112 CUs

                                                                                                        +

                                                                                                        Queue A

                                                                                                        +

                                                                                                        16–32 CUs

                                                                                                        +

                                                                                                        Queue B

                                                                                                        +

                                                                                                        16–56 CUs

                                                                                                        +
                                                                                                        +
                                                                                                        +
                                                                                                      +
                                                                                                    +
                                                                                                  • Used CUs: CUs that have been used by jobs or tasks. These CU resources may be currently engaged in computing tasks and therefore temporarily unavailable.
                                                                                                  • CU range: CU settings are used to control the maximum and minimum CU ranges for elastic resource pools to avoid unlimited resource scaling.
                                                                                                    • The total minimum CUs of all queues in an elastic resource pool must be no more than the minimum CUs of the pool.
                                                                                                    • The maximum CUs of any queue in an elastic resource pool must be no more than the maximum CUs of the pool.
                                                                                                    • An elastic resource pool should at least ensure that all queues in it can run with the minimum CUs and should try to ensure that all queues in it can run with the maximum CUs.
                                                                                                    +
                                                                                                  • Specifications: The minimum CUs selected during elastic resource pool purchase are elastic resource pool specifications.
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Managing Elastic Resource Pools
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0623.html b/docs/dli/umn/dli_01_0623.html new file mode 100644 index 000000000..b35a3fc8e --- /dev/null +++ b/docs/dli/umn/dli_01_0623.html @@ -0,0 +1,17 @@ + + +

                                                                                                  Viewing Basic Information About an Enhanced Datasource Connection

                                                                                                  +

                                                                                                  After creating an enhanced datasource connection, you can view and manage it on the management console.

                                                                                                  +

                                                                                                  This section describes how to view basic information about an enhanced datasource connection on the management console, including whether the enhanced datasource connection supports IPv6, host information, and more.

                                                                                                  +

                                                                                                  Procedure

                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Datasource Connections.
                                                                                                  3. On the displayed Enhanced tab, locate the enhanced datasource connection whose basic information you want to view.
                                                                                                    • In the upper right corner of the list page, click to customize the columns to display and set the rules for displaying the table content and the Operation column.
                                                                                                    • In the search box above the list, you can filter the required enhanced datasource connection by name or tag.
                                                                                                    +
                                                                                                  4. Click to expand details about the enhanced datasource connection.

                                                                                                    You can view the following information:

                                                                                                    +
                                                                                                    1. IPv6 Support: If you selected a subnet with IPv6 enabled when creating the enhanced datasource connection, then your enhanced datasource connection will support IPv6.
                                                                                                    2. Host Information: When accessing an MRS HBase cluster, you need to configure the host name (domain name) and the corresponding IP address of the instance. For details, see Modifying Host Information in an Elastic Resource Pool.
                                                                                                    +
                                                                                                  +
                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0624.html b/docs/dli/umn/dli_01_0624.html deleted file mode 100644 index 8dc87e3b3..000000000 --- a/docs/dli/umn/dli_01_0624.html +++ /dev/null @@ -1,71 +0,0 @@ - - -

                                                                                                  Establishing a Network Connection Between DLI and Resources in a Shared VPC

                                                                                                  -

                                                                                                  VPC Sharing Overview

                                                                                                  VPC sharing allows sharing VPC resources created in one account with other accounts using Resource Access Manager (RAM). For example, account A can share its VPC and subnets with account B. After accepting the share, account B can view the shared VPC and subnets and use them to create resources.

                                                                                                  -
                                                                                                  -

                                                                                                  DLI Use Cases

                                                                                                  An enterprise IT management account creates a VPC and subnets and shares them with other service accounts to facilitate centralized configuration of VPC security policies and orderly resource management.

                                                                                                  -

                                                                                                  Service accounts use the shared VPC and subnets to create resources and want to use DLI to submit jobs and access resources in the shared VPC. To do this, they need to establish a network connection between DLI and the resources in the shared VPC.

                                                                                                  -

                                                                                                  For example, account A is the enterprise IT management account and the owner of VPC resources. It creates the VPC and subnets and shares them with service account B.

                                                                                                  -

                                                                                                  Account B is a service account that uses the shared VPC and subnets to create resources and uses DLI to access them.

                                                                                                  -
                                                                                                  -

                                                                                                  Prerequisites

                                                                                                  Account A, as the resource owner, has created a VPC and subnets and designated account B as the principal.

                                                                                                  -
                                                                                                  -

                                                                                                  Establishing a Network Connection Between DLI and Resources in a Shared VPC

                                                                                                  1. Account A creates an enhanced datasource connection.

                                                                                                    1. Log in to the DLI management console using account A.
                                                                                                    2. In the navigation pane on the left, choose Datasource Connections.
                                                                                                    3. On the displayed Enhanced tab, click Create.

                                                                                                      Set parameters based on Table 1.

                                                                                                      - -
                                                                                                      - - - - - - - - - - - - - - - - - - - - - - -
                                                                                                      Table 1 Parameters for creating an enhanced datasource connection

                                                                                                      Parameter

                                                                                                      -

                                                                                                      Description

                                                                                                      -

                                                                                                      Connection Name

                                                                                                      -

                                                                                                      Name of the datasource connection to be created

                                                                                                      -

                                                                                                      Resource Pool

                                                                                                      -

                                                                                                      You do not need to set this parameter in this scenario.

                                                                                                      -

                                                                                                      VPC

                                                                                                      -

                                                                                                      VPC shared by account A to account B

                                                                                                      -

                                                                                                      Subnet

                                                                                                      -

                                                                                                      Subnet shared by account A to account B

                                                                                                      -

                                                                                                      Host Information

                                                                                                      -

                                                                                                      You do not need to set this parameter in this scenario.

                                                                                                      -

                                                                                                      Tags

                                                                                                      -

                                                                                                      Tags used to identify cloud resources. A tag includes the tag key and tag value.

                                                                                                      -
                                                                                                      -
                                                                                                      -
                                                                                                    4. Click OK.
                                                                                                    -

                                                                                                  2. Account A grants account B access to the enhanced datasource connection created in 1.

                                                                                                    1. In the enhanced datasource connection list, locate the row containing the newly created one, click More in the Operation column, and select Manage Permission from the drop-down list.
                                                                                                    2. In the displayed Permissions dialog box, select Grant Permission for Set Permission, enter the ID of the project account B belongs to in Project ID, and click OK.
                                                                                                    -

                                                                                                  3. Account B binds a DLI elastic resource pool to the shared enhanced datasource connection.

                                                                                                    1. Log in to the DLI management console using account B.
                                                                                                    2. In the navigation pane on the left, choose Datasource Connections.
                                                                                                    3. On the displayed Enhanced tab, locate the row containing the enhanced datasource connection shared by account A, click More in the Operation column, and select Bind Resource Pool from the drop-down list.
                                                                                                    4. In the displayed Bind Resource Pool dialog box, select the created elastic resource pool for Resource Pool and click OK.

                                                                                                      If there is no elastic resource pool available, create one by referring to Creating an Elastic Resource Pool.

                                                                                                      -
                                                                                                    -

                                                                                                  4. Account B tests the network connectivity between the elastic resource pool and resources in the VPC.

                                                                                                    If there are resources in the shared VPC, ensure that the security group the resources belong to has allowed access to the elastic resource pool's CIDR block.

                                                                                                    -
                                                                                                    -
                                                                                                    1. Obtain the private IP address and port number of the data source in the shared VPC.

                                                                                                      Take the RDS data source as an example. On the Instances page, click the target DB instance. On the displayed page, locate the Connection Information pane and view the private IP address. In the Connection Information pane, locate the Database Port to view the port number of the RDS DB instance.

                                                                                                      -
                                                                                                    2. In the navigation pane of the DLI management console, choose Resources > Queue Management.
                                                                                                    3. Locate the queue under the elastic resource pool bound with the enhanced datasource connection, click More in the Operation column, and select Test Address Connectivity.
                                                                                                    4. Enter the data source connection address and port number to test the network connectivity.

                                                                                                      If the address is reachable, it means that account B has established a network connection between the DLI resource and the resources in the shared VPC. Account B can then submit jobs to the elastic resource pool's queue and access the resources in the shared VPC.

                                                                                                      -
                                                                                                    -

                                                                                                  -
                                                                                                  -
                                                                                                  -
                                                                                                  -
                                                                                                  -
                                                                                                  Parent topic: Enhanced Datasource Connections
                                                                                                  -
                                                                                                  -
                                                                                                  - diff --git a/docs/dli/umn/dli_01_0625.html b/docs/dli/umn/dli_01_0625.html new file mode 100644 index 000000000..0143c5db2 --- /dev/null +++ b/docs/dli/umn/dli_01_0625.html @@ -0,0 +1,26 @@ + + +

                                                                                                  Managing Table Resources on the DLI Console

                                                                                                  +

                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0626.html b/docs/dli/umn/dli_01_0626.html new file mode 100644 index 000000000..8591baf39 --- /dev/null +++ b/docs/dli/umn/dli_01_0626.html @@ -0,0 +1,20 @@ + + +

                                                                                                  Deleting a Table on the DLI Console

                                                                                                  +

                                                                                                  You can delete tables as needed.

                                                                                                  +

                                                                                                  Precautions

                                                                                                  • Databases or tables that are being used for running jobs cannot be deleted.
                                                                                                  • Only administrators, table owners, and users with table deletion permission can delete tables.

                                                                                                    Deleted data tables cannot be restored. Exercise caution when performing this operation.

                                                                                                    +
                                                                                                    +
                                                                                                  +
                                                                                                  +

                                                                                                  Deleting a Table

                                                                                                  You can delete a table on either the Data Management page or the SQL Editor page.
                                                                                                  • On the Data Management page
                                                                                                    1. In the navigation pane on the left of the console, choose Data Management > Databases and Tables.
                                                                                                    2. Locate the database whose tables you want to delete and click its name.
                                                                                                    3. On the displayed page, select the table you want to delete, click More in the Operation column, and select Drop Table.
                                                                                                    4. In the displayed dialog box, click Yes.
                                                                                                    +
                                                                                                  • On the SQL Editor page
                                                                                                    1. In the navigation pane on the left of the console, choose SQL Editor.
                                                                                                    2. In the middle pane, click the Databases tab. Click the name of the database whose tables you want to delete.
                                                                                                    3. Click next to the table to delete and select Delete.
                                                                                                    4. In the Drop Table dialog box that appears, click OK.
                                                                                                    +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0627.html b/docs/dli/umn/dli_01_0627.html new file mode 100644 index 000000000..f1530fc8a --- /dev/null +++ b/docs/dli/umn/dli_01_0627.html @@ -0,0 +1,13 @@ + + +

                                                                                                  Changing the Table Owner on the DLI Console

                                                                                                  +

                                                                                                  In practical use, developers create databases and tables, which are then handed over to testers for testing. Once testing is complete, the databases and tables are handed over to O&M personnel for experience. In this scenario, ownership of the data can be transferred to another owner by changing the table owner.

                                                                                                  +

                                                                                                  Changing the Table Owner

                                                                                                  1. In the navigation pane on the left of the console, choose Data Management > Databases and Tables.
                                                                                                  2. Click the name of the database whose tables you want to modify.
                                                                                                  3. Locate the table for which you want to change the owner, click More in the Operation column, and select Modify Owner.
                                                                                                  4. In the displayed Modify Owner dialog box, enter a new owner name (an existing username) and click OK.
                                                                                                  +
                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0634.html b/docs/dli/umn/dli_01_0634.html new file mode 100644 index 000000000..7d34ff539 --- /dev/null +++ b/docs/dli/umn/dli_01_0634.html @@ -0,0 +1,27 @@ + + + +

                                                                                                  Configuring DLI to Read and Write Data from and to External Data Sources

                                                                                                  + +

                                                                                                  +
                                                                                                  + + + diff --git a/docs/dli/umn/dli_01_0635.html b/docs/dli/umn/dli_01_0635.html new file mode 100644 index 000000000..9f16ea3a5 --- /dev/null +++ b/docs/dli/umn/dli_01_0635.html @@ -0,0 +1,17 @@ + + +

                                                                                                  Configuring DLI to Read and Write External Data Sources

                                                                                                  +

                                                                                                  To read and write external data sources when running DLI jobs, two conditions must be met:

                                                                                                  +
                                                                                                  • Establish network connectivity between DLI and the external data source to ensure that the DLI queue is connected to the data source network.
                                                                                                  • Securely store the access credentials for the data source to ensure authentication security and facilitate secure DLI access to the data source.
                                                                                                  +

                                                                                                  This section describes how to configure DLI to read and write external data sources.

                                                                                                  + +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0636.html b/docs/dli/umn/dli_01_0636.html new file mode 100644 index 000000000..f797b62b5 --- /dev/null +++ b/docs/dli/umn/dli_01_0636.html @@ -0,0 +1,32 @@ + + +

                                                                                                  Using DEW to Manage Access Credentials for Data Sources

                                                                                                  +

                                                                                                  When using DLI to submit jobs that involve reading and writing data from external sources, it is crucial to securely access these sources by properly storing their access credentials. This ensures the authentication of the data source and enables secure access by DLI. DEW is a comprehensive cloud-based encryption service that addresses data security, key security, and complex key management issues. This section describes how to use DEW to store authentication information for a data source.

                                                                                                  +

                                                                                                  +

                                                                                                  Creating a Shared Secret in DEW

                                                                                                  This example describes how to configure a credential for accessing RDS DB instances in a DLI job and store the credential in DEW.
                                                                                                  1. Log in to the DEW management console.
                                                                                                  2. In the navigation pane on the left, choose Cloud Secret Management Service > Secrets.
                                                                                                  3. Click Create Secret. On the displayed page, configure basic secret information.
                                                                                                    • Secret Name: Enter a secret name. In this example, the name is secretInfo.
                                                                                                    • Secret Value: Enter the username and password for logging in to the RDS for MySQL DB instance.
                                                                                                      • The key in the first line is MySQLUsername, and the value is the username for logging in to the DB instance.
                                                                                                      • The key in the second line is MySQLPassword, and the value is the password for logging in to the DB instance.
                                                                                                      +
                                                                                                    +
                                                                                                  4. Set other parameters as required and click OK.

                                                                                                    +
                                                                                                  +
                                                                                                  +
                                                                                                  +

                                                                                                  Using the Secret Created in DEW in a DLI Job

                                                                                                  This part uses a Flink job as an example to describe how to use credentials created in DEW.

                                                                                                  +
                                                                                                  WITH (  
+ 'connector' = 'jdbc',  
+  'url? = 'jdbc:mysql://MySQLAddress:MySQLPort/flink',--flink is the MySQL database where the orders table locates.
+ 'table-name' = 'orders',  
+ 'username' = 'MySQLUsername',  -- Shared secret in DEW whose name is secretInfo and version is v1. The key MySQLUsername defines the secret value. The value is the user's sensitive information.
+ 'password' = 'MySQLPassword',  -- Shared secret in DEW whose name is secretInfo and version is v1. The key MySQLPassword defines the secret value. The value is the user's sensitive information.
+ 'sink.buffer-flush.max-rows' = '1',
+ 'dew.endpoint'='kms.xxxx.com', --Endpoint information for the DEW service being used
+ 'dew.csms.secretName'='secretInfo', --Name of the DEW shared secret
+ 'dew.csms.decrypt.fields'='username,password', --The password field value must be decrypted and replaced using DEW secret management.
+ 'dew.csms.version'='v1'
+);
                                                                                                  +
                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0637.html b/docs/dli/umn/dli_01_0637.html new file mode 100644 index 000000000..0f55c753c --- /dev/null +++ b/docs/dli/umn/dli_01_0637.html @@ -0,0 +1,23 @@ + + +

                                                                                                  Querying Logs for SQL Jobs

                                                                                                  +

                                                                                                  Scenario

                                                                                                  DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.

                                                                                                  +

                                                                                                  This section describes how to configure a bucket for DLI jobs on the DLI console and obtain SQL job logs.

                                                                                                  +
                                                                                                  +

                                                                                                  Notes

                                                                                                  • To avoid disordered job results, do not use the OBS bucket configured for DLI jobs for any other purposes.
                                                                                                  • DLI jobs must be set and modified by the main account as IAM users do not have required permissions.
                                                                                                  • You cannot view the logs for DLI jobs before configuring a bucket.
                                                                                                  • You can configure lifecycle rules to periodically delete objects from buckets or change storage classes of objects.
                                                                                                  • Exercise caution when modifying the job bucket, as it may result in the inability to retrieve historical data.
                                                                                                  +
                                                                                                  +

                                                                                                  Prerequisites

                                                                                                  Before the configuration, create an OBS bucket or parallel file system (PFS). In big data scenarios, you are advised to create a PFS. PFS is a high-performance file system provided by OBS, with access latency in milliseconds. PFS can achieve a bandwidth performance of up to TB/s and millions of IOPS, which makes it ideal for processing high-performance computing (HPC) workloads.

                                                                                                  +
                                                                                                  +

                                                                                                  Configuring a Bucket for DLI Jobs

                                                                                                  1. In the navigation pane of the DLI console, choose Global Configuration > Project.
                                                                                                  2. On the Project page, click next to Job Bucket to configure bucket information.
                                                                                                  3. Click to view available buckets.
                                                                                                  4. In the displayed OBS dialog box, click the name of a bucket or search for and click a bucket name and then click OK. In the Set Job Bucket dialog box, click OK.

                                                                                                    Temporary data generated during DLI job running will be stored in the OBS bucket.

                                                                                                    +
                                                                                                  +
                                                                                                  +

                                                                                                  Querying Logs for SQL Jobs

                                                                                                  1. Log in to the DLI console. In the navigation pane on the left, choose Job Management > SQL Jobs.
                                                                                                  2. Select the SQL job whose jobs you want to query, click More in the Operation column, and select View Log.

                                                                                                    The system automatically switches to the log path of the DLI job bucket.

                                                                                                    +
                                                                                                  3. On the Files tab, select the log file of the desired date and time and click Download in the Operation column to download the file to your local host.
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Submitting a SQL Job Using DLI
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0638.html b/docs/dli/umn/dli_01_0638.html new file mode 100644 index 000000000..081d0edb4 --- /dev/null +++ b/docs/dli/umn/dli_01_0638.html @@ -0,0 +1,18 @@ + + +

                                                                                                  Viewing a SQL Execution Plan

                                                                                                  +

                                                                                                  A SQL execution plan is a logical flowchart of a database query that shows how a database management system executes a specific SQL query. The execution plan details the steps needed to execute the query, such as table scans, index lookups, join operations (for example, inner join, outer join), sorting, and aggregation. Viewing an execution plan can help analyze query performance, identify potential performance bottlenecks, understand the query's execution logic, and use this information to adjust the query or database structure to improve SQL query efficiency.

                                                                                                  +

                                                                                                  This section describes how to view a SQL execution plan on the DLI management console.

                                                                                                  +

                                                                                                  Notes and Constraints

                                                                                                  • You can only view SQL execution plans for Spark 3.3.x or later queues and HetuEngine queues.
                                                                                                  • You can only view a SQL execution plan after a SQL job is executed.
                                                                                                  • You can only view the SQL execution plan for SQL jobs that have reached the Finished state.
                                                                                                  • Make sure you have authorized DLI to use OBS buckets for saving the SQL execution plans of user jobs.
                                                                                                  • SQL execution plans are stored in paid storage buckets for DLI jobs. The system does not automatically delete them. You are advised to configure the bucket lifecycle and specify rules to regularly delete or migrate unused SQL execution plans. Refer to Configuring a DLI Job Bucket.
                                                                                                  +
                                                                                                  +

                                                                                                  Procedure

                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Job Management > SQL Jobs.
                                                                                                  3. Select the SQL job you want to query.
                                                                                                  4. At the bottom of the page, click the name of the job you selected to view its details.

                                                                                                    In the details area, click Expand next to SQL Execution Plan. The system queries the SQL execution plan of the job from the DLI job bucket and displays the plan on the console.

                                                                                                    +

                                                                                                    If the SQL execution plan in the DLI job bucket is deleted, the plan may not be displayed because the source file is missing.

                                                                                                    +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Submitting a SQL Job Using DLI
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0639.html b/docs/dli/umn/dli_01_0639.html new file mode 100644 index 000000000..c280cb78f --- /dev/null +++ b/docs/dli/umn/dli_01_0639.html @@ -0,0 +1,19 @@ + + +

                                                                                                  Developing and Submitting a SQL Job Using a SQL Job Template

                                                                                                  +

                                                                                                  DLI allows you to create custom templates or save currently used SQL statements as templates for quick and convenient SQL operations. Once a template is saved, you can execute SQL operations directly through the template without the need to write SQL statements.

                                                                                                  +

                                                                                                  The system offers sample templates that include various standard TPC-H query statements. You can choose to use one of these templates or create a custom template to create a SQL job.

                                                                                                  +

                                                                                                  This example shows how to use a TPC-H sample template to develop and submit a SQL job.

                                                                                                  +

                                                                                                  Procedure

                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Job Templates > SQL Templates.
                                                                                                  3. On the displayed Sample Templates tab, find a sample template that matches your service scenario under tpchQuery and click Execute in the Operation column.
                                                                                                  4. In the upper part of the editing window, set Engine to Spark, Queues to default, and Databases to default, and click Execute.

                                                                                                    +
                                                                                                  5. Check the query result on the View Result tab below the editing window.

                                                                                                    +
                                                                                                  +

                                                                                                  This example uses the default queue and database preset in the system as an example. You can also run the command in a self-created queue and database.

                                                                                                  +

                                                                                                  For details, see "Creating a Queue" in the Data Lake Insight User Guide. For how to create a database, see "Data Management" > "Databases and Tables" > "Creating a Database" in the Data Lake Insight User Guide.

                                                                                                  +
                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0650.html b/docs/dli/umn/dli_01_0650.html new file mode 100644 index 000000000..ee1a0d79f --- /dev/null +++ b/docs/dli/umn/dli_01_0650.html @@ -0,0 +1,33 @@ + + +

                                                                                                  Setting the Priority for a Flink Job

                                                                                                  +

                                                                                                  Scenario

                                                                                                  In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. This requires providing the necessary compute resources for their normal operations.

                                                                                                  +

                                                                                                  DLI offers a feature to set job priorities for each Flink job, which prioritizes the allocation of compute resources to higher priority jobs when resources are limited.

                                                                                                  +

                                                                                                  You can set the priority for Flink 1.12 or later jobs.

                                                                                                  +
                                                                                                  +
                                                                                                  +

                                                                                                  Notes

                                                                                                  • You can assign a priority level of 1 to 10 for each job, with a larger value indicating a higher priority. Compute resources are preferentially allocated to high-priority jobs. That is, if compute resources required for high-priority jobs are insufficient, compute resources for low-priority jobs are reduced.
                                                                                                  • Flink jobs running on a general-purpose queue have a default priority level of 5.
                                                                                                  • The job priority change will only be in effect once the job has been stopped, edited, and resubmitted.
                                                                                                  • You can set the priority for Flink jobs only after enabling dynamic scaling by setting flink.dli.job.scale.enable to true. For details, see Enabling Dynamic Scaling for Flink Jobs.
                                                                                                  • To change the priority for a job, you must first stop the job, change the priority level, and then submit the job for the modification to take effect.
                                                                                                  +
                                                                                                  +

                                                                                                  Setting the Priority for a Flink OpenSource SQL Job

                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Job Management > Flink Jobs.
                                                                                                  3. Select the job for which you want to set the priority and click Edit in the Operation column.
                                                                                                  4. On the far right of the displayed page, click Runtime Configuration.
                                                                                                  5. Enter statements in the text box to enable dynamic scaling and then set the job priority.

                                                                                                    To set the priority for Flink jobs, you must first enable dynamic scaling by setting flink.dli.job.scale.enable to true.

                                                                                                    +

                                                                                                    For more parameter settings, see Enabling Dynamic Scaling for Flink Jobs.

                                                                                                    +
                                                                                                    +
                                                                                                    flink.dli.job.scale.enable=true
+flink.dli.job.priority=x
                                                                                                    +
                                                                                                  +
                                                                                                  +

                                                                                                  Setting the Priority for a Flink Jar Job

                                                                                                  Enter the following statement in the Runtime Configuration text box, where x indicates the priority value:

                                                                                                  +
                                                                                                  flink.dli.job.priority=x
                                                                                                  +
                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Job Management > Flink Jobs.
                                                                                                  3. Select the job for which you want to set the priority and click Edit in the Operation column.
                                                                                                  4. In the Runtime Configuration text box, enter the following statements to enable dynamic scaling and set the job priority:

                                                                                                    To set the priority for Flink jobs, you must first enable dynamic scaling by setting flink.dli.job.scale.enable to true.

                                                                                                    +

                                                                                                    For more parameter settings, see Enabling Dynamic Scaling for Flink Jobs.

                                                                                                    +
                                                                                                    +
                                                                                                    flink.dli.job.scale.enable=true
+flink.dli.job.priority=x
                                                                                                    +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Managing Flink Jobs
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0651.html b/docs/dli/umn/dli_01_0651.html new file mode 100644 index 000000000..59fa8da7a --- /dev/null +++ b/docs/dli/umn/dli_01_0651.html @@ -0,0 +1,29 @@ + + +

                                                                                                  Querying Logs for Flink Jobs

                                                                                                  +

                                                                                                  Scenario

                                                                                                  DLI job buckets are used to store temporary data generated during DLI job running, such as job logs and results.

                                                                                                  +

                                                                                                  This section describes how to configure a bucket for DLI jobs on the DLI console and obtain Flink job logs.

                                                                                                  +
                                                                                                  +

                                                                                                  Notes

                                                                                                  • To avoid disordered job results, do not use the OBS bucket configured for DLI jobs for any other purposes.
                                                                                                  • DLI jobs must be set and modified by the main account as IAM users do not have required permissions.
                                                                                                  • You cannot view the logs for DLI jobs before configuring a bucket.
                                                                                                  • You can configure lifecycle rules to periodically delete objects from buckets or change storage classes of objects.
                                                                                                  • Exercise caution when modifying the job bucket, as it may result in the inability to retrieve historical data.
                                                                                                  +
                                                                                                  +

                                                                                                  Prerequisites

                                                                                                  Before the configuration, create an OBS bucket or parallel file system (PFS). In big data scenarios, you are advised to create a PFS. PFS is a high-performance file system provided by OBS, with access latency in milliseconds. PFS can achieve a bandwidth performance of up to TB/s and millions of IOPS, which makes it ideal for processing high-performance computing (HPC) workloads.

                                                                                                  +
                                                                                                  +

                                                                                                  Configuring a Bucket for DLI Jobs

                                                                                                  1. In the navigation pane of the DLI console, choose Global Configuration > Project.
                                                                                                  2. On the Project page, click next to Job Bucket to configure bucket information.
                                                                                                  3. Click to view available buckets.
                                                                                                  4. In the displayed OBS dialog box, click the name of a bucket or search for and click a bucket name and then click OK. In the Set Job Bucket dialog box, click OK.

                                                                                                    Temporary data generated during DLI job running will be stored in the OBS bucket.

                                                                                                    +
                                                                                                  +
                                                                                                  +

                                                                                                  Viewing Commit Logs

                                                                                                  You can check commit logs to locate commit faults.

                                                                                                  +
                                                                                                  1. In the navigation pane of the DLI console, choose Job Management > Flink Jobs.
                                                                                                  2. Click the name of the Flink job whose commit logs you want to check.
                                                                                                  3. Click the Commit Logs tab and check the job commit process.
                                                                                                  +
                                                                                                  +

                                                                                                  Viewing Run Logs

                                                                                                  You can check run logs to locate job running faults.

                                                                                                  +
                                                                                                  1. In the navigation pane of the DLI console, choose Job Management > Flink Jobs.
                                                                                                  2. Click the name of the Flink job whose commit logs you want to check.
                                                                                                  3. Click the Run Log tab and check the JobManager and TaskManager information of the running job.

                                                                                                    JobManager and TaskManager information is updated every minute. By default, the run logs generated in the last minute are displayed.

                                                                                                    +

                                                                                                    If you have configured an OBS bucket to store job logs, you can access it to download and check historical logs.

                                                                                                    +

                                                                                                    If the job is not running, you cannot check Task Manager information.

                                                                                                    +

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Managing Flink Jobs
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0652.html b/docs/dli/umn/dli_01_0652.html new file mode 100644 index 000000000..a6614def7 --- /dev/null +++ b/docs/dli/umn/dli_01_0652.html @@ -0,0 +1,21 @@ + + +

                                                                                                  Setting the Priority for a Spark Job

                                                                                                  +

                                                                                                  Scenario

                                                                                                  In actual job running, it is necessary to prioritize and ensure the normal running of important and urgent tasks due to their varying levels of importance and urgency. This requires providing the necessary compute resources for their normal operations.

                                                                                                  +

                                                                                                  DLI offers a feature to set job priorities for each Spark job, which prioritizes the allocation of compute resources to higher priority jobs when resources are limited.

                                                                                                  +

                                                                                                  You can set the priority for Spark 2.4.5 or later jobs.

                                                                                                  +
                                                                                                  +
                                                                                                  +

                                                                                                  Notes

                                                                                                  • You can assign a priority level of 1 to 10 for each job, with a larger value indicating a higher priority. Compute resources are preferentially allocated to high-priority jobs. That is, if compute resources required for high-priority jobs are insufficient, compute resources for low-priority jobs are reduced.
                                                                                                  • Spark jobs running on a general-purpose queue have a default priority level of 3.
                                                                                                  • To change the priority for a job, you must first stop the job, change the priority level, and then submit the job for the modification to take effect.
                                                                                                  +
                                                                                                  +

                                                                                                  Procedure

                                                                                                  Enter the following statement in the Spark Arguments(--conf) text box, where x indicates the priority value:

                                                                                                  +
                                                                                                  spark.dli.job.priority=x
                                                                                                  +
                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Job Management > Spark Jobs.
                                                                                                  3. Select the job for which you want to set the priority and click Edit in the Operation column.
                                                                                                  4. Set the spark.dli.job.priority parameter in the Spark Arguments(--conf) text box.
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Submitting a Spark Job Using DLI
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0661.html b/docs/dli/umn/dli_01_0661.html new file mode 100644 index 000000000..a7b4e3f8d --- /dev/null +++ b/docs/dli/umn/dli_01_0661.html @@ -0,0 +1,17 @@ + + + +

                                                                                                  Data Migration and Transmission

                                                                                                  + +

                                                                                                  +
                                                                                                  + + + diff --git a/docs/dli/umn/dli_01_0662.html b/docs/dli/umn/dli_01_0662.html new file mode 100644 index 000000000..18b7b0fec --- /dev/null +++ b/docs/dli/umn/dli_01_0662.html @@ -0,0 +1,16 @@ + + +

                                                                                                  Overview

                                                                                                  +

                                                                                                  Importing Data to OBS

                                                                                                  DLI enables direct access to data stored in OBS for query and analysis, eliminating the need for data migration.

                                                                                                  +

                                                                                                  To begin using DLI for data analysis, just import your local data into OBS.

                                                                                                  +
                                                                                                  +

                                                                                                  Configuring DLI to Read and Write External Data Sources

                                                                                                  If you prefer not to import data into OBS or DLI tables, DLI offers cross-source access capabilities, allowing you to connect to data sources for analysis without the need to migrate the data.

                                                                                                  +

                                                                                                  For details, see Configuring DLI to Read and Write External Data Sources.

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Data Migration and Transmission
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0663.html b/docs/dli/umn/dli_01_0663.html new file mode 100644 index 000000000..3f57fa577 --- /dev/null +++ b/docs/dli/umn/dli_01_0663.html @@ -0,0 +1,16 @@ + + +

                                                                                                  Viewing Basic Information About a Queue

                                                                                                  +

                                                                                                  This section walks you through how to view basic information about a queue on the management console, including the engine type and version.

                                                                                                  +

                                                                                                  Procedure

                                                                                                  1. Log in to the DLI management console.
                                                                                                  2. In the navigation pane on the left, choose Resources > Queue Management.
                                                                                                  3. On the displayed page, locate the queue whose basic information you want to view.
                                                                                                    • In the upper right corner of the list page, click to customize the columns to display and set the rules for displaying the table content and the Operation column.
                                                                                                    • In the search box above the list, you can filter the required queue by name or tag.
                                                                                                    +
                                                                                                  4. Click to view queue details.

                                                                                                    The parameters of the queue are:

                                                                                                    +
                                                                                                    • Engine: type of the engine that executes jobs on the queue.
                                                                                                    • Default Version: default version of the execution engine or the version the system will use if no specific version is specified.
                                                                                                    • Supported Versions: all versions supported by the execution engine. By viewing the supported versions of the queue, you can find out which versions of the execution engine can be used to process jobs on the queue.
                                                                                                    +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Managing Queues
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0664.html b/docs/dli/umn/dli_01_0664.html new file mode 100644 index 000000000..319faf50a --- /dev/null +++ b/docs/dli/umn/dli_01_0664.html @@ -0,0 +1,16 @@ + + +

                                                                                                  Using AOM to Monitor DLI

                                                                                                  +

                                                                                                  +
                                                                                                  + + diff --git a/docs/dli/umn/dli_01_0665.html b/docs/dli/umn/dli_01_0665.html new file mode 100644 index 000000000..49a86821b --- /dev/null +++ b/docs/dli/umn/dli_01_0665.html @@ -0,0 +1,118 @@ + + +

                                                                                                  Connecting DLI to AOM's Prometheus Monitoring

                                                                                                  +

                                                                                                  AOM's Prometheus Monitoring is a comprehensive solution that integrates with the open source Prometheus ecosystem. It supports monitoring of various types of components, provides pre-configured monitoring dashboards, and offers fully managed Prometheus services. It uses Prometheus monitoring to collect, store, and display data from monitored objects, making it suitable for collecting and processing time series databases, especially for monitoring Flink jobs.

                                                                                                  +

                                                                                                  This section describes how to connect DLI to AOM Prometheus for monitoring.

                                                                                                  +

                                                                                                  Notes

                                                                                                  • Only Flink 1.15 supports the connection to AOM's Prometheus Monitoring.
                                                                                                  • A general-purpose AOM Prometheus cluster has been created.
                                                                                                  • Only common AOM Prometheus instances are supported.
                                                                                                  • After an elastic resource pool is connected to a Prometheus instance, the metrics of all newly submitted Flink 1.15 jobs in the elastic resource pool are reported to the bound Prometheus instance. Only basic metrics are reported by default. To report all metrics, set the metrics.reporter.remote.report-all-metrics parameter in Configuration Items for Connecting DLI to AOM's Prometheus Monitoring.
                                                                                                  • The default reporting interval for DLI Flink metrics is 30 seconds, resulting in a slight delay in metric reporting. To adjust the reporting interval, set the metrics.reporter.remote.interval parameter in Configuration Items for Connecting DLI to AOM's Prometheus Monitoring.

                                                                                                    To prevent frequent metric reporting, you are advised not to set this parameter to a small value. The recommended interval is 30 seconds.

                                                                                                    +
                                                                                                  • In Flink 1.15 or later, after an elastic resource pool is unbound from a Prometheus instance, metrics for new jobs are no longer reported to that instance. However, metrics for submitted jobs will continue to be reported until those jobs are completed.
                                                                                                  • In Flink 1.15 or later, if the bound Prometheus instance is changed, metrics for new jobs will be reported to the new instance. However, metrics for submitted jobs will continue to be reported to the original instance until those jobs are completed.
                                                                                                  +
                                                                                                  +

                                                                                                  Step 1: Create an AOM Prometheus Instance

                                                                                                  1. Log in to the AOM 2.0 management console.
                                                                                                  2. In the navigation pane on the left, choose Prometheus Monitoring > Instances. On the displayed page, click Add Prometheus Instance.
                                                                                                  3. Set the instance name, enterprise project, and instance type. +
                                                                                                    + + + + + + + + + + + + + +
                                                                                                    Table 1 Creating a Prometheus instance

                                                                                                    Parameter

                                                                                                    +

                                                                                                    Description

                                                                                                    +

                                                                                                    Instance Name

                                                                                                    +

                                                                                                    Prometheus instance name.

                                                                                                    +

                                                                                                    Enterprise Project

                                                                                                    +

                                                                                                    Enterprise project name.

                                                                                                    +
                                                                                                    • If you have not changed the default value ALL for Enterprise Project in the left navigation pane of the console, select an enterprise project from the drop-down list on this panel.
                                                                                                    • If you have manually chosen a value other than ALL for Enterprise Project, this parameter is automatically populated and cannot be modified.
                                                                                                    +

                                                                                                    Instance Type

                                                                                                    +

                                                                                                    Type of the Prometheus instance. Select Common Prometheus Instance.

                                                                                                    +
                                                                                                    +
                                                                                                    +
                                                                                                  +
                                                                                                  +

                                                                                                  Step 2: Bind an Elastic Resource to an AOM Prometheus Cluster

                                                                                                  1. Log in to the DLI management console. In the navigation pane on the left, choose Resources > Resource Pool.
                                                                                                  2. Locate the elastic resource pool you want to bind, click More in the Operation column, and select Prometheus > Bind Prometheus Instance.
                                                                                                  3. In the dialog box that appears, select the Prometheus cluster created in Step 1: Create an AOM Prometheus Instance.
                                                                                                  4. Click OK.

                                                                                                    Once bound, monitoring metrics for newly submitted jobs are reported to AOM and billed based on AOM billing rules.

                                                                                                    +

                                                                                                    To bind an elastic resource pool to a Prometheus instance, you must have the permission to access AOM Prometheus. Otherwise, your binding fails.

                                                                                                    +
                                                                                                    The permissions include:
                                                                                                    • aom:prometheusInstances:list
                                                                                                    • aom:metric:list
                                                                                                    • aom:metric:get
                                                                                                    +
                                                                                                    +
                                                                                                    +
                                                                                                  +
                                                                                                  +

                                                                                                  Step 3: Create and Submit a Flink Job

                                                                                                  Create a Flink job by referring to Creating a Flink OpenSource SQL Job.

                                                                                                  +

                                                                                                  Set Flink Version to 1.15. Only Flink 1.15 or later supports AOM monitoring.

                                                                                                  +

                                                                                                  After approximately 30 seconds of running the job, the system reports the job's monitoring metrics to the AOM Prometheus instance.

                                                                                                  +
                                                                                                  +

                                                                                                  Step 4: View Monitoring Metrics on the AOM Dashboard

                                                                                                  For the Prometheus monitoring metrics supported by DLI, see Basic Monitoring Metrics Reported by DLI to Prometheus.

                                                                                                  +
                                                                                                  Open an AOM dashboard to view monitoring metrics. You can use either of the following methods to switch to the AOM console:
                                                                                                  • Method 1: Switching from the DLI management console to AOM's Dashboard page
                                                                                                    1. Log in to the DLI management console. In the navigation pane on the left, choose Job Management > Flink Jobs.
                                                                                                    2. On the displayed page, click the name of the job whose monitoring metrics you want to view.
                                                                                                    3. On the Job Details tab, click More in the upper right corner and select Prometheus Monitoring.

                                                                                                      AOM's Dashboard page is displayed.

                                                                                                      +
                                                                                                    +
                                                                                                  • Method 2: Viewing monitoring metrics on AOM's preset dashboard
                                                                                                    1. Log in to the AOM 2.0 management console.
                                                                                                    2. In the navigation pane on the left, choose Dashboard > Dashboard.

                                                                                                      In the navigation pane of the Dashboard page, choose System > Applications. On the displayed page, find the dashboard with type of DLI_FLINK.

                                                                                                      +
                                                                                                    3. Click the name of the dashboard to access its details.
                                                                                                    4. Configure filter criteria to view detailed monitoring metrics.
                                                                                                      By default, all metrics of the current Prometheus instance is displayed. To view metrics for a specific elastic resource pool, job, or even a specific job submission, you need to apply filters based on your specific needs. +
                                                                                                      + + + + + + + + + + + + + + + + + + + +
                                                                                                      Table 2 Monitoring metrics

                                                                                                      Filter

                                                                                                      +

                                                                                                      Description

                                                                                                      +

                                                                                                      Prometheus Instance

                                                                                                      +

                                                                                                      If you filter metrics by Prometheus instance, all metrics of the instance are displayed.

                                                                                                      +

                                                                                                      Resource Pool

                                                                                                      +

                                                                                                      If you filter metrics by elastic resource pool, all metrics of the elastic resource pool are displayed.

                                                                                                      +

                                                                                                      DLI-flink-jobId

                                                                                                      +

                                                                                                      If you filter metrics by DLI Flink job ID, all metrics of the current DLI Flink job are displayed.

                                                                                                      +

                                                                                                      You can query the DLI Flink job ID on the Job Management > Flink Jobs page of the DLI console.

                                                                                                      +

                                                                                                      DLI-flink-job name

                                                                                                      +

                                                                                                      If you filter metrics by DLI Flink job name, all metrics of the current DLI Flink job are displayed.

                                                                                                      +

                                                                                                      You can query the DLI Flink job name on the Job Management > Flink Jobs page of the DLI console.

                                                                                                      +

                                                                                                      jobId

                                                                                                      +

                                                                                                      If you filter metrics by DLI Flink job ID, you can only view the metrics of the current Flink job.

                                                                                                      +
                                                                                                      • View the job ID through the FlinkUI.
                                                                                                      • View the job ID by searching for keywords in the Flink JobManager log.
                                                                                                      +
                                                                                                      +
                                                                                                      +
                                                                                                      +
                                                                                                    +
                                                                                                  +
                                                                                                  +
                                                                                                  +

                                                                                                  (Optional) Step 5: Configure Alarm Notifications for Prometheus Monitoring

                                                                                                  To stay updated on the monitoring status of Prometheus and take prompt action, you need to configure alarm notifications. SMN offers flexible message push capabilities, allowing you to send Prometheus alarm events to various endpoints, enabling multi-channel alarm notifications. This step walks you through how to configure alarm notifications for Prometheus monitoring.

                                                                                                  +
                                                                                                  1. Create an SMN topic and add a subscription to the topic.

                                                                                                    1. Create an SMN topic.
                                                                                                      1. Log in to the SMN management console.
                                                                                                      2. In the navigation pane on the left, choose Topic Management > Topics.
                                                                                                      3. On the displayed Topics page, click Create Topic in the upper right corner.
                                                                                                      4. Set topic parameters.

                                                                                                        Set Topic Name and Display Name.

                                                                                                        +
                                                                                                      5. In Topic Name, enter a topic name. In Display Name, enter a display name.
                                                                                                      +
                                                                                                    2. Add a subscription to the topic.

                                                                                                      To receive messages published to a topic, you need to add subscription endpoints to that topic.

                                                                                                      +
                                                                                                      1. Log in to the SMN management console.
                                                                                                      2. In the navigation pane on the left, choose Topic Management > Topics.
                                                                                                      3. On the displayed Topics page, locate the topic to which you want to add a subscription and click Add Subscription in the Operation column.
                                                                                                      4. In the Add Subscription slide-out panel, select a protocol from the Protocol drop-down list.
                                                                                                      5. In Endpoint, enter a subscription endpoint.

                                                                                                        Once you have added a subscription, SMN will send a confirmation message to the subscription endpoint, which includes a link for confirming the subscription. The subscription confirmation link is valid for 48 hours. Make sure to confirm your subscription on your mobile phone, mailbox, or other endpoints within this time frame.

                                                                                                        +
                                                                                                      +
                                                                                                    +

                                                                                                  2. Create an alarm action rule on the AOM management console.

                                                                                                    You can create an alarm action rule and associate it with an SMN topic and a message template. If the log, resource, or metric data meets the alarm condition, the system sends an alarm notification based on the associated SMN topic and message template.

                                                                                                    +

                                                                                                    Ensure that you have created an SMN topic and added subscriptions to the topic.

                                                                                                    +
                                                                                                    1. Log in to the AOM 2.0 console.
                                                                                                    2. In the navigation pane on the left, choose Alarm Management > Alarm Action Rules.
                                                                                                    3. On the displayed Action Rules tab, click Create.
                                                                                                    4. In the slide-out panel, set the rule name, rule type, and action.

                                                                                                      When the alarm conditions corresponding to the resources are triggered, the system sends alarm notifications based on the associated SMN topics and message templates.

                                                                                                      +
                                                                                                    +

                                                                                                  3. Create a metric alarm rule.

                                                                                                    You can set threshold conditions in metric alarm rules for resource metrics. When a metric value meets a threshold condition, a threshold alarm will be triggered. If there is no metric data available, an insufficient data event will be reported.

                                                                                                    +

                                                                                                    The following uses creating a metric alarm rule from all metrics as an example.

                                                                                                    +
                                                                                                    1. Log in to the AOM 2.0 console.
                                                                                                    2. In the navigation pane on the left, choose Alarm Management > Alarm Rules.
                                                                                                    3. Click Create.
                                                                                                    4. Set basic information and details about the alarm rule.
                                                                                                      • When configuring alarm rules, the Prometheus instance selected should be the one associated with the elastic resource pool where the job requiring alarm notifications is located.
                                                                                                      • Configure advanced settings. The Action Taken for Insufficient Data parameter in the Advanced Settings area is available only when Select from all metrics is selected for Configuration Mode. You are advised to enable Action Taken for Insufficient Data. This determines how the system will handle the absence or insufficient availability of metric data during the configured monitoring period.
                                                                                                      • Action rule for alarm notifications: You are advised to enable Action Rule for alarm notification to ensure that alarms can be notified by email or SMS. Select the alarm action rule configured in 2.
                                                                                                      +
                                                                                                    +

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Using AOM to Monitor DLI
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0666.html b/docs/dli/umn/dli_01_0666.html new file mode 100644 index 000000000..7e2ba9c12 --- /dev/null +++ b/docs/dli/umn/dli_01_0666.html @@ -0,0 +1,110 @@ + + +

                                                                                                  Configuration Items for Connecting DLI to AOM's Prometheus Monitoring

                                                                                                  +

                                                                                                  When connecting DLI to AOM's Prometheus Monitoring, the system automatically configures the parameters listed in table 1 in Configuration Items for Connecting DLI to AOM's Prometheus Monitoring. If the default configurations do not meet your requirements, you can manually configure the following parameters on the Runtime Configuration tab on the right of the Flink job's editing page. Your configurations are preferred.

                                                                                                  + +
                                                                                                  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                  Table 1 Configuration items for Connecting DLI to AOM Prometheus Monitoring

                                                                                                  Parameter

                                                                                                  +

                                                                                                  Mandatory

                                                                                                  +

                                                                                                  Default Value

                                                                                                  +

                                                                                                  Data Type

                                                                                                  +

                                                                                                  Default Value

                                                                                                  +

                                                                                                  Description

                                                                                                  +

                                                                                                  metrics.reporter.remote.interval

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  None

                                                                                                  +

                                                                                                  Duration

                                                                                                  +

                                                                                                  30 SECONDS

                                                                                                  +

                                                                                                  Metric reporting period, which is recommended to be 30 seconds. If you set it to a smaller value, the metric is frequently reported.

                                                                                                  +

                                                                                                  metrics.reporter.remote.remote-write-url

                                                                                                  +

                                                                                                  Yes

                                                                                                  +

                                                                                                  None

                                                                                                  +

                                                                                                  String

                                                                                                  +

                                                                                                  -

                                                                                                  +

                                                                                                  Remote write URL of AOM Prometheus common instances.

                                                                                                  +

                                                                                                  You do not need to manually add https:// to the URL when configuring it. The system will automatically include https:// in the URL.

                                                                                                  +

                                                                                                  Example configuration:

                                                                                                  +
                                                                                                  aom-internal-access.{regionId}.xxxxx.com:8xx3/v1/{projectId}/{prometheusId}/push
                                                                                                  +

                                                                                                  metrics.reporter.remote.report-all-metrics

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  false

                                                                                                  +

                                                                                                  Boolean

                                                                                                  +

                                                                                                  false

                                                                                                  +

                                                                                                  Whether all metrics are reported. The default value is false, indicating that only basic metrics are reported.

                                                                                                  +

                                                                                                  metrics.reporter.remote.pool-name

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  None

                                                                                                  +

                                                                                                  String

                                                                                                  +

                                                                                                  -

                                                                                                  +

                                                                                                  Add the name of the elastic resource pool where the current job is located as a tag to the metric.

                                                                                                  +

                                                                                                  metrics.reporter.remote.dli-job-id

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  None

                                                                                                  +

                                                                                                  String

                                                                                                  +

                                                                                                  -

                                                                                                  +

                                                                                                  Add the ID of the current DLI Flink job as a tag to the metric.

                                                                                                  +

                                                                                                  metrics.reporter.remote.dli-job-name

                                                                                                  +

                                                                                                  No

                                                                                                  +

                                                                                                  None

                                                                                                  +

                                                                                                  String

                                                                                                  +

                                                                                                  -

                                                                                                  +

                                                                                                  Add the name of the current DLI Flink job as a tag to the metric.

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Using AOM to Monitor DLI
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_01_0667.html b/docs/dli/umn/dli_01_0667.html new file mode 100644 index 000000000..4e9699eca --- /dev/null +++ b/docs/dli/umn/dli_01_0667.html @@ -0,0 +1,436 @@ + + +

                                                                                                  Basic Monitoring Metrics Reported by DLI to Prometheus

                                                                                                  +

                                                                                                  Table 1 lists the basic monitoring metrics that are reported by DLI to Prometheus. AOM Prometheus can store these basic metrics for free.

                                                                                                  + +
                                                                                                  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                                                                                                  Table 1 Basic monitoring metrics reported by DLI to Prometheus

                                                                                                  Monitoring Metric

                                                                                                  +

                                                                                                  Description

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_CPU_Load

                                                                                                  +

                                                                                                  JobManager CPU load

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_CPU_Time

                                                                                                  +

                                                                                                  JobManager CPU usage

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_CPU_Load

                                                                                                  +

                                                                                                  TaskManager CPU load

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_CPU_Time

                                                                                                  +

                                                                                                  TaskManager CPU usage

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Heap_Used

                                                                                                  +

                                                                                                  Used heap memory of JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Heap_Committed

                                                                                                  +

                                                                                                  Available JVM heap memory of JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Heap_Max

                                                                                                  +

                                                                                                  Maximum heap memory available for memory management in JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_NonHeap_Used

                                                                                                  +

                                                                                                  Used off-heap memory of JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_NonHeap_Committed

                                                                                                  +

                                                                                                  Available JVM off-heap memory of JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_NonHeap_Max

                                                                                                  +

                                                                                                  Maximum off-heap memory available for memory management in JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Metaspace_Used

                                                                                                  +

                                                                                                  Used memory of the JobManager metaspace memory pool

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Metaspace_Committed

                                                                                                  +

                                                                                                  Available JVM memory of the JobManager metaspace memory pool

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Metaspace_Max

                                                                                                  +

                                                                                                  Maximum memory available in the JobManager metaspace memory pool

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Direct_Count

                                                                                                  +

                                                                                                  Number of buffers in the direct buffer pool of JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Direct_MemoryUsed

                                                                                                  +

                                                                                                  Memory for the direct buffer pool in JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Direct_TotalCapacity

                                                                                                  +

                                                                                                  Total capacity of all buffers in the direct buffer pool of JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Mapped_Count

                                                                                                  +

                                                                                                  Number of buffers in the mapped buffer pool of JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Mapped_MemoryUsed

                                                                                                  +

                                                                                                  Memory for the mapped buffer pool in JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Memory_Mapped_TotalCapacity

                                                                                                  +

                                                                                                  Total capacity of all buffers in the mapped buffer pool of JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_Flink_Memory_Managed_Used

                                                                                                  +

                                                                                                  Managed memory that has been used in JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_Flink_Memory_Managed_Total

                                                                                                  +

                                                                                                  Total managed memory of JobManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Heap_Used

                                                                                                  +

                                                                                                  Used heap memory of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Heap_Committed

                                                                                                  +

                                                                                                  Available JVM heap memory of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Heap_Max

                                                                                                  +

                                                                                                  Maximum heap memory available for memory management in TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_NonHeap_Used

                                                                                                  +

                                                                                                  Used off-heap memory of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_NonHeap_Committed

                                                                                                  +

                                                                                                  Available JVM off-heap memory of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_NonHeap_Max

                                                                                                  +

                                                                                                  Maximum off-heap memory available for memory management in TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Metaspace_Used

                                                                                                  +

                                                                                                  Used memory of the TaskManager metaspace memory pool

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Metaspace_Committed

                                                                                                  +

                                                                                                  Available JVM memory of the TaskManager metaspace memory pool

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Metaspace_Max

                                                                                                  +

                                                                                                  Maximum memory available in the TaskManager metaspace memory pool

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Direct_Count

                                                                                                  +

                                                                                                  Number of buffers in the direct buffer pool of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Direct_MemoryUsed

                                                                                                  +

                                                                                                  Memory for the direct buffer pool in TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Direct_TotalCapacity

                                                                                                  +

                                                                                                  Total capacity of all buffers in the direct buffer pool of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Mapped_Count

                                                                                                  +

                                                                                                  Number of buffers in the mapped buffer pool of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Mapped_MemoryUsed

                                                                                                  +

                                                                                                  Memory for the mapped buffer pool in TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Memory_Mapped_TotalCapacity

                                                                                                  +

                                                                                                  Total capacity of all buffers in the mapped buffer pool of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Flink_Memory_Managed_Used

                                                                                                  +

                                                                                                  Managed memory that has been used in TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Flink_Memory_Managed_Total

                                                                                                  +

                                                                                                  Total managed memory of TaskManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_Threads_Count

                                                                                                  +

                                                                                                  Total number of active threads in JobManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_Threads_Count

                                                                                                  +

                                                                                                  Total number of active threads in TaskManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_GarbageCollector_ConcurrentMarkSweep_Count

                                                                                                  +

                                                                                                  Number of garbage collection (GC) times of the JobManager Concurrent Mark Sweep (CMS) collector

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_GarbageCollector_ConcurrentMarkSweep_Time

                                                                                                  +

                                                                                                  Total time required for the JobManager CMS collector to collect garbage

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_GarbageCollector_ParNew_Count

                                                                                                  +

                                                                                                  Number of GCs in JobManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_GarbageCollector_ParNew_Time

                                                                                                  +

                                                                                                  Time taken for each GC in JobManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_GarbageCollector_ConcurrentMarkSweep_Count

                                                                                                  +

                                                                                                  Number of GCs of the CMS collector in TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_GarbageCollector_ConcurrentMarkSweep_Time

                                                                                                  +

                                                                                                  Total time required for the TaskManager CMS collector to collect garbage

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_GarbageCollector_ParNew_Count

                                                                                                  +

                                                                                                  Number of GCs in TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_GarbageCollector_ParNew_Time

                                                                                                  +

                                                                                                  Time taken for each GC in TaskManager

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_ClassLoader_ClassesLoaded

                                                                                                  +

                                                                                                  Total number of classes that JobManager has loaded since JVM startup

                                                                                                  +

                                                                                                  flink_jobmanager_Status_JVM_ClassLoader_ClassesUnloaded

                                                                                                  +

                                                                                                  Total number of classes that JobManager has unloaded since JVM startup

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_ClassLoader_ClassesLoaded

                                                                                                  +

                                                                                                  Total number of classes that TaskManager has loaded since JVM startup

                                                                                                  +

                                                                                                  flink_taskmanager_Status_JVM_ClassLoader_ClassesUnloaded

                                                                                                  +

                                                                                                  Total number of classes that TaskManager has unloaded since JVM startup

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Network_AvailableMemorySegments

                                                                                                  +

                                                                                                  Number of unused memory segments of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Network_TotalMemorySegments

                                                                                                  +

                                                                                                  Total number of allocated memory segments of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Shuffle_Netty_AvailableMemorySegments

                                                                                                  +

                                                                                                  Number of unused memory segments of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Shuffle_Netty_UsedMemorySegments

                                                                                                  +

                                                                                                  Number of used memory segments of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Shuffle_Netty_TotalMemorySegments

                                                                                                  +

                                                                                                  Number of allocated memory segments of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Shuffle_Netty_AvailableMemory

                                                                                                  +

                                                                                                  Unused memory of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Shuffle_Netty_UsedMemory

                                                                                                  +

                                                                                                  Used memory of TaskManager

                                                                                                  +

                                                                                                  flink_taskmanager_Status_Shuffle_Netty_TotalMemory

                                                                                                  +

                                                                                                  Allocated memory of TaskManager

                                                                                                  +

                                                                                                  flink_jobmanager_job_numRestarts

                                                                                                  +

                                                                                                  Total number of restarts since job submission

                                                                                                  +

                                                                                                  flink_jobmanager_job_lastCheckpointDuration

                                                                                                  +

                                                                                                  Time taken for the latest checkpoint completion

                                                                                                  +

                                                                                                  flink_jobmanager_job_lastCheckpointSize

                                                                                                  +

                                                                                                  Size of the latest checkpoint (may differ from lastCheckpointFullSize if incremental checkpoints are enabled or logs are changed)

                                                                                                  +

                                                                                                  flink_jobmanager_job_numberOfInProgressCheckpoints

                                                                                                  +

                                                                                                  Number of checkpoints in progress

                                                                                                  +

                                                                                                  flink_jobmanager_job_numberOfCompletedCheckpoints

                                                                                                  +

                                                                                                  Number of completed checkpoints

                                                                                                  +

                                                                                                  flink_jobmanager_job_numberOfFailedCheckpoints

                                                                                                  +

                                                                                                  Number of failed checkpoints

                                                                                                  +

                                                                                                  flink_jobmanager_job_totalNumberOfCheckpoints

                                                                                                  +

                                                                                                  Total number of checkpoints

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_numBytesOut

                                                                                                  +

                                                                                                  Total number of bytes output by a task

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_numBytesOutPerSecond

                                                                                                  +

                                                                                                  Total number of bytes output by a task per second

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_isBackPressured

                                                                                                  +

                                                                                                  Whether a backpressure occurs

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_numRecordsIn

                                                                                                  +

                                                                                                  Total number of records received by a task

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_numRecordsInPerSecond

                                                                                                  +

                                                                                                  Total number of records received by a task per second

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_numBytesIn

                                                                                                  +

                                                                                                  Number of bytes received by a task

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_numBytesInPerSecond

                                                                                                  +

                                                                                                  Number of bytes received by a task per second

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_numRecordsOut

                                                                                                  +

                                                                                                  Total number of records sent by a task

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_numRecordsOutPerSecond

                                                                                                  +

                                                                                                  Total number of records sent by a task per second

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_operator_numRecordsIn

                                                                                                  +

                                                                                                  Total number of records received by an operator

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_operator_numRecordsInPerSecond

                                                                                                  +

                                                                                                  Total number of records received by an operator per second

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_operator_numRecordsOut

                                                                                                  +

                                                                                                  Total number of records sent by an operator

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_operator_numRecordsOutPerSecond

                                                                                                  +

                                                                                                  Total number of records sent by an operator per second

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_operator_sourceIdleTime

                                                                                                  +

                                                                                                  Idle duration of the source

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_operator_currentEmitEventTimeLag

                                                                                                  +

                                                                                                  Time interval between event time and the time the data leaves the source

                                                                                                  +

                                                                                                  flink_taskmanager_job_task_operator_pendingRecords

                                                                                                  +

                                                                                                  Number of data records not yet pulled by the source

                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  +
                                                                                                  Parent topic: Using AOM to Monitor DLI
                                                                                                  +
                                                                                                  +
                                                                                                  + diff --git a/docs/dli/umn/dli_03_0001.html b/docs/dli/umn/dli_03_0001.html index b3ee22483..4c6f6cc45 100644 --- a/docs/dli/umn/dli_03_0001.html +++ b/docs/dli/umn/dli_03_0001.html @@ -1,14 +1,22 @@ -

                                                                                                  Product Consultation

                                                                                                  -
                                                                                                  +

                                                                                                  DLI Basics

                                                                                                  +
                                                                                                  diff --git a/docs/dli/umn/dli_03_0008.html b/docs/dli/umn/dli_03_0008.html index 9d76034f9..e47424b99 100644 --- a/docs/dli/umn/dli_03_0008.html +++ b/docs/dli/umn/dli_03_0008.html @@ -1,13 +1,13 @@ -

                                                                                                  What Is Column Permission Granting of a DLI Partition Table?

                                                                                                  -

                                                                                                  You cannot perform permission-related operations on the partition column of a partitioned table.

                                                                                                  -

                                                                                                  However, when you grant the permission of any non-partition column in a partitioned table to another user, the user gets the permission of the partition column by default.

                                                                                                  -

                                                                                                  When the user views the permission of the partition table, the permission of the partition column will not be displayed.

                                                                                                  +

                                                                                                  What Is Column-Level Authorization for DLI Partitioned Tables?

                                                                                                  +

                                                                                                  You are unable to perform permission operations on the partition columns of partitioned tables.

                                                                                                  +

                                                                                                  However, when you grant the permission of any non-partition column in a partitioned table to another user, the user gets the permission of the partition column by default.

                                                                                                  +

                                                                                                  When the user views the permission of the partition table, the permission of the partition column will not be displayed.

                                                                                                  -
                                                                                                  Parent topic: Usage
                                                                                                  +
                                                                                                  Parent topic: DLI Permissions Management
                                                                                                  diff --git a/docs/dli/umn/dli_03_0009.html b/docs/dli/umn/dli_03_0009.html index ba3dba028..bb5057ca6 100644 --- a/docs/dli/umn/dli_03_0009.html +++ b/docs/dli/umn/dli_03_0009.html @@ -1,11 +1,13 @@ -

                                                                                                  How Can I Avoid Garbled Characters Caused by Inconsistent Character Codes?

                                                                                                  -

                                                                                                  DLI supports only UTF-8-encoded texts. Ensure that data is encoded using UTF-8 during table creation and import.

                                                                                                  +

                                                                                                  How Do I Do If Inconsistent Character Encoding Leads to Garbled Characters?

                                                                                                  +

                                                                                                  To avoid garbled characters caused by inconsistent character encoding, you are advised to unify the encoding format of your data source when executing jobs in DLI.

                                                                                                  +

                                                                                                  DLI only supports UTF-8-encoded text, so your data needs to be encoded in UTF-8 when creating tables and importing data.

                                                                                                  +

                                                                                                  Before importing data into DLI, make sure that the source data file (such as CSV, JSON, etc.) is saved in UTF-8 encoding. If the data source is not encoded in UTF-8, convert it to UTF-8 encoding before importing.

                                                                                                  -
                                                                                                  Parent topic: O&M Guide
                                                                                                  +
                                                                                                  Parent topic: DLI Databases and Tables
                                                                                                  diff --git a/docs/dli/umn/dli_03_0010.html b/docs/dli/umn/dli_03_0010.html deleted file mode 100644 index eedf23343..000000000 --- a/docs/dli/umn/dli_03_0010.html +++ /dev/null @@ -1,11 +0,0 @@ - - -

                                                                                                  How Can I Use DLI If Data Is Not Uploaded to OBS?

                                                                                                  -

                                                                                                  Currently, DLI supports analysis only on the data uploaded to the cloud. In scenarios where regular (for example, on a per day basis) one-off analysis on incremental data is conducted for business, you can do as follows: Anonymize data to be analyzed and store anonymized data on OBS temporarily. After analysis is complete, export the analysis report and delete the data temporarily stored on OBS.

                                                                                                  -
                                                                                                  -
                                                                                                  -
                                                                                                  -
                                                                                                  Parent topic: Usage
                                                                                                  -
                                                                                                  -
                                                                                                  - diff --git a/docs/dli/umn/dli_03_0011.html b/docs/dli/umn/dli_03_0011.html deleted file mode 100644 index c11ee61e1..000000000 --- a/docs/dli/umn/dli_03_0011.html +++ /dev/null @@ -1,12 +0,0 @@ - - -

                                                                                                  How Can I Perform Query on Data Stored on Services Rather Than DLI?

                                                                                                  -

                                                                                                  To perform query on data stored on services rather than DLI, perform the following steps:

                                                                                                  -
                                                                                                  1. Assume that the data to be queried is stored on multiple services (for example, OBS) on cloud.
                                                                                                  2. On the DLI management console, create a table and set the Path parameter to the save path of the target data, for example, path of an OBS bucket. The data is actually stored on OBS and data migration is not required.
                                                                                                  3. On the DLI management console, edit SQL statements to query and analyze the data.
                                                                                                  -
                                                                                                  -
                                                                                                  -
                                                                                                  -
                                                                                                  Parent topic: Cross-Source Analysis
                                                                                                  -
                                                                                                  -
                                                                                                  - diff --git a/docs/dli/umn/dli_03_0013.html b/docs/dli/umn/dli_03_0013.html index 7733a01b4..1f9e130a3 100644 --- a/docs/dli/umn/dli_03_0013.html +++ b/docs/dli/umn/dli_03_0013.html @@ -1,11 +1,14 @@ -

                                                                                                  The Compression Ratio of OBS Tables Is Too High

                                                                                                  -

                                                                                                  A high compression ratio of OBS tables in the Parquet or ORC format (for example, a compression ratio of 5 or higher compared with text compression) will lead to large data volumes to be processed by a single task. In this case, you are advised to set dli.sql.files.maxPartitionBytes to 33554432 (default: 134217728) in the conf field in the submit-job request body to reduce the data to be processed per task.

                                                                                                  +

                                                                                                  How Do I Do If the Compression Rate of an OBS Table Is High?

                                                                                                  +

                                                                                                  When submitting a job to import data into a DLI table, if the compression rate of the Parquet/ORC file corresponding to the OBS table is high, exceeding 5 times the compression rate, you can optimize the job performance by adjusting the configuration.

                                                                                                  +

                                                                                                  Specifically, configure dli.sql.files.maxPartitionBytes=33554432 in the conf field of the submit-job request body.

                                                                                                  +

                                                                                                  The default value of this configuration item is 128 MB. Configuring it to 32 MB can reduce the amount of data read by a single task and avoid processing a large amount of data by a single task after decompression due to a high compression rate.

                                                                                                  +

                                                                                                  However, adjusting this parameter may affect the execution efficiency and resource consumption of the job. Therefore, when making adjustments, you need to choose a suitable parameter value based on the actual amount of data and compression rate.

                                                                                                  -
                                                                                                  Parent topic: O&M Guide
                                                                                                  +
                                                                                                  Parent topic: DLI Databases and Tables
                                                                                                  diff --git a/docs/dli/umn/dli_03_0014.html b/docs/dli/umn/dli_03_0014.html index 077f41f07..4777c2458 100644 --- a/docs/dli/umn/dli_03_0014.html +++ b/docs/dli/umn/dli_03_0014.html @@ -1,11 +1,16 @@

                                                                                                  Why Is Error "path obs://xxx already exists" Reported When Data Is Exported to OBS?

                                                                                                  -

                                                                                                  Create an OBS directory with a unique name. Alternatively, you can manually delete the existing OBS directory and submit the job again. However, exercise caution when deleting the existing OBS directory because the operation will delete all data in the directory.

                                                                                                  +

                                                                                                  This message indicates that you are exporting data to an existing OBS path.

                                                                                                  +

                                                                                                  Solution:

                                                                                                  +
                                                                                                  • Create an OBS directory.

                                                                                                    You can create an OBS directory that does not exist to store the data to be exported.

                                                                                                    +
                                                                                                  • Delete the existing OBS directory.

                                                                                                    Deleting an existing OBS directory will also delete all data in the directory. Perform this operation with caution.

                                                                                                    +
                                                                                                  • Check permissions on the directory.

                                                                                                    Make sure you have the permission to access and write data to the OBS path. If you do not have the permission, contact the administrator to obtain it.

                                                                                                    +
                                                                                                  -
                                                                                                  Parent topic: Job O&M Errors
                                                                                                  +
                                                                                                  Parent topic: SQL Job O&M
                                                                                                  diff --git a/docs/dli/umn/dli_03_0017.html b/docs/dli/umn/dli_03_0017.html index 83dc65b1c..ae24e6ef4 100644 --- a/docs/dli/umn/dli_03_0017.html +++ b/docs/dli/umn/dli_03_0017.html @@ -1,21 +1,21 @@ -

                                                                                                  How Do I Set the AK/SK for a Queue to Operate an OBS Table?

                                                                                                  -

                                                                                                  Setting Up a Spark Jar Job to Obtain the AK/SK

                                                                                                  • To obtain the AK/SK, set the parameters as follows:
                                                                                                    • Create a SparkContext using code.
                                                                                                      val sc: SparkContext = new SparkContext()
+
                                                                                                      How Do I Set Up AK/SK So That a General Queue Can Access Tables Stored in OBS?
                                                                                                      
+
                                                                                                      Setting Up a Spark Jar Job to Obtain the AK/SK
                                                                                                      • To obtain the AK/SK, set the parameters as follows:
                                                                                                        • Create a SparkContext using code.
                                                                                                          val sc: SparkContext = new SparkContext()
 sc.hadoopConfiguration.set("fs.obs.access.key", ak)
 sc.hadoopConfiguration.set("fs.obs.secret.key", sk)
                                                                                                          
-
                                                                                                        • Create a SparkSession using code.
                                                                                                          val sparkSession: SparkSession = SparkSession
+
                                                                                                        • Create a SparkSession using code.
                                                                                                          val sparkSession: SparkSession = SparkSession
       .builder()
       .config("spark.hadoop.fs.obs.access.key", ak)
       .config("spark.hadoop.fs.obs.secret.key", sk)
       .enableHiveSupport()
       .getOrCreate()
                                                                                                          
 
                                                                                                        
-
                                                                                                      • To obtain the AK/SK and security token and use them together for authentication, set the parameters as follows:
                                                                                                        • Create a SparkContext using code.
                                                                                                          val sc: SparkContext = new SparkContext()
+
                                                                                                        • To obtain the AK/SK and security token and use them together for authentication, set the parameters as follows:
                                                                                                          • Create a SparkContext using code.
                                                                                                            val sc: SparkContext = new SparkContext()
 sc.hadoopConfiguration.set("fs.obs.access.key", ak)
 sc.hadoopConfiguration.set("fs.obs.secret.key", sk)
 sc.hadoopConfiguration.set("fs.obs.session.token", sts)
                                                                                                            
-
                                                                                                          • Create a SparkSession using code.
                                                                                                            val sparkSession: SparkSession = SparkSession
+
                                                                                                          • Create a SparkSession using code.
                                                                                                            val sparkSession: SparkSession = SparkSession
       .builder()
       .config("spark.hadoop.fs.obs.access.key", ak)
       .config("spark.hadoop.fs.obs.secret.key", sk)
@@ -28,7 +28,7 @@ sc.hadoopConfiguration.set("fs.obs.session.token", sts)
                                                                                                            
 
                                                                                                      
 
                                                                                                      
 
                                                                                                      
-
                                                                                                      Parent topic: Job Development
                                                                                                      
+
                                                                                                      Parent topic: Spark Job Development
                                                                                                      
 
                                                                                                      
 
                                                                                                      
 
diff --git a/docs/dli/umn/dli_03_0020.html b/docs/dli/umn/dli_03_0020.html
index f2067d07b..e7d419de6 100644
--- a/docs/dli/umn/dli_03_0020.html
+++ b/docs/dli/umn/dli_03_0020.html
@@ -1,16 +1,12 @@
 
 
-
                                                                                                      Problems Related to SQL Jobs
                                                                                                      
-
                                                                                                      
+
                                                                                                      SQL Jobs
                                                                                                      
+
                                                                                                      
 
                                                                                                      
 
 
diff --git a/docs/dli/umn/dli_03_0021.html b/docs/dli/umn/dli_03_0021.html
index f220113c9..035bf244c 100644
--- a/docs/dli/umn/dli_03_0021.html
+++ b/docs/dli/umn/dli_03_0021.html
@@ -1,16 +1,12 @@
 
 
-
                                                                                                      Problems Related to Spark Jobs
                                                                                                      
-
                                                                                                      
+
                                                                                                      Spark Jobs
                                                                                                      
+
                                                                                                      
 
                                                                                                      
 
 
diff --git a/docs/dli/umn/dli_03_0022.html b/docs/dli/umn/dli_03_0022.html
index 1fe0c02e7..87f258276 100644
--- a/docs/dli/umn/dli_03_0022.html
+++ b/docs/dli/umn/dli_03_0022.html
@@ -1,15 +1,37 @@
 
 
-
                                                                                                      Datasource Connections
                                                                                                      
-
                                                                                                      
+
                                                                                                      Enhanced Datasource Connections
                                                                                                      
+
                                                                                                      
 
                                                                                                      
 
                                                                                                      
 
 
diff --git a/docs/dli/umn/dli_03_0025.html b/docs/dli/umn/dli_03_0025.html
deleted file mode 100644
index 7e48a1d28..000000000
--- a/docs/dli/umn/dli_03_0025.html
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
-
                                                                                                      Which Data Formats Does DLI Support?
                                                                                                      
-
                                                                                                      DLI supports the following data formats:
                                                                                                      
-
                                                                                                      • Parquet
                                                                                                      • CSV
                                                                                                      • ORC
                                                                                                      • Json
                                                                                                      • Avro
                                                                                                      
-
                                                                                                      
-
                                                                                                      
-
                                                                                                      
-
                                                                                                      Parent topic: Usage
                                                                                                      
-
                                                                                                      
-
                                                                                                      
-
diff --git a/docs/dli/umn/dli_03_0028.html b/docs/dli/umn/dli_03_0028.html
index 2a5ea254a..dd9702261 100644
--- a/docs/dli/umn/dli_03_0028.html
+++ b/docs/dli/umn/dli_03_0028.html
@@ -1,11 +1,12 @@
 
 
-
                                                                                                      How Do I Set the Auto-increment Primary Key or Other Fields That Are Automatically Filled in the RDS Table When Creating a DLI and Associating It with the RDS Table?
                                                                                                      
-
                                                                                                      When data is inserted into DLI, set the ID field to NULL.
                                                                                                      
+
                                                                                                      How Do I Insert Data into an RDS Table with an Auto-Increment Primary Key Using DLI?
                                                                                                      
+
                                                                                                      When creating an associated RDS table in DLI, if the RDS table contains an auto-increment primary key or other auto-populated fields, you can take the following measures when inserting data in DLI:
                                                                                                      
+
                                                                                                      1. Omit auto-populated fields when inserting data: When inserting data in DLI, for auto-increment primary key fields or other auto-populated fields, you can omit these fields in the insert statement. The database will automatically generate values for these fields. For example, if there is an auto-increment primary key field named id in the table, you can exclude this field when inserting data, and the database will assign a unique id value for the newly inserted row.
                                                                                                      2. Use NULL values: If you need to explicitly specify that certain fields should be auto-populated by the database when inserting data, you can fill in NULL values in those fields. This way, the database will recognize that these fields should be automatically generated by the system, rather than specified by the user.
                                                                                                      
 
                                                                                                      
 
 
diff --git a/docs/dli/umn/dli_03_0029.html b/docs/dli/umn/dli_03_0029.html
index 09033cc93..233a08a9f 100644
--- a/docs/dli/umn/dli_03_0029.html
+++ b/docs/dli/umn/dli_03_0029.html
@@ -1,12 +1,18 @@
 
 
-
                                                                                                      Where Can DLI Data Be Stored?
                                                                                                      
-
                                                                                                      DLI data can be stored in either of the following:
                                                                                                      
-
                                                                                                      • OBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.
                                                                                                      • DLI: The column-based Parquet format is used in DLI. That is, the data is stored in the Parquet format. The storage cost is relatively high.
                                                                                                      • Datasource connection jobs can be stored in the connected services. Currently, CloudTable, CSS, DCS, DDS, GaussDB(DWS), MRS, and RDS are supported.
                                                                                                      
+
                                                                                                      Where Can Data Be Stored in DLI?
                                                                                                      
+
                                                                                                      What Data Formats Can Be Stored in DLI?
                                                                                                      Supported data formats:
                                                                                                      
+
                                                                                                      • Parquet
                                                                                                      • CSV
                                                                                                      • ORC
                                                                                                      • JSON
                                                                                                      • Avro
                                                                                                      
+
                                                                                                      
+
                                                                                                      Where Can Data Be Stored in DLI?
                                                                                                      • OBS: Data used by SQL jobs, Spark jobs, and Flink jobs can be stored in OBS, reducing storage costs.
                                                                                                      • DLI: The column-based Parquet format is used in DLI. That is, the data is stored in the Parquet format. The storage cost is relatively high.
                                                                                                      • Datasource connection jobs can be stored in the connected services. Currently, CloudTable, CSS, DCS, DDS, GaussDB(DWS), MRS, and RDS are supported.
                                                                                                      
+
                                                                                                      
+
                                                                                                      What Are the Differences Between DLI Tables and OBS Tables?
                                                                                                      • DLI tables store data within the DLI service, keeping you unaware of the storage path.
                                                                                                      • OBS tables store data in your OBS buckets, allowing you to manage source data files.
                                                                                                      
+
                                                                                                      DLI tables offer advanced features like permission control and caching acceleration, providing better performance than foreign tables. However, they come with storage fees.
                                                                                                      
+
                                                                                                      
 
                                                                                                      
 
                                                                                                      
 
                                                                                                      
-
                                                                                                      Parent topic: Usage
                                                                                                      
+
                                                                                                      Parent topic: DLI Basics
                                                                                                      
 
                                                                                                      
 
                                                                                                      
 
diff --git a/docs/dli/umn/dli_03_0031.html b/docs/dli/umn/dli_03_0031.html
index c001beb49..c4e80e760 100644
--- a/docs/dli/umn/dli_03_0031.html
+++ b/docs/dli/umn/dli_03_0031.html
@@ -1,13 +1,13 @@
 
 
 
                                                                                                      How Do I View My Quotas?
                                                                                                      
-
                                                                                                      1. Log in to the management console.
                                                                                                      2. Click  in the upper left corner and select Region and Project.
                                                                                                      3. Click  (the My Quotas icon) in the upper right corner.
                                                                                                        The Service Quota page is displayed.
                                                                                                        
-
                                                                                                      4. View the used and total quota of each type of resources on the displayed page.
                                                                                                        If a quota cannot meet service requirements, increase a quota.
                                                                                                        
+
                                                                                                        1. Log in to the management console.
                                                                                                        2. Click  in the upper left corner and select a region and a project.
                                                                                                        3. Click the My Quota icon  in the upper right corner of the page.
                                                                                                          The Service Quota page is displayed.
                                                                                                          
+
                                                                                                        4. View the used and total quota of each type of resources on the displayed page.
                                                                                                          If a quota cannot meet service requirements, increase a quota.
                                                                                                          
 
                                                                                                        
 
                                                                                                        
 
                                                                                                        
 
                                                                                                        
-
                                                                                                        Parent topic: Quota
                                                                                                        
+
                                                                                                        Parent topic: DLI Resource Quotas
                                                                                                        
 
                                                                                                        
 
                                                                                                        
 
diff --git a/docs/dli/umn/dli_03_0032.html b/docs/dli/umn/dli_03_0032.html
index f196f3b0a..71657ecfa 100644
--- a/docs/dli/umn/dli_03_0032.html
+++ b/docs/dli/umn/dli_03_0032.html
@@ -1,17 +1,17 @@
 
 
-
                                                                                                        How Do I Increase a Quota?
                                                                                                        
-
                                                                                                        Increasing a Quota
                                                                                                        The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your application and inform you of the progress by phone call or email.
                                                                                                        
-
                                                                                                        Before dialing the hotline number or sending an email, ensure that the following information has been obtained:
                                                                                                        
-
                                                                                                        • Domain name, project name, and project ID
                                                                                                          To obtain the preceding information, log in to the management console, click the username in the upper right corner, and choose My Credentials from the drop-down list.
                                                                                                          
-
                                                                                                        • Quota information, including:
                                                                                                          • Service name
                                                                                                          • Quota type
                                                                                                          • Required quota
                                                                                                          
+
                                                                                                          How Do I Apply for a Higher Quota?
                                                                                                          
+
                                                                                                          How Do I Apply for a Higher Quota?
                                                                                                          The system does not support online quota adjustment. To increase a resource quota, dial the hotline or send an email to the customer service. We will process your application and inform you of the progress by phone call or email.
                                                                                                          
+
                                                                                                          Before dialing the hotline number or sending an email, ensure that the following information has been obtained:
                                                                                                          
+
                                                                                                          • Domain name, project name, and project ID
                                                                                                            To obtain the preceding information, log in to the management console, click the username in the upper right corner, and choose My Credentials from the drop-down list.
                                                                                                            
+
                                                                                                          • Quota information, including:
                                                                                                            • ServiceName
                                                                                                            • Quota type
                                                                                                            • Required quota
                                                                                                            
 
                                                                                                          
-
                                                                                                          Learn how to obtain the service hotline and email address.
                                                                                                          
+
                                                                                                          Learn how to obtain the service hotline and email address.
                                                                                                          
 
                                                                                                          
 
                                                                                                          
 
                                                                                                          
 
                                                                                                          
-
                                                                                                          Parent topic: Quota
                                                                                                          
+
                                                                                                          Parent topic: DLI Resource Quotas
                                                                                                          
 
                                                                                                          
 
                                                                                                          
 
diff --git a/docs/dli/umn/dli_03_0036.html b/docs/dli/umn/dli_03_0036.html
index a4bff550e..5be818b6f 100644
--- a/docs/dli/umn/dli_03_0036.html
+++ b/docs/dli/umn/dli_03_0036.html
@@ -1,12 +1,16 @@
 
 
 
                                                                                                          Why Is a Message Displayed Indicating That the SMN Topic Does Not Exist When I Use the SMN Topic in DLI?
                                                                                                          
-
                                                                                                          When you set running parameters of a DLI Flink job, you can enable Alarm Generation upon Job Exception to receive alarms when the job runs abnormally or is in arrears.
                                                                                                          
-
                                                                                                          If the SMN topic you select for your DLI job does not exist, log in to the Identity and Access Management (IAM) console, select the user group to which the IAM user belongs, and add the SMN policy for your region to the group.
                                                                                                          
+
                                                                                                          When you set running parameters of a DLI Flink job, you can enable Alarm Generation upon Job Exception to receive alarms when the job runs abnormally or is in arrears.
                                                                                                          
+
                                                                                                          If a message is displayed indicating that the SMN topic does not exist, perform the following steps:
                                                                                                          
+
                                                                                                          1. Check whether the SMN topic has been created.
                                                                                                            If not, create a topic on the SMN console.
                                                                                                            
+
                                                                                                          2. Check IAM permissions.
                                                                                                            If the SMN topic already exists but the system still prompts that it does not exist, go to the IAM service, select the user group where the corresponding IAM user is, and ensure that the SMN policy for the corresponding region has been added to the user group.
                                                                                                            
+
                                                                                                          3. Confirm the topic name and region.
                                                                                                            Make sure that the SMN topic name and region configured in DLI match the actual SMN topic created. If the SMN topic name is not consistent, the system will prompt that the SMN topic does not exist.
                                                                                                            
+
                                                                                                          
 
                                                                                                          
 
                                                                                                          
 
                                                                                                          
-
                                                                                                          Parent topic: Usage
                                                                                                          
+
                                                                                                          Parent topic: Flink Job Consulting
                                                                                                          
 
                                                                                                          
 
                                                                                                          
 
diff --git a/docs/dli/umn/dli_03_0037.html b/docs/dli/umn/dli_03_0037.html
index a02a1dcea..1362c76bf 100644
--- a/docs/dli/umn/dli_03_0037.html
+++ b/docs/dli/umn/dli_03_0037.html
@@ -1,18 +1,16 @@
 
 
 
                                                                                                          Flink Jobs
                                                                                                          
-
                                                                                                          
+
                                                                                                          
 
                                                                                                          
 
 
diff --git a/docs/dli/umn/dli_03_0040.html b/docs/dli/umn/dli_03_0040.html
index e4847d710..691b63673 100644
--- a/docs/dli/umn/dli_03_0040.html
+++ b/docs/dli/umn/dli_03_0040.html
@@ -1,13 +1,13 @@
 
 
 
                                                                                                          Why Does DIS Stream Not Exist During Job Semantic Check?
                                                                                                          
-
                                                                                                          To rectify this fault, perform the following steps:
                                                                                                          
-
                                                                                                          1. Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statements to check whether the DIS stream exists.
                                                                                                          2. If the DIS stream was not created, create a DIS stream by referring to "Creating a DIS Stream" in the Data Ingestion Service User Guide.
                                                                                                            Ensure that the created DIS stream and Flink job are in the same region.
                                                                                                            
-
                                                                                                          3. If a DIS stream has been created, check whether the DIS stream and the Flink job are in the same region.
                                                                                                          
+
                                                                                                          To rectify this fault, perform the following steps:
                                                                                                          
+
                                                                                                          1. Log in to the DIS management console. In the navigation pane, choose Stream Management. View the Flink job SQL statements to check whether the DIS stream exists.
                                                                                                          2. If the DIS stream was not created, create a DIS stream by referring to "Creating a DIS Stream" in the Data Ingestion Service User Guide.
                                                                                                            Ensure that the created DIS stream and Flink job are in the same region.
                                                                                                            
+
                                                                                                          3. If a DIS stream has been created, check whether the DIS stream and the Flink job are in the same region.
                                                                                                          
 
                                                                                                          
 
                                                                                                          
 
                                                                                                          
-
                                                                                                          Parent topic: O&M Guide
                                                                                                          
+
                                                                                                          Parent topic: Flink SQL Jobs
                                                                                                          
 
                                                                                                          
 
                                                                                                          
 
diff --git a/docs/dli/umn/dli_03_0043.html b/docs/dli/umn/dli_03_0043.html
deleted file mode 100644
index c1a4b5d61..000000000
--- a/docs/dli/umn/dli_03_0043.html
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-
                                                                                                          How Can I Check Flink Job Results?
                                                                                                          
-
                                                                                                          • DLI can output Flink job results to DIS. You can view the results in DIS. For details, see "Obtaining Data from DIS" in Data Ingestion Service User Guide.
                                                                                                          • DLI can output Flink job results to RDS. You can view the results in RDS. For details, see Relational Database Service Getting Started.
                                                                                                          • DLI can output Flink job results to SMN, and SMN sends the results to the user's terminal. For details, see Simple Message Notification Getting Started.
                                                                                                          • DLI can output Flink job results to Kafka. You can view the results in Kafka clusters. For details, visit the Kafka official website.
                                                                                                          • DLI can output Flink job results to CloudTable. You can view the results in CloudTable. For details, see CloudTable Service User Guide.
                                                                                                          • DLI can export Flink job results to CSS. You can view the results in CSS. For details, see "Getting Started" in Cloud Search Service User Guide.
                                                                                                          • DLI can export Flink job results to DCS. You can view the results in DCS. For details, see Distributed Cache Service User Guide.
                                                                                                          
-
                                                                                                          
-
                                                                                                          
-
                                                                                                          
-
                                                                                                          Parent topic: Usage
                                                                                                          
-
                                                                                                          
-
                                                                                                          
-
diff --git a/docs/dli/umn/dli_03_0044.html b/docs/dli/umn/dli_03_0044.html
index 7ae7fa0b1..13fce0fd4 100644
--- a/docs/dli/umn/dli_03_0044.html
+++ b/docs/dli/umn/dli_03_0044.html
@@ -1,11 +1,12 @@
 
 
 
                                                                                                          Can I Upload Configuration Files for Flink Jar Jobs?
                                                                                                          
-
                                                                                                          Uploading a Configuration File for a Flink Jar Job
                                                                                                          You can upload configuration files for custom jobs (Jar).
                                                                                                          
-
                                                                                                          1. Upload the configuration file to DLI through Package Management.
                                                                                                          2. In the Other Dependencies area of the Flink Jar job, select the created DLI package.
                                                                                                          3. Load the file through ClassName.class.getClassLoader().getResource("userData/fileName") in the code. In the file name, fileName indicates the name of the file to be accessed, and ClassName indicates the name of the class that needs to access the file.
                                                                                                          
+
                                                                                                          Uploading a Configuration File for a Flink Jar Job
                                                                                                          You can upload configuration files for custom jobs (Jar).
                                                                                                          
+
                                                                                                          1. Upload the configuration file to DLI through Package Management.
                                                                                                          2. In the Other Dependencies area of the Flink Jar job, select the created DLI package.
                                                                                                          3. Load the file through ClassName.class.getClassLoader().getResource("userData/fileName") in the code.
                                                                                                            • ClassName indicates the name of the class that needs to access the file.
                                                                                                            • userData is a fixed file path name, which cannot be changed or customized.
                                                                                                            • fileName indicates the name of the file to be accessed.
                                                                                                            
+
                                                                                                          
 
                                                                                                          
-
                                                                                                          Using a Configuration File
                                                                                                          • Solution 1: Load the file content to the memory in the main function and broadcast the content to each taskmanager. This method is applicable to the scenario where a small number of variables need to be loaded in advance.
                                                                                                          • Solution 2: Load the file when initializing the operator in open. A relative or absolute path can be used.
                                                                                                            Take Kafka sink as an example. Two files (userData/kafka-sink.conf and userData/client.truststore.jks) need to be loaded.
                                                                                                            
-
                                                                                                            • Example of using a relative path:
                                                                                                              Relative path: confPath = userData/kafka-sink.conf
+
                                                                                                              Using a Configuration File
                                                                                                              • Solution 1: Load the file content to the memory in the main function and broadcast the content to each taskmanager. This method is applicable to the scenario where a small number of variables need to be loaded in advance.
                                                                                                              • Solution 2: Load the file when initializing the operator in open. A relative or absolute path can be used.
                                                                                                                Take Kafka sink as an example. Two files (userData/kafka-sink.conf and userData/client.truststore.jks) need to be loaded.
                                                                                                                
+
                                                                                                                • Example of using a relative path:
                                                                                                                  Relative path: confPath = userData/kafka-sink.conf
 @Override
 public void open(Configuration parameters) throws Exception {
     super.open(parameters);
@@ -30,8 +31,8 @@ private void initConf() {
         e.printStackTrace();
     }
 }
                                                                                                                  
-
                                                                                                                  Figure 1 Example relative path configuration
                                                                                                                  
-
                                                                                                                • Example of using an absolute path:
                                                                                                                  Absolute path: confPath = userData/kafka-sink.conf / path = /opt/data1/hadoop/tmp/usercache/omm/appcache/application_xxx_0015/container_xxx_0015_01_000002/userData/client.truststore.jks
+
                                                                                                                  Figure 1 Example relative path configuration
                                                                                                                  
+
                                                                                                                • Example of using an absolute path:
                                                                                                                  Absolute path: confPath = userData/kafka-sink.conf / path = /opt/data1/hadoop/tmp/usercache/omm/appcache/application_xxx_0015/container_xxx_0015_01_000002/userData/client.truststore.jks
 @Override
 public void open(Configuration parameters) throws Exception {
     super.open(parameters);
@@ -59,7 +60,7 @@ private void initConf() {
         e.printStackTrace();
     }
 }
                                                                                                                  
-
                                                                                                                  Figure 2 Example absolute path configuration
                                                                                                                  
+
                                                                                                                  Figure 2 Example absolute path configuration
                                                                                                                  
 
                                                                                                                
 
                                                                                                              
 
                                                                                                              
diff --git a/docs/dli/umn/dli_03_0045.html b/docs/dli/umn/dli_03_0045.html
deleted file mode 100644
index 301889fa0..000000000
--- a/docs/dli/umn/dli_03_0045.html
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-
                                                                                                              Why Is the OBS Bucket Selected for Job Not Authorized?
                                                                                                              
-
                                                                                                              If the OBS bucket selected for a job is not authorized, perform the following steps:
                                                                                                              
-
                                                                                                              1. On the DLI management console, choose Global Configuration > Service Authorization and select the Tenant Administrator (Global service) permission.
                                                                                                              2. In the navigation pane, choose Job Management > Flink Jobs.
                                                                                                              3. In the row where the target job resides, click Edit under Operation to switch to the Edit page.
                                                                                                              4. Configure parameters under Running Parameters on the Edit page.
                                                                                                                1. Select Enable Checkpointing or Save Job Log.
                                                                                                                2. Specify OBS Bucket.
                                                                                                                3. Select Authorize OBS.
                                                                                                                
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              Parent topic: O&M Guide
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
diff --git a/docs/dli/umn/dli_03_0046.html b/docs/dli/umn/dli_03_0046.html
index 822d7ca9c..f50e51cb7 100644
--- a/docs/dli/umn/dli_03_0046.html
+++ b/docs/dli/umn/dli_03_0046.html
@@ -1,13 +1,14 @@
 
 
 
                                                                                                              Why Is a Schema Parsing Error Reported When I Create a Hive Table Using CTAS?
                                                                                                              
-
                                                                                                              Currently, DLI supports the Hive syntax for creating tables of the TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET file types. If the file format specified for creating a table in the CTAS is AVRO and digits are directly used as the input of the query statement (SELECT), for example, if the query is CREATE TABLE tb_avro STORED AS AVRO AS SELECT 1, a schema parsing exception is reported.
                                                                                                              
-
                                                                                                              If the column name is not specified, the content after SELECT is used as both the column name and inserted value. The column name of the AVRO table cannot be a digit. Otherwise, an error will be reported, indicating that the schema fails to be parsed.
                                                                                                              
-
                                                                                                              Solution: You can use CREATE TABLE tb_avro STORED AS AVRO AS SELECT 1 AS colName to specify the column name or set the storage format to a format other than AVRO.
                                                                                                              
+
                                                                                                              Currently, DLI supports the creation of TEXTFILE, SEQUENCEFILE, RCFILE, ORC, AVRO, and PARQUET tables using the Hive syntax.
                                                                                                              
+
                                                                                                              If you create a table using CTAS statements, specify the AVRO format, and then use a number as input for the SELECT query statement (for example, CREATE TABLE tb_avro STORED AS AVRO AS SELECT 1), a schema parsing exception will be reported.
                                                                                                              
+
                                                                                                              If the column name is not specified, the content after SELECT is used as both the column name and inserted value. The column name of the AVRO table cannot be a digit. Otherwise, an error will be reported, indicating that the schema fails to be parsed.
                                                                                                              
+
                                                                                                              Solution: You can use CREATE TABLE tb_avro STORED AS AVRO AS SELECT 1 AS colName to specify the column name or set the storage format to a format other than AVRO.
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: Job O&M Errors
                                                                                                              
+
                                                                                                              Parent topic: SQL Job O&M
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0047.html b/docs/dli/umn/dli_03_0047.html
index 8da071eb8..39076b898 100644
--- a/docs/dli/umn/dli_03_0047.html
+++ b/docs/dli/umn/dli_03_0047.html
@@ -1,18 +1,19 @@
 
 
-
                                                                                                              Why Is the Error Message "communication link failure" Displayed When I Use a Newly Activated Datasource Connection?
                                                                                                              
-
                                                                                                              • Possible Causes
                                                                                                                The network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.
                                                                                                                
-
                                                                                                              • Solution
                                                                                                                Example: When you create an RDS datasource connection, the system displays the error message Communication link failure.
                                                                                                                
-
                                                                                                                1. Delete the original datasource connection and create a new one. When you create a new connection, ensure that the selected Security Group, VPC, Subnet, and Destination Address are the same as those in RDS.
                                                                                                                   
                                                                                                                  Select a correct Service Type. In this example, select RDS.
                                                                                                                  
+
                                                                                                                  How Do I Resolve the "communication link failure" Error When Using a Newly Created Datasource Connection That Appears to Be Activated?
                                                                                                                  
+
                                                                                                                  Possible Causes
                                                                                                                  The network connectivity is abnormal. Check whether the security group is correctly selected and whether the VPC is correctly configured.
                                                                                                                  
+
                                                                                                                  
+
                                                                                                                  Solution
                                                                                                                  Example: When you create an RDS datasource connection, the system displays the error message Communication link failure.
                                                                                                                  
+
                                                                                                                  1. Delete the original datasource connection and create a new one. When you create a connection, ensure that the selected Security Group, VPC, Subnet, and Destination Address are the same as those in RDS.
                                                                                                                     
                                                                                                                    Select a correct Service Type. In this example, select RDS.
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    
-
                                                                                                                  2. Check the configurations of VPC.
                                                                                                                    If the error message is still displayed after you create a new datasource connection according to Step 1, check the VPC configuration.
                                                                                                                    
+
                                                                                                                    
+
                                                                                                                  3. Check the configurations of VPC.
                                                                                                                    If the error message is still displayed after you create a datasource connection according to Step 1, check the VPC configuration.
                                                                                                                    
 
                                                                                                                  
-
                                                                                                              
+
                                                                                                              
 
                                                                                                              
 
 
diff --git a/docs/dli/umn/dli_03_0048.html b/docs/dli/umn/dli_03_0048.html
index 91fcb8fa3..d2aa84d80 100644
--- a/docs/dli/umn/dli_03_0048.html
+++ b/docs/dli/umn/dli_03_0048.html
@@ -1,8 +1,11 @@
 
 
 
                                                                                                              How Do I Write Data to Different Elasticsearch Clusters in a Flink Job?
                                                                                                              
-
                                                                                                              Add the following SQL statements to the Flink job:
                                                                                                              
-
                                                                                                              create source stream ssource(xx);
+
                                                                                                              In a Flink job, you can use CREATE statements to define source and sink tables, and specify their connector types and related attributes.
                                                                                                              
+
                                                                                                              If you need to write data to different Elasticsearch clusters, you need to configure different connection parameters for each cluster and ensure that the Flink job can correctly route data to each cluster.
                                                                                                              
+
                                                                                                              In this example, the connector type and related attributes are defined for es1 and es2.
                                                                                                              
+
                                                                                                              Add the following SQL statements to the Flink job:
                                                                                                              
+
                                                                                                              create source stream ssource(xx);
 create sink stream es1(xx) with (xx);
 create sink stream es2(xx) with (xx);
 insert into es1 select * from ssource;
@@ -10,7 +13,7 @@ insert into es2 select * from ssource;
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: Performance Tuning
                                                                                                              
+
                                                                                                              Parent topic: Flink SQL Jobs
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0049.html b/docs/dli/umn/dli_03_0049.html
index 21acaedce..955408c9e 100644
--- a/docs/dli/umn/dli_03_0049.html
+++ b/docs/dli/umn/dli_03_0049.html
@@ -1,12 +1,20 @@
 
 
-
                                                                                                              Queue
                                                                                                              
-
                                                                                                              
+
                                                                                                              DLI Elastic Resource Pools and Queues
                                                                                                              
+
                                                                                                              
 
                                                                                                              
 
 
diff --git a/docs/dli/umn/dli_03_0053.html b/docs/dli/umn/dli_03_0053.html
index 86061b438..eb618b1ed 100644
--- a/docs/dli/umn/dli_03_0053.html
+++ b/docs/dli/umn/dli_03_0053.html
@@ -1,13 +1,13 @@
 
 
-
                                                                                                              Quota
                                                                                                              
-
                                                                                                              
+
                                                                                                              DLI Resource Quotas
                                                                                                              
+
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
 
diff --git a/docs/dli/umn/dli_03_0054.html b/docs/dli/umn/dli_03_0054.html
index 18bfccccb..a6bd6e987 100644
--- a/docs/dli/umn/dli_03_0054.html
+++ b/docs/dli/umn/dli_03_0054.html
@@ -1,12 +1,22 @@
 
 
-
                                                                                                              Permission
                                                                                                              
-
                                                                                                              
+
                                                                                                              DLI Permissions Management
                                                                                                              
+
                                                                                                              
 
                                                                                                              
 
 
diff --git a/docs/dli/umn/dli_03_0056.html b/docs/dli/umn/dli_03_0056.html
index 7117d5046..072be28b2 100644
--- a/docs/dli/umn/dli_03_0056.html
+++ b/docs/dli/umn/dli_03_0056.html
@@ -1,14 +1,15 @@
 
 
-
                                                                                                              APIs
                                                                                                              
-
                                                                                                              
+
+  
                                                                                                              DLI APIs
                                                                                                              
+
+  
                                                                                                              
 
                                                                                                              
+
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0057.html b/docs/dli/umn/dli_03_0057.html
index e9910c005..ee55eb6f1 100644
--- a/docs/dli/umn/dli_03_0057.html
+++ b/docs/dli/umn/dli_03_0057.html
@@ -1,13 +1,14 @@
 
 
-
                                                                                                              Will an Error Be Reported if the Inherited Permissions Are Regranted to a Table That Inherits Database Permissions?
                                                                                                              
-
                                                                                                              If a table inherits database permissions, you do not need to regrant the inherited permissions to the table.
                                                                                                              
-
                                                                                                              When you grant permissions on a table on the console:
                                                                                                              
-
                                                                                                              • If you set Authorization Object to User and select the permissions that are the same as the inherited permissions, the system displays a message indicating that the permissions already exist and do not need to be regranted.
                                                                                                              • If you set Authorization Object to Project and select the permissions that are the same as the inherited permissions, the system does not notify you of duplicate permissions.
                                                                                                              
+
                                                                                                              Will Granting Duplicate Permissions to a Table After Inheriting Database Permissions Cause an Error?
                                                                                                              
+
                                                                                                              If a table inherits database permissions, you do not need to regrant the inherited permissions to the table.
                                                                                                              
+
                                                                                                              Re-authorizing may cause confusion in table permission management, as the inherited permissions are already sufficient.
                                                                                                              
+
                                                                                                              When you grant permissions on a table on the console:
                                                                                                              
+
                                                                                                              • If you set Authorization Object to User and select the permissions that are the same as the inherited permissions, the system displays a message indicating that the permissions already exist and do not need to be regranted.
                                                                                                              • If you set Authorization Object to Project and select the permissions that are the same as the inherited permissions, the system does not notify you of duplicate permissions.
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: O&M Guide
                                                                                                              
+
                                                                                                              Parent topic: DLI Permissions Management
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0058.html b/docs/dli/umn/dli_03_0058.html
deleted file mode 100644
index fd316814b..000000000
--- a/docs/dli/umn/dli_03_0058.html
+++ /dev/null
@@ -1,17 +0,0 @@
-
-
-
                                                                                                              SDKs
                                                                                                              
-
                                                                                                              
-
-
diff --git a/docs/dli/umn/dli_03_0060.html b/docs/dli/umn/dli_03_0060.html
index 54b25d776..21f17b638 100644
--- a/docs/dli/umn/dli_03_0060.html
+++ b/docs/dli/umn/dli_03_0060.html
@@ -1,14 +1,14 @@
 
 
 
                                                                                                              Why Is Error "unsupported media Type" Reported When I Subimt a SQL Job?
                                                                                                              
-
                                                                                                              In the REST API provided by DLI, the request header can be added to the request URI, for example, Content-Type.
                                                                                                              
-
                                                                                                              Content-Type indicates the request body type or format. The default value is application/json.
                                                                                                              
-
                                                                                                              URI for submitting a SQL job: POST /v1.0/{project_id}/jobs/submit-job
                                                                                                              
-
                                                                                                              Content-Type can be only application/json. If Content-Type is set to text, "unsupported media Type" is displayed.
                                                                                                              
+
                                                                                                              In the REST API provided by DLI, the request header can be added to the request URI, for example, Content-Type.
                                                                                                              
+
                                                                                                              Content-Type indicates the request body type or format. The default value is application/json.
                                                                                                              
+
                                                                                                              URI for submitting a SQL job: POST /v1.0/{project_id}/jobs/submit-job
                                                                                                              
+
                                                                                                              Content-Type can be only application/json. If Content-Type is set to text, "unsupported media Type" is displayed.
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: APIs
                                                                                                              
+
                                                                                                              Parent topic: DLI APIs
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0061.html b/docs/dli/umn/dli_03_0061.html
deleted file mode 100644
index beab11147..000000000
--- a/docs/dli/umn/dli_03_0061.html
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-
                                                                                                              Does Data in the Temporary Stream of Flink SQL Need to Be Cleared Periodically? How Do I Clear the Data?
                                                                                                              
-
                                                                                                              The temp stream in Flink SQL is similar to a subquery. It is a logical stream used to simplify the SQL logic and does not generate data storage. Therefore, there is no need to clean data.
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              Parent topic: Flink SQL
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
diff --git a/docs/dli/umn/dli_03_0064.html b/docs/dli/umn/dli_03_0064.html
index a8a7300d7..59a07fe82 100644
--- a/docs/dli/umn/dli_03_0064.html
+++ b/docs/dli/umn/dli_03_0064.html
@@ -1,16 +1,16 @@
 
 
 
                                                                                                              Why Are Logs Not Written to the OBS Bucket After a DLI Flink Job Fails to Be Submitted for Running?
                                                                                                              
-
                                                                                                              Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:
                                                                                                              
-
                                                                                                              • If the submission fails, a submission log is generated only in the submit-client directory.
                                                                                                              • You can view the logs generated within 1 minute when the job fails to be executed on the management console.
                                                                                                                Choose Job Management > Flink Jobs, click the target job name to go to the job details page, and click Run Log to view real-time logs.
                                                                                                                
-
                                                                                                              • If the running fails and exceeds 1 minute (the log dump period is 1 minute), run logs are generated in the application_xx directory.
                                                                                                              
-
                                                                                                              Flink dependencies have been built in the DLI server and security hardening has been performed based on the open-source community version. To avoid dependency package compatibility issues or log output and dump issues, be careful to exclude the following files when packaging:
                                                                                                              • Built-in dependencies (or set the package dependency scope to "provided") 
                                                                                                              • Log configuration files (example, log4j.properties/logback.xml)
                                                                                                              • JAR file for log output implementation (example, log4j). 
                                                                                                              
+
                                                                                                              Mode for storing generated job logs when a DLI Flink job fails to be submitted or executed. The options are as follows:
                                                                                                              
+
                                                                                                              • If the submission fails, a submission log is generated only in the submit-client directory.
                                                                                                              • You can view the logs generated within 1 minute when the job fails to be executed on the management console.
                                                                                                                Choose Job Management > Flink Jobs, click the target job name to go to the job details page, and click Run Log to view real-time logs.
                                                                                                                
+
                                                                                                              • If the running fails and exceeds 1 minute (the log dump period is 1 minute), run logs are generated in the application_xx directory.
                                                                                                              
+
                                                                                                              Flink dependencies have been built in the DLI server and security hardening has been performed based on the open-source community version. To avoid dependency package compatibility issues or log output and dump issues, be careful to exclude the following files when packaging:
                                                                                                              • Built-in dependencies (or set the package dependency scope to "provided") 
                                                                                                              • Log configuration files (example, log4j.properties/logback.xml)
                                                                                                              • JAR file for log output implementation (example, log4j). 
                                                                                                              
 
                                                                                                              
-
                                                                                                              On this basis, the taskmanager.log file rolls as the log file size and time change.
                                                                                                              
+
                                                                                                              On this basis, the taskmanager.log file rolls as the log file size and time change.
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: O&M Guide
                                                                                                              
+
                                                                                                              Parent topic: Flink Job Performance Tuning
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0065.html b/docs/dli/umn/dli_03_0065.html
index b6b9ddc9b..795e2331f 100644
--- a/docs/dli/umn/dli_03_0065.html
+++ b/docs/dli/umn/dli_03_0065.html
@@ -1,13 +1,13 @@
 
 
-
                                                                                                              How Do I Switch an Earlier-Version Spark Queue to a General-Purpose Queue?
                                                                                                              
-
                                                                                                              Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earlier versions and are used to run Spark and Flink jobs.
                                                                                                              
-
                                                                                                              You can perform the following steps to convert an old Spark queue to a general purpose queue.
                                                                                                              
-
                                                                                                              1. Purchase a general purpose queue again.
                                                                                                              2. Migrate the jobs in the old Spark queue to the new general queue. That is, specify a new queue when submitting Spark jobs.
                                                                                                              3. Release the old Spark queue, that is, delete it or unsubscribe it from the queue.
                                                                                                              
+
                                                                                                              How Do I Migrate an Old Version Spark Queue to a General-Purpose Queue?
                                                                                                              
+
                                                                                                              Currently, DLI provides two types of queues, For SQL and For general use. SQL queues are used to run SQL jobs. General-use queues are compatible with Spark queues of earlier versions and are used to run Spark and Flink jobs.
                                                                                                              
+
                                                                                                              You can perform the following steps to convert an old Spark queue to a general purpose queue.
                                                                                                              
+
                                                                                                              1. Purchase a general purpose queue again.
                                                                                                              2. Migrate the jobs in the old Spark queue to the new general queue. That is, specify a new queue when submitting Spark jobs.
                                                                                                              3. Release the old Spark queue, that is, delete it or unsubscribe it from the queue.
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: O&M Guide
                                                                                                              
+
                                                                                                              Parent topic: DLI Elastic Resource Pools and Queues
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0066.html b/docs/dli/umn/dli_03_0066.html
index a7e842fb2..e1e4044d9 100644
--- a/docs/dli/umn/dli_03_0066.html
+++ b/docs/dli/umn/dli_03_0066.html
@@ -1,16 +1,16 @@
 
 
 
                                                                                                              Why Is Error "SQL_ANALYSIS_ERROR: Reference 't.id' is ambiguous, could be: t.id, t.id.;" Displayed When Two Tables Are Joined?
                                                                                                              
-
                                                                                                              This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.
                                                                                                              
-
                                                                                                              For example, tables tb1 and tb2 contain the id field.
                                                                                                              
-
                                                                                                              Incorrect command:
                                                                                                              select id from tb1 join tb2;
                                                                                                              
+
                                                                                                              This message indicates that the two tables to be joined contain the same column, but the owner of the column is not specified when the command is executed.
                                                                                                              
+
                                                                                                              For example, tables tb1 and tb2 contain the id field.
                                                                                                              
+
                                                                                                              Incorrect command:
                                                                                                              select id from tb1 join tb2;
                                                                                                              
 
                                                                                                              
-
                                                                                                              Correct command:
                                                                                                              
-
                                                                                                              select tb1.id from tb1 join tb2;
                                                                                                              
+
                                                                                                              Correct command:
                                                                                                              
+
                                                                                                              select tb1.id from tb1 join tb2;
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: Job O&M Errors
                                                                                                              
+
                                                                                                              Parent topic: SQL Job O&M
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0067.html b/docs/dli/umn/dli_03_0067.html
index 4d9f150a9..1f6194540 100644
--- a/docs/dli/umn/dli_03_0067.html
+++ b/docs/dli/umn/dli_03_0067.html
@@ -1,18 +1,18 @@
 
 
 
                                                                                                              Why Can't I Query a View After I'm Granted the Select Table Permission on the View?
                                                                                                              
-
                                                                                                              Symptom
                                                                                                              User A created Table1.
                                                                                                              
-
                                                                                                              User B created View1 based on Table1.
                                                                                                              
-
                                                                                                              After the Select Table permission on Table1 is granted to user C, user C fails to query View1.
                                                                                                              
+
                                                                                                              Symptom
                                                                                                              User A created Table1.
                                                                                                              
+
                                                                                                              User B created View1 based on Table1.
                                                                                                              
+
                                                                                                              After the Select Table permission on Table1 is granted to user C, user C fails to query View1.
                                                                                                              
 
                                                                                                              
-
                                                                                                              Possible Causes
                                                                                                              User B does not have the Select Table permission on Table1.
                                                                                                              
+
                                                                                                              Possible Causes
                                                                                                              User B does not have the Select Table permission on Table1.
                                                                                                              
 
                                                                                                              
-
                                                                                                              Solution
                                                                                                              Grant the Select Table permission on Table1 to user B. Then, query View1 as user C again.
                                                                                                              
+
                                                                                                              Solution
                                                                                                              Grant the Select Table permission on Table1 to user B. Then, query View1 as user C again.
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: O&M Guide
                                                                                                              
+
                                                                                                              Parent topic: DLI Permissions Management
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0068.html b/docs/dli/umn/dli_03_0068.html
index fe77615bd..4cf178493 100644
--- a/docs/dli/umn/dli_03_0068.html
+++ b/docs/dli/umn/dli_03_0068.html
@@ -1,17 +1,17 @@
 
 
 
                                                                                                              How Do I Use JDBC to Set the spark.sql.shuffle.partitions Parameter to Improve the Task Concurrency?
                                                                                                              
-
                                                                                                              Scenario
                                                                                                              When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.
                                                                                                              
-
                                                                                                              To solve this problem, you can configure spark.sql.shuffle.partitions to improve the concurrency of shuffle read tasks.
                                                                                                              
+
                                                                                                              Scenario
                                                                                                              When shuffle statements, such as GROUP BY and JOIN, are executed in Spark jobs, data skew occurs, which slows down the job execution.
                                                                                                              
+
                                                                                                              To solve this problem, you can configure spark.sql.shuffle.partitions to improve the concurrency of shuffle read tasks.
                                                                                                              
 
                                                                                                              
-
                                                                                                              Configuring spark.sql.shuffle.partitions
                                                                                                              You can use the set clause to configure the dli.sql.shuffle.partitions parameter in JDBC. The statement is as follows:
                                                                                                              
-
                                                                                                              Statement st = conn.stamte()
+
                                                                                                              Configuring spark.sql.shuffle.partitions
                                                                                                              You can use the set clause to configure the dli.sql.shuffle.partitions parameter in JDBC. The statement is as follows:
                                                                                                              
+
                                                                                                              Statement st = conn.stamte()
 st.execute("set spark.sql.shuffle.partitions=20")
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: Job Development
                                                                                                              
+
                                                                                                              Parent topic: Spark Job Development
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0069.html b/docs/dli/umn/dli_03_0069.html
index 7bcd07c6a..d1f7f4566 100644
--- a/docs/dli/umn/dli_03_0069.html
+++ b/docs/dli/umn/dli_03_0069.html
@@ -1,23 +1,23 @@
 
 
 
                                                                                                              How Can I Use the count Function to Perform Aggregation?
                                                                                                              
-
                                                                                                              The correct method for using the count function to perform aggregation is as follows:
                                                                                                              
-
                                                                                                              SELECT
+
                                                                                                              The correct method for using the count function to perform aggregation is as follows:
                                                                                                              
+
                                                                                                              SELECT
   http_method,
   count(http_method)
 FROM
   apigateway
 WHERE
   service_id = 'ecs' Group BY http_method
                                                                                                              
-
                                                                                                              Or
                                                                                                              
-
                                                                                                              SELECT
+
                                                                                                              Or
                                                                                                              
+
                                                                                                              SELECT
   http_method
 FROM
   apigateway
 WHERE
   service_id = 'ecs' DISTRIBUTE BY http_method
                                                                                                              
-
                                                                                                              If an incorrect method is used, an error will be reported.
                                                                                                              
-
                                                                                                              SELECT
+
                                                                                                              If an incorrect method is used, an error will be reported.
                                                                                                              
+
                                                                                                              SELECT
   http_method,
   count(http_method)
 FROM
@@ -27,7 +27,7 @@ WHERE
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: Job Development
                                                                                                              
+
                                                                                                              Parent topic: SQL Job Development
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0071.html b/docs/dli/umn/dli_03_0071.html
index 659ff90cd..21b8a5b1a 100644
--- a/docs/dli/umn/dli_03_0071.html
+++ b/docs/dli/umn/dli_03_0071.html
@@ -1,12 +1,15 @@
 
 
 
                                                                                                              Why Is Error "The current account does not have permission to perform this operation,the current account was restricted. Restricted for no budget." Reported when a SQL Statement Is Executed?
                                                                                                              
-
                                                                                                              Check if your account is in arrears and top it up if necessary.
                                                                                                              
-
                                                                                                              If the same error message persists after the top-up, log out of your account and log back in.
                                                                                                              
+
                                                                                                              This message indicates that your operation is restricted because your account is in arrears or there is insufficient balance in your account.
                                                                                                              
+
                                                                                                              Solution:
                                                                                                              
+
                                                                                                              1. Check the account status.
                                                                                                                Check if your account is in arrears and top it up if necessary.
                                                                                                                
+
                                                                                                              2. Relog in to the system.
                                                                                                                If the same error message persists after the top-up, log out of your account and log back in.
                                                                                                                
+
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: Job O&M Errors
                                                                                                              
+
                                                                                                              Parent topic: SQL Job O&M
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0072.html b/docs/dli/umn/dli_03_0072.html
index 754c35510..9e6b34531 100644
--- a/docs/dli/umn/dli_03_0072.html
+++ b/docs/dli/umn/dli_03_0072.html
@@ -1,12 +1,12 @@
 
 
-
                                                                                                              How Do I Synchronize DLI Table Data from One Region to Another?
                                                                                                              
-
                                                                                                              You can use the cross-region replication function of OBS. The procedure is as follows:
                                                                                                              
-
                                                                                                              1. Export the DLI table data in region 1 to the user-defined OBS bucket.
                                                                                                              2. Use the OBS cross-region replication function to replicate data to the OBS bucket in region 2.
                                                                                                              3. Import or use the corresponding data as required.
                                                                                                              
+
                                                                                                              How Do I Synchronize DLI Table Data Across Regions?
                                                                                                              
+
                                                                                                              You can use the cross-region replication function of OBS. The procedure is as follows:
                                                                                                              
+
                                                                                                              1. Export the DLI table data in region 1 to the user-defined OBS bucket.
                                                                                                              2. Use the OBS cross-region replication function to replicate data to the OBS bucket in region 2.
                                                                                                              3. Import or use the corresponding data as required.
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
                                                                                                              
-
                                                                                                              Parent topic: Job Development
                                                                                                              
+
                                                                                                              Parent topic: SQL Job Development
                                                                                                              
 
                                                                                                              
 
                                                                                                              
 
diff --git a/docs/dli/umn/dli_03_0073.html b/docs/dli/umn/dli_03_0073.html
deleted file mode 100644
index 188fbbade..000000000
--- a/docs/dli/umn/dli_03_0073.html
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
-
                                                                                                              How Do I Set the Timeout Duration for Querying SQL Job Results Using SDK?
                                                                                                              
-
                                                                                                              When you query the SQL job results using SDK, the system checks the job status when the job is submitted. The timeout interval set in the system is 300s. If the job is not in the FINISHED state, a timeout error is reported after 300s.
                                                                                                              
-
                                                                                                              You are advised to use getJobId() to obtain the job ID and then call queryJobResultInfo(String jobId) or cancelJob(String jobId) to obtain the result or cancel the job.
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              Parent topic: SDKs
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
diff --git a/docs/dli/umn/dli_03_0075.html b/docs/dli/umn/dli_03_0075.html
deleted file mode 100644
index e1b3d6570..000000000
--- a/docs/dli/umn/dli_03_0075.html
+++ /dev/null
@@ -1,61 +0,0 @@
-
-
-
                                                                                                              How Do I Dump Data to OBS and Create an OBS Partitioned Table?
                                                                                                              
-
                                                                                                              In this example, the day field is used as the partition field with the parquet encoding format (only the parquet format is supported currently) to dump car_info data to OBS. For more information, see "File System Sink Stream (Recommended)" in Data Lake Insight SQL Syntax Reference.
                                                                                                              
-
                                                                                                               1
- 2
- 3
- 4
- 5
- 6
- 7
- 8
- 9
-10
-11
-12
-13
                                                                                                              create sink stream car_infos (
-  carId string,
-  carOwner string,
-  average_speed double,
-  day string
-  ) partitioned by (day)
-  with (
-    type = "filesystem",
-    file.path = "obs://obs-sink/car_infos",
-    encode = "parquet",
-    ak = "{{myAk}}",
-    sk = "{{mySk}}"
-);
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              Structure of the data storage directory in OBS: obs://obs-sink/car_infos/day=xx/part-x-x.
                                                                                                              
-
                                                                                                              After the data is generated, the OBS partition table can be established for subsequent batch processing through the following SQL statements:
                                                                                                              
-
                                                                                                              1. Create an OBS partition table. 
                                                                                                                1
-2
-3
-4
-5
-6
-7
-8
                                                                                                                create table car_infos (
-  carId string,
-  carOwner string,
-  average_speed double
-)
-  partitioned by (day string)
-  stored as parquet
-  location 'obs://obs-sink/car-infos';
-
                                                                                                                
-
                                                                                                                
-
                                                                                                              2. Restore partition information from the associated OBS path.
                                                                                                                1
                                                                                                                alter table car_infos recover partitions;
-
                                                                                                                
-
                                                                                                                
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
                                                                                                              Parent topic: Flink SQL
                                                                                                              
-
                                                                                                              
-
                                                                                                              
-
diff --git a/docs/dli/umn/dli_03_0076.html b/docs/dli/umn/dli_03_0076.html
index be60c760b..eb276329e 100644
--- a/docs/dli/umn/dli_03_0076.html
+++ b/docs/dli/umn/dli_03_0076.html
@@ -1,15 +1,15 @@
 
 
 
                                                                                                              How Do I Use Python Scripts to Access the MySQL Database If the pymysql Module Is Missing from the Spark Job Results Stored in MySQL?
                                                                                                              
-
                                                                                                              1. If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management page. The procedure is as follows:
                                                                                                                1. Upload the egg package to the specified OBS path.
                                                                                                                2. Log in to the DLI management console and choose Data Management > Package Management.
                                                                                                                3. On the Package Management page, click Create Package in the upper right corner to create a package.
                                                                                                                4. In the Create Package dialog, set the following parameters:
                                                                                                                  • Type: Select PyFile.
                                                                                                                  • OBS Path: Select the OBS path where the egg package is stored.
                                                                                                                  • Set Group and Group Name as you need.
                                                                                                                  
-
                                                                                                                5. Click OK.
                                                                                                                6. On the Spark job editing page where the error is reported, choose the uploaded egg package from the Python File Dependencies drop-down list and run the Spark job again.
                                                                                                                
-
                                                                                                              2. To interconnect PySpark jobs with MySQL, you need to create a datasource connection to enable the network between DLI and RDS.
                                                                                                                For details about how to create a datasource connection on the management console, see  "Enhanced Datasource Connections" in Data Lake Insight User Guide.
                                                                                                                
-
                                                                                                                For details about how to call an API to create a datasource connection, see  "Creating an Enhanced Datasource Connection" in Data Lake Insight API Reference.
                                                                                                                
+
                                                                                                                1. If the pymysql module is missing, check whether the corresponding EGG package exists. If the package does not exist, upload the pyFile package on the Package Management page. The procedure is as follows:
                                                                                                                  1. Upload the egg package to the specified OBS path.
                                                                                                                  2. Log in to the DLI management console and choose Data Management > Package Management.
                                                                                                                  3. On the Package Management page, click Create Package in the upper right corner to create a package.
                                                                                                                  4. In the Create Package dialog, set the following parameters:
                                                                                                                    • Type: Select PyFile.
                                                                                                                    • OBS Path: Select the OBS path where the egg package is stored.
                                                                                                                    • Set Group and Group Name as you need.
                                                                                                                    
+
                                                                                                                  5. Click OK.
                                                                                                                  6. On the Spark job editing page where the error is reported, choose the uploaded egg package from the Python File Dependencies drop-down list and run the Spark job again.
                                                                                                                  
+
                                                                                                                2. To interconnect PySpark jobs with MySQL, you need to create a datasource connection to enable the network between DLI and RDS.
                                                                                                                  For how to create a datasource connection on the management console, see "Enhanced Datasource Connections" in Data Lake Insight User Guide.
                                                                                                                  
+
                                                                                                                  For how to call an API to create a datasource connection, see "Creating an Enhanced Datasource Connection" in Data Lake Insight API Reference.
                                                                                                                  
 
                                                                                                                
 
                                                                                                                
 
                                                                                                                
 
                                                                                                                
-
                                                                                                                Parent topic: Job Development
                                                                                                                
+
                                                                                                                Parent topic: Spark Job Development
                                                                                                                
 
                                                                                                                
 
                                                                                                                
 
diff --git a/docs/dli/umn/dli_03_0077.html b/docs/dli/umn/dli_03_0077.html
index a60751ac3..4a30d8f34 100644
--- a/docs/dli/umn/dli_03_0077.html
+++ b/docs/dli/umn/dli_03_0077.html
@@ -1,13 +1,13 @@
 
 
 
                                                                                                                Why Can't I Find the Specified Python Environment After Adding the Python Package?
                                                                                                                
-
                                                                                                                I cannot find the specified Python environment after adding the Python 3 package.
                                                                                                                
-
                                                                                                                Set spark.yarn.appMasterEnv.PYSPARK_PYTHON to python3 in the conf file to specify the Python 3 environment for the compute cluster.
                                                                                                                
-
                                                                                                                New clusters use the Python 3 environment by default.
                                                                                                                
+
                                                                                                                I cannot find the specified Python environment after adding the Python 3 package.
                                                                                                                
+
                                                                                                                Set spark.yarn.appMasterEnv.PYSPARK_PYTHON to python3 in the conf file to specify the Python 3 environment for the compute cluster.
                                                                                                                
+
                                                                                                                New clusters use the Python 3 environment by default.
                                                                                                                
 
                                                                                                                
 
                                                                                                                
 
                                                                                                                
-
                                                                                                                Parent topic: O&M Guide
                                                                                                                
+
                                                                                                                Parent topic: Spark Job Development
                                                                                                                
 
                                                                                                                
 
                                                                                                                
 
diff --git a/docs/dli/umn/dli_03_0080.html b/docs/dli/umn/dli_03_0080.html
index 0038f27fd..eae2a6318 100644
--- a/docs/dli/umn/dli_03_0080.html
+++ b/docs/dli/umn/dli_03_0080.html
@@ -1,13 +1,15 @@
 
 
-
                                                                                                                Connection Times Out During MRS HBase Datasource Connection, and No Error Is Recorded in Logs
                                                                                                                
-
                                                                                                                The cluster host information is not added to the datasource connection. As a result, the KRB authentication fails, the connection times out, and no error is recorded in logs. Configure the host information and try again.
                                                                                                                
-
                                                                                                                On the Enhanced page, select the connection and click Modify Host. In the dialog box that is displayed, enter the host information. The format is Host IP address Host name/Domain name. Multiple records are separated by line breaks.
                                                                                                                
-
                                                                                                                For details, see "Modifying the Host Information" in Data Lake Insight User Guide.
                                                                                                                
+
                                                                                                                How Do I Troubleshoot a Connection Timeout Issue That Isn't Recorded in Logs When Accessing MRS HBase Through a Datasource Connection?
                                                                                                                
+
                                                                                                                If you have not added the cluster host information to the datasource connection, it can lead to KRB authentication failure, resulting in a connection timeout. In this case, there will not be any error message in the logs.
                                                                                                                
+
                                                                                                                You are advised to reconfigure the host information and then try to access MRS HBase again.
                                                                                                                
+
                                                                                                                Log in to the DLI console and choose Datasource Connections > Enhanced. On the Enhanced tab, select a connection and click Modify Host. In the displayed dialog box, enter the host information.
                                                                                                                
+
                                                                                                                The information is in the format of Host IP address Host name/Domain name. Each piece of information is separated by a line break.
                                                                                                                
+
                                                                                                                For details, see "Modifying Host Information" in Data Lake Insight User Guide.
                                                                                                                
 
                                                                                                                
 
 
diff --git a/docs/dli/umn/dli_03_0082.html b/docs/dli/umn/dli_03_0082.html
index bfcb9467f..6d21cbece 100644
--- a/docs/dli/umn/dli_03_0082.html
+++ b/docs/dli/umn/dli_03_0082.html
@@ -1,16 +1,16 @@
 
 
 
                                                                                                                How Do I Run a Complex PySpark Program in DLI?
                                                                                                                
-
                                                                                                                DLI natively supports PySpark.
                                                                                                                
-
                                                                                                                For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are packed into JAR files and depend on third-party JAR files. Similarly, Python programs also depend on third-party libraries, especially big data analysis programs related to PySpark-based converged machine learning. Traditionally, Python libraries are installed directly on the execution machine using pip. However, for serverless services like DLI, users do not need to and are not aware of the underlying compute resources. How can we ensure that users can run their programs more effectively?
                                                                                                                
-
                                                                                                                DLI has built-in algorithm libraries for machine learning in its compute resources. These common algorithm libraries meet the requirements of most users. What if a user's PySpark program depends on a program library that is not provided by the built-in algorithm library? Actually, the dependency of PySpark is specified based on PyFiles. On the DLI Spark job page, you can directly select the Python third-party program library (such as ZIP and EGG) stored on OBS.
                                                                                                                
-
                                                                                                                The compressed package of the dependent third-party Python library has structure requirements. For example, if the PySpark program depends on moduleA (import moduleA), the compressed package must meet the following structure requirement:
                                                                                                                
-
                                                                                                                Figure 1 Compressed package structure requirement
                                                                                                                
-
                                                                                                                That is, the compressed package contains a folder named after a module name, and then the Python file of the corresponding class. Generally, the downloaded Python library may not meet this requirement. Therefore, you need to compress the Python library again. In addition, there is no requirement on the name of the compressed package. Therefore, it is recommended that you compress the packages of multiple modules into a compressed package. Now, a large and complex PySpark program is configured and runs normally.
                                                                                                                
+
                                                                                                                DLI natively supports PySpark.
                                                                                                                
+
                                                                                                                For most cases, Python is preferred for data analysis, and PySpark is the best choice for big data analysis. Generally, JVM programs are packed into JAR files and depend on third-party JAR files. Similarly, Python programs also depend on third-party libraries, especially big data analysis programs related to PySpark-based converged machine learning. Traditionally, Python libraries are installed directly on the execution machine using pip. However, for serverless services like DLI, users do not need to and are not aware of the underlying compute resources. How can we ensure that users can run their programs more effectively?
                                                                                                                
+
                                                                                                                DLI has built-in algorithm libraries for machine learning in its compute resources. These common algorithm libraries meet the requirements of most users. What if a user's PySpark program depends on a program library that is not provided by the built-in algorithm library? Actually, the dependency of PySpark is specified based on PyFiles. On the DLI Spark job page, you can directly select the Python third-party program library (such as ZIP and EGG) stored on OBS.
                                                                                                                
+
                                                                                                                The compressed package of the dependent third-party Python library has structure requirements. For example, if the PySpark program depends on moduleA (import moduleA), the compressed package must meet the following structure requirement:
                                                                                                                
+
                                                                                                                Figure 1 Compressed package structure requirement
                                                                                                                
+
                                                                                                                That is, the compressed package contains a folder named after a module name, and then the Python file of the corresponding class. Generally, the downloaded Python library may not meet this requirement. Therefore, you need to compress the Python library again. In addition, there is no requirement on the name of the compressed package. Therefore, it is recommended that you compress the packages of multiple modules into a compressed package. Now, a large and complex PySpark program is configured and runs normally.
                                                                                                                
 
                                                                                                                
 
                                                                                                                
 
                                                                                                                
-
                                                                                                                Parent topic: Job Development
                                                                                                                
+
                                                                                                                Parent topic: Spark Job Development
                                                                                                                
 
                                                                                                                
 
                                                                                                                
 
diff --git a/docs/dli/umn/dli_03_0083.html b/docs/dli/umn/dli_03_0083.html
deleted file mode 100644
index ec783ec28..000000000
--- a/docs/dli/umn/dli_03_0083.html
+++ /dev/null
@@ -1,14 +0,0 @@
-
-
-
                                                                                                                What Data Formats and Data Sources Are Supported by DLI Flink Jobs?
                                                                                                                
-
                                                                                                                DLI Flink jobs support the following data formats:
                                                                                                                
-
                                                                                                                Avro, Avro_merge, BLOB, CSV, EMAIL, JSON, ORC, Parquet, and XML.
                                                                                                                
-
                                                                                                                DLI Flink jobs support data from the following data sources:
                                                                                                                
-
                                                                                                                CloudTable HBase, CloudTable OpenTSDB, CSS Elasticsearch, DCS, DDS, DIS, DMS, GaussDB(DWS), EdgeHub, MRS HBase, MRS Kafka, open-source Kafka, file systems, OBS, RDS, and SMN
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
                                                                                                                Parent topic: Usage
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
diff --git a/docs/dli/umn/dli_03_0085.html b/docs/dli/umn/dli_03_0085.html
deleted file mode 100644
index 92ede8cc8..000000000
--- a/docs/dli/umn/dli_03_0085.html
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-
                                                                                                                How Can I Access Data Across Regions?
                                                                                                                
-
                                                                                                                1. Connect VPCs in different regions.
                                                                                                                2. Create an enhanced datasource connection on DLI and bind it to a queue. 
                                                                                                                3. Add a DLI route. 
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
                                                                                                                Parent topic: Cross-Source Analysis
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
diff --git a/docs/dli/umn/dli_03_0086.html b/docs/dli/umn/dli_03_0086.html
index f67a1c924..c840f01f5 100644
--- a/docs/dli/umn/dli_03_0086.html
+++ b/docs/dli/umn/dli_03_0086.html
@@ -1,15 +1,16 @@
 
 
 
                                                                                                                How Do I Merge Small Files?
                                                                                                                
-
                                                                                                                If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.
                                                                                                                
-
                                                                                                                1. Set the configuration item as follows:
                                                                                                                  spark.sql.shuffle.partitions = Number of partitions (number of the generated small files in this case)
                                                                                                                  
-
                                                                                                                2. Execute the following SQL statements:
                                                                                                                  INSERT OVERWRITE TABLE tablename
-select  * FROM  tablename distribute by rand()
                                                                                                                  
-
                                                                                                                
+
                                                                                                                If a large number of small files are generated during SQL execution, job execution and table query will take a long time. In this case, you should merge small files.
                                                                                                                
+
                                                                                                                You are advised to use temporary tables for data transfer. There is a risk of data loss in self-read and self-write operations during unexpected exceptional scenarios.
                                                                                                                
+
                                                                                                                Run the following SQL statements:
                                                                                                                INSERT OVERWRITE TABLE tablename
+select  * FROM  tablename
+DISTRIBUTE BY floor(rand()*20)
                                                                                                                
+
                                                                                                                
 
                                                                                                                
 
                                                                                                                
 
                                                                                                                
-
                                                                                                                Parent topic: Job Development
                                                                                                                
+
                                                                                                                Parent topic: SQL Job Development
                                                                                                                
 
                                                                                                                
 
                                                                                                                
 
diff --git a/docs/dli/umn/dli_03_0087.html b/docs/dli/umn/dli_03_0087.html
deleted file mode 100644
index c429a6a19..000000000
--- a/docs/dli/umn/dli_03_0087.html
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
-
                                                                                                                How Do I Set Local Variables in SQL Statements?
                                                                                                                
-
                                                                                                                You can use the where condition statement in the select statement to filter data. For example:
                                                                                                                
-
                                                                                                                select * from table where part = '202012'
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
                                                                                                                Parent topic: Job Development
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
diff --git a/docs/dli/umn/dli_03_0088.html b/docs/dli/umn/dli_03_0088.html
deleted file mode 100644
index 26d22ac44..000000000
--- a/docs/dli/umn/dli_03_0088.html
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
-
                                                                                                                How Do I Allocate Queue Resources for Running Spark Jobs If I Have Purchased 64 CUs?
                                                                                                                
-
                                                                                                                In DLI, 64 CU = 64 cores and 256 GB memory.
                                                                                                                
-
                                                                                                                In a Spark job, if the driver occupies 4 cores and 16 GB memory, the executor can occupy 60 cores and 240 GB memory.
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
                                                                                                                Parent topic: O&M Guide
                                                                                                                
-
                                                                                                                
-
                                                                                                                
-
diff --git a/docs/dli/umn/dli_03_0089.html b/docs/dli/umn/dli_03_0089.html
index 1165b642b..d52ea11c2 100644
--- a/docs/dli/umn/dli_03_0089.html
+++ b/docs/dli/umn/dli_03_0089.html
@@ -1,10 +1,10 @@
 
 
-
                                                                                                                How Do I Create an OBS Partitioned Table for a Flink SQL Job?
                                                                                                                
-
                                                                                                                Scenario
                                                                                                                When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.
                                                                                                                
+
                                                                                                                How Do I Map an OBS Table to a DLI Partitioned Table?
                                                                                                                
+
                                                                                                                Scenario
                                                                                                                When using a Flink SQL job, you need to create an OBS partition table for subsequent batch processing.
                                                                                                                
 
                                                                                                                
-
                                                                                                                Procedure
                                                                                                                In the following example, the day field is used as the partition field with the parquet encoding format to dump car_info data to OBS. 
                                                                                                                
-
                                                                                                                 1
+
                                                                                                                Procedure
                                                                                                                In the following example, the day field is used as the partition field with the parquet encoding format to dump car_info data to OBS. 
                                                                                                                
+
                                                                                                                 1
  2
  3
  4
@@ -31,9 +31,9 @@
 );
 
                                                                                                                
 
                                                                                                                
-
                                                                                                                Structure of the data storage directory in OBS: obs://obs-sink/car_infos/day=xx/part-x-x.
                                                                                                                
-
                                                                                                                After the data is generated, the OBS partition table can be established for subsequent batch processing through the following SQL statements:
                                                                                                                
-
                                                                                                                1. Create an OBS partitioned table. 
                                                                                                                  1
+
                                                                                                                  Structure of the data storage directory in OBS: obs://obs-sink/car_infos/day=xx/part-x-x.
                                                                                                                  
+
                                                                                                                  After the data is generated, the OBS partition table can be established for subsequent batch processing through the following SQL statements:
                                                                                                                  
+
                                                                                                                  1. Create an OBS partitioned table. 
                                                                                                                    1
 2
 3
 4
@@ -50,7 +50,7 @@
   location 'obs://obs-sink/car-infos';
 
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                  2. Restore partition information from the associated OBS path.
                                                                                                                    1
                                                                                                                    alter table car_infos recover partitions;
+
                                                                                                                  3. Restore partition information from the associated OBS path.
                                                                                                                    1
                                                                                                                    alter table car_infos recover partitions;
 
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    4. 
@@ -58,7 +58,7 @@
 
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    Parent topic: Flink SQL
                                                                                                                    
+
                                                                                                                    Parent topic: Flink SQL Jobs
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
diff --git a/docs/dli/umn/dli_03_0090.html b/docs/dli/umn/dli_03_0090.html
index ab4041007..9d777892d 100644
--- a/docs/dli/umn/dli_03_0090.html
+++ b/docs/dli/umn/dli_03_0090.html
@@ -1,14 +1,14 @@
 
 
-
                                                                                                                    How Do I Set Auto Restart upon Exception for a Flink Job?
                                                                                                                    
-
                                                                                                                    Scenario
                                                                                                                    DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are rectified.
                                                                                                                    
+
                                                                                                                    How Do I Configure Auto Restart upon Exception for a Flink Job?
                                                                                                                    
+
                                                                                                                    Scenario
                                                                                                                    DLI Flink jobs are highly available. You can enable the automatic restart function to automatically restart your jobs after short-time faults of peripheral services are rectified.
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    Procedure
                                                                                                                    1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                                                                                                                    2. Click a job name and select Auto-restart upon exception on the job editing page.
                                                                                                                    
+
                                                                                                                    Procedure
                                                                                                                    1. Log in to the DLI console. In the navigation pane, choose Job Management > Flink Jobs.
                                                                                                                    2. Click a job name and select Auto-restart upon exception on the job editing page.
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    Parent topic: Usage
                                                                                                                    
+
                                                                                                                    Parent topic: Flink Job Consulting
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
diff --git a/docs/dli/umn/dli_03_0091.html b/docs/dli/umn/dli_03_0091.html
index 3eaa10671..436a87a9e 100644
--- a/docs/dli/umn/dli_03_0091.html
+++ b/docs/dli/umn/dli_03_0091.html
@@ -1,15 +1,15 @@
 
 
 
                                                                                                                    How Do I View DLI SQL Logs?
                                                                                                                    
-
                                                                                                                    Scenario
                                                                                                                    You can view SQL job logs for routine O&M.
                                                                                                                    
+
                                                                                                                    Scenario
                                                                                                                    You can view SQL job logs for routine O&M.
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    Procedure
                                                                                                                    1. Obtain the ID of the DLI job executed on the DataArts Studio console.
                                                                                                                      Figure 1 Job ID
                                                                                                                      
-
                                                                                                                    2. On the DLI console, choose Job Management > SQL Jobs.
                                                                                                                    3. On the SQL Jobs page, enter the job ID.
                                                                                                                    4. In the Operation column of the target job, choose More > View Log and download the logs to your PC.
                                                                                                                    5. Search for job ID in the downloaded logs to view the execution logs.
                                                                                                                    
+
                                                                                                                    Procedure
                                                                                                                    1. Obtain the ID of the DLI job executed on the DataArts Studio console.
                                                                                                                      Figure 1 Job ID
                                                                                                                      
+
                                                                                                                    2. On the DLI console, choose Job Management > SQL Jobs.
                                                                                                                    3. On the SQL Jobs page, enter the job ID.
                                                                                                                    4. In the Operation column of the target job, choose More > View Log and download the logs to your PC.
                                                                                                                    5. Search for job ID in the downloaded logs to view the execution logs.
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    Parent topic: O&M Guide
                                                                                                                    
+
                                                                                                                    Parent topic: SQL Job Development
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
diff --git a/docs/dli/umn/dli_03_0092.html b/docs/dli/umn/dli_03_0092.html
index 702a907f3..ffc23e2b9 100644
--- a/docs/dli/umn/dli_03_0092.html
+++ b/docs/dli/umn/dli_03_0092.html
@@ -1,20 +1,20 @@
 
 
 
                                                                                                                    How Do I Specify an OBS Path When Creating an OBS Table?
                                                                                                                    
-
                                                                                                                    Scenario
                                                                                                                    When creating an OBS table, you must specify a table path in the database. The path format is as follows: obs://xxx/database name/table name.
                                                                                                                    
+
                                                                                                                    Scenario
                                                                                                                    When creating an OBS table, you must specify a table path in the database. The path format is as follows: obs://xxx/database name/table name.
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    Correct Example
                                                                                                                    CREATE TABLE `di_seller_task_activity_30d` (`user_id` STRING COMMENT' user ID...) SORTED as parquet
+
                                                                                                                    Correct Example
                                                                                                                    CREATE TABLE `di_seller_task_activity_30d` (`user_id` STRING COMMENT' user ID...) SORTED as parquet
 LOCATION 'obs://akc-bigdata/akdc.db/di_seller_task_activity_30d'
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    Incorrect Example
                                                                                                                    CREATE TABLE `di_seller_task_activity_30d` (`user_id` STRING COMMENT' user ID...) SORTED as parquet
+
                                                                                                                    Analysis of a Typical Error Example
                                                                                                                    CREATE TABLE `di_seller_task_activity_30d` (`user_id` STRING COMMENT' user ID...) SORTED as parquet
 LOCATION 'obs://akc-bigdata/akdc.db'
                                                                                                                    
-
                                                                                                                     
                                                                                                                    If the specified path is akdc.db, data in this path will be cleared when the insert overwrite statement is executed.
                                                                                                                    
+
                                                                                                                     
                                                                                                                    If the specified path is akdc.db, data in this path will be cleared when the insert overwrite statement is executed.
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    Parent topic: Job Development
                                                                                                                    
+
                                                                                                                    Parent topic: SQL Job Development
                                                                                                                    
 
                                                                                                                    
 
                                                                                                                    
 
diff --git a/docs/dli/umn/dli_03_0093.html b/docs/dli/umn/dli_03_0093.html
index 41bd0d9fa..149faa7a8 100644
--- a/docs/dli/umn/dli_03_0093.html
+++ b/docs/dli/umn/dli_03_0093.html
@@ -1,18 +1,18 @@
 
 
 
                                                                                                                    How Do I Do When Data Skew Occurs During the Execution of a SQL Job?
                                                                                                                    
-
                                                                                                                    What Is Data Skew?
                                                                                                                    Data skew is a common issue during the execution of SQL jobs. When data is unevenly distributed, some compute nodes process significantly more data than others, which can impact the efficiency of the entire computation process.
                                                                                                                    
-
                                                                                                                    For example, if you notice that a SQL query is taking a long time to execute, you can check its status in SparkUI. See Figure 1. If you see a stage that has been running for over 20 minutes with only one task remaining, it is likely due to data skew.
                                                                                                                    
-
                                                                                                                    Figure 1 Data skew example
                                                                                                                    
+
                                                                                                                    What Is Data Skew?
                                                                                                                    Data skew is a common issue during the execution of SQL jobs. When data is unevenly distributed, some compute nodes process significantly more data than others, which can impact the efficiency of the entire computation process.
                                                                                                                    
+
                                                                                                                    For example, if you notice that a SQL query is taking a long time to execute, you can check its status in SparkUI. See Figure 1. If you see a stage that has been running for over 20 minutes with only one task remaining, it is likely due to data skew.
                                                                                                                    
+
                                                                                                                    Figure 1 Data skew example
                                                                                                                    
 
                                                                                                                    
-
                                                                                                                    Common Data Skew Scenarios