ClickHouse data source

Configure a single-table batch synchronization task

• For the procedure, see Codeless UI configuration and Script mode configuration.

• For all script mode parameters and sample scripts, see Appendix: Sample scripts and parameters.

Configure a full-database batch read synchronization task

For the procedure, see Batch database synchronization configuration.

Configure a batch synchronization task by using the code editor

If you want to configure a batch synchronization task by using the code editor, you must configure the related parameters in the script based on the unified script format requirements. For more information, see Script mode configuration. The following information describes the parameters that you must configure for data sources when you configure a batch synchronization task by using the code editor.

Reader sample script

{
    "type": "job",
    "version": "2.0",
    "steps": [
        {
            "stepType": "clickhouse", //Plug-in name.
            "parameter": {
                "fetchSize":1024,//Number of records fetched in each batch by the plug-in and database server.
                "datasource": "example",
                "column": [   //Column names.
                    "id",
                    "name"
                ],
                "where": "",    //Filter condition.
                "splitPk": "",  //Split key.
                "table": ""    //Table name.
            },
            "name": "Reader",
            "category": "reader"
        },
        {
            "stepType": "clickhouse",
            "parameter": {
                "postSql": [
                    "update @table set db_modify_time = now() where db_id = 1"
                ],
                "datasource": "example",    //Data source.
                "batchByteSize": "67108864",
                "column": [
                    "id",
                    "name"
                ],
                "writeMode": "insert",
                "encoding": "UTF-8",
                "batchSize": 1024,
                "table": "ClickHouse_table",
                "preSql": [
                    "delete from @table where db_id = -1"
                ]
            },
            "name": "Writer",
            "category": "writer"
        }
    ],
    "setting": {
        "executeMode": null,
        "errorLimit": {
            "record": "0"  //Maximum number of error records during synchronization.
        },
        "speed": {
         "throttle":true,//When throttle is false, mbps is ignored and throughput is unlimited; when true, throughput is limited.
            "concurrent":1 //Job concurrency.
            "mbps":"12",//Rate limit. Here, 1 mbps = 1 MB/s.
        }
    },
    "order": {
        "hops": [
            {
                "from": "Reader",
                "to": "Writer"
            }
        ]
    }
}

Reader script parameters

Writer sample script

{
    "type":"job",
    "version":"2.0",//Version number.
    "steps":[
        {
            "stepType":"stream",
            "parameter":{},
            "name":"Reader",
            "category":"reader"
        },
        {
            "stepType":"clickhouse",//Plug-in name.
            "parameter":{
                "username": "",
                "password": "",
                "column": [//Fields.
                    "id",
                    "name"
                ],
                "connection": [
                    {
                        "table": [//Table name.
                            "ClickHouse_table"
                        ],
                        "jdbcUrl": "jdbc:clickhouse://ip:port/database"
                    }
                ],
                "preSql": [ //SQL statements to run before the synchronization task.
                    "TRUNCATETABLEIFEXISTStablename"
                ],
                "postSql": [//SQL statements to run after the synchronization task.
                    "ALTERTABLEtablenameUPDATEcol1=1WHEREcol2=2"
                ],
                "batchSize": "1024",
                "batchByteSize": "67108864",
                "writeMode": "insert"
            },
            "name":"Writer",
            "category":"writer"
        }
    ],
    "setting":{
        "errorLimit":{
            "record":"0"//Number of error records.
        },
        "speed":{
            "throttle":true,//When throttle is false, mbps is ignored and throughput is unlimited; when true, throughput is limited.
            "concurrent":1, //Job concurrency.
            "mbps":"12"//Rate limit. Here, 1 mbps = 1 MB/s.
        }
    },
    "order":{
        "hops":[
            {
                "from":"Reader",
                "to":"Writer"
            }
        ]
    }
}

Writer script parameters