Create an OSS import job
Import data from Object Storage Service (OSS) into Simple Log Service (SLS) by calling the CreateOSSIngestion operation with the SLS Python SDK.
Prerequisites
-
Simple Log Service is activated.
-
Python is installed (see Install Python), a Python development environment is set up (see PyCharm), and a Python package management tool is installed (see pip).
-
The new version of Simple Log Service SDK for Python supports Python 3.7 and later.
-
Run the
python-Vcommand to check the installed Python version. -
Run the
pip3 -Vcommand to check the installed pip version.
-
-
The new version of Simple Log Service SDK for Python is installed.
-
Run the following command to install the new version of Simple Log Service SDK for Python:
pip install alibabacloud_sls20201230 -
After the installation is complete, run the following command to verify it:
pip show alibabacloud_sls20201230
-
-
The ALIBABA_CLOUD_ACCESS_KEY_ID and ALIBABA_CLOUD_ACCESS_KEY_SECRET environment variables are configured. For more information, see Configure environment variables on Linux, macOS, and Windows.
Parameters
def create_ossingestion(
self,
project: str,
request: sls_20201230_models.CreateOSSIngestionRequest,
) -> sls_20201230_models.CreateOSSIngestionResponse:
Request parameters
|
Parameter |
Type |
Required |
Description |
Example |
|
project |
String |
Yes |
The name of the SLS project. |
ali-test-project |
|
request |
object |
Yes |
The OSS import job configuration. |
- |
Request parameters:
class CreateOSSIngestionRequest(TeaModel):
def __init__(
self,
configuration: OSSIngestionConfiguration = None,
description: str = None,
display_name: str = None,
name: str = None,
schedule: Schedule = None,
):
|
Parameter |
Type |
Required |
Description |
Example |
|
configuration |
object |
Yes |
The OSS import configuration. |
- |
|
description |
String |
No |
The description of the import job. |
OSS import job. |
|
display_name |
String |
Yes |
The display name of the import job. |
OSS import test. |
|
name |
String |
Yes |
The unique name of the OSS import job. |
ingest-oss-123456 |
|
schedule |
object |
No |
The scheduling configuration. Leave this parameter empty in most cases. |
- |
Schedule parameters
class Schedule(TeaModel):
def __init__(
self,
cron_expression: str = None,
delay: int = None,
interval: str = None,
run_immediately: bool = None,
time_zone: str = None,
type: str = None,
):
|
Parameter |
Type |
Required |
Description |
Example |
|
type |
String |
Yes |
The schedule type. Valid values: |
FixedRate |
|
cron_expression |
String |
No |
Required only when |
0/5 * * * * |
|
run_immediately |
bool |
No |
Whether to run the job immediately. |
True |
|
time_zone |
String |
No |
The time zone of the cron expression. Defaults to UTC+8 when empty. |
+0800 |
|
delay |
int |
No |
The delay before a scheduled job is run. Unit: seconds (s). |
4 |
|
interval |
String |
No |
The execution interval when |
1m |
Configuration parameters
class OSSIngestionConfiguration(TeaModel):
def __init__(
self,
logstore: str = None,
source: OSSIngestionConfigurationSource = None,
):
|
Parameter |
Type |
Required |
Description |
Example |
|
logstore |
String |
Yes |
The name of the destination Logstore. |
test-logstore |
|
source |
object |
Yes |
The OSS data source configuration. |
- |
Source parameters:
class OSSIngestionConfigurationSource(TeaModel):
def __init__(
self,
bucket: str = None,
compression_codec: str = None,
encoding: str = None,
end_time: int = None,
endpoint: str = None,
format: Dict[str, Any] = None,
interval: str = None,
pattern: str = None,
prefix: str = None,
restore_object_enabled: bool = None,
role_arn: str = None,
start_time: int = None,
time_field: str = None,
time_format: str = None,
time_pattern: str = None,
time_zone: str = None,
use_meta_index: bool = None,
):
|
Parameter |
Type |
Required |
Description |
Example |
|
bucket |
String |
Yes |
The name of the OSS bucket. |
ossbucket |
|
compression_code |
String |
Yes |
The compression type of the source data. Set to |
none |
|
encoding |
String |
Yes |
The character encoding of the source data. |
UTF-8 |
|
endpoint |
String |
Yes |
The endpoint of OSS. Use an internal endpoint for cross-service access within the same region. |
oss-cn-hangzhou-internal.aliyuncs.com |
|
format |
Dictionary |
The data format of the OSS object.
|
{"type": "Line"} |
|
|
interval |
String |
Yes |
The interval for checking new files in the OSS bucket. |
60s |
|
pattern |
String |
No |
The regular expression that is used to filter file paths. |
.* |
|
prefix |
String |
No |
The prefix that is used to filter file paths. |
prefix |
|
restore_object_enabled |
bool |
No |
Whether to import archived objects. |
true |
|
role_arn |
String |
No |
The Alibaba Cloud Resource Name (ARN) of the RAM role. For more information, see View the ARN of a RAM role. |
acs:ram::12345:role/aliyunlogdefaultrole |
|
start_time |
int |
No |
A Unix timestamp in seconds. Only files modified after this time are imported. |
1714274081 |
|
time_field |
String |
No |
The time field to be extracted. |
__time__ |
|
time_format |
String |
No |
The format of the time field. |
yyyy-MM-dd HH:mm:ss |
|
time_pattern |
String |
No |
The regular expression that is used to extract the time. |
[0-9]{0,2}\/[0-9a-zA-Z]+\/[0-9:,]+ |
|
time_zone |
String |
No |
Time field partition. |
GMT+08:00 |
|
use_meta_index |
bool |
Yes |
Whether to use the OSS metadata index for faster file discovery. |
False |
Response parameters
For more information about the response parameters, see Create an OSS import job.
Sample code
import os
from alibabacloud_sls20201230.client import Client as Sls20201230Client
from alibabacloud_tea_openapi import models as open_api_models
from alibabacloud_sls20201230 import models as sls_20201230_models
from alibabacloud_tea_util.client import Client as UtilClient
def main():
config = open_api_models.Config(
# Required. Ensure the ALIBABA_CLOUD_ACCESS_KEY_ID environment variable is set.
access_key_id=os.environ['ALIBABA_CLOUD_ACCESS_KEY_ID'],
# Required. Ensure the ALIBABA_CLOUD_ACCESS_KEY_SECRET environment variable is set.
access_key_secret=os.environ['ALIBABA_CLOUD_ACCESS_KEY_SECRET']
)
# The endpoint of Simple Log Service.
config.endpoint = "cn-hangzhou.log.aliyuncs.com"
project_name = "ali-test-project"
client = Sls20201230Client(config)
# Create an instance of OSSIngestionConfigurationSource.
source = sls_20201230_models.OSSIngestionConfigurationSource(
bucket="oss-test",
compression_codec="none",
encoding="UTF-8",
end_time=None,
endpoint="oss-cn-hangzhou-internal.aliyuncs.com",
format={"type": "Line"},
interval="30s",
use_meta_index=False,
)
# Create an instance of OSSIngestionConfiguration.
oss_config = sls_20201230_models.OSSIngestionConfiguration(
logstore="test-logstore",
source=source
)
# Create an instance of CreateOSSIngestionRequest.
create_ossingestions_request = sls_20201230_models.CreateOSSIngestionRequest(
configuration=oss_config,
description="Create an OSS import job",
display_name="OSS import test",
name="ingest-oss-123456"
)
try:
create_ossingestions_response = client.create_ossingestion(project_name, create_ossingestions_request)
print(create_ossingestions_response)
except Exception as error:
print(error.message)
print(error.data.get("Recommend"))
UtilClient.assert_as_string(error.message)
if __name__ == '__main__':
main()
References
-
Related API operations for OSS data import: