前段时间使用阿里云官网提供的OSS C SDK上传和下载数据，想在上传和下载过程中对数据进行一些简单的自定义预处理，看了一下oss c sdk的具体实现，大致了解如何通过自定义上传和下载的callback达到上述目的，这里做一个简单的分享。
OSS C SDK在上传和下载数据时使用了CURL进行通信，之前简单学习过CURL的一些知识，知道CURL提供了一系列Callback，在上传下载时对数据进行一些处理，大家感兴趣的话可以参考： http://curl.haxx.se/libcurl/c/curl_easy_setopt.html 。OSS C SDK针对OSS的常用场景，对用户暴露了两个最基本的Callback：CURLOPT_WRITEFUNCTION 和 CURLOPT_READFUNCTION。结合OSS C SDK使用这两个Callback实现上传数据和下载数据，可以大致了解整个过程。
1、上传数据
OSS C SDK上传数据提供了两种方式：从memory读取、从文件读取。以从文件读取数据为例，我们看一下如何利用Callback实现从文件读取数据上传到OSS的。
首先，我们熟悉oss c sdk中使用的一些数据结构：
struct aos_http_request_s {
char *host;
uint8_t port;
char *signed_url;
http_method_e method;
char *uri;
char *resource;
aos_table_t *headers;
aos_table_t *query_params;
aos_list_t body;
int64_t body_len;
char *file_path;
aos_file_buf_t *file_buf;
aos_pool_t *pool;
void *user_data;
aos_read_http_body_pt read_body;
aos_http_body_type_e type;
};
typedef enum {
BODY_IN_MEMORY = 0,
BODY_IN_FILE,
BODY_IN_CALLBACK
} aos_http_body_type_e;
typedef int (*aos_read_http_body_pt)(aos_http_request_t *req, char *buffer, int len);
OSS C SDK提供了oss_put_object_from_file接口用于从文件读取数据上传到OSS，重点看一下oss_write_request_body_from_file的实现：
int oss_write_request_body_from_file(aos_pool_t *p, const aos_string_t *filename, aos_http_request_t *req)
{
int res = AOSE_OK;
aos_file_buf_t *fb = aos_create_file_buf(p);
res = aos_open_file_for_all_read(p, filename->data, fb);
if (res != AOSE_OK) {
aos_error_log("Open read file fail, filename:%s\n", filename->data);
return res;
}
req->body_len = fb->file_last;
req->file_path = filename->data;
req->file_buf = fb;
req->type = BODY_IN_FILE;
req->read_body = aos_read_http_body_file;
return res;
}
和上面的aos_http_request_s数据结构联系起来，可以看到在这里设置读取文件的相关信息的。接下来看一下这些设置如何和CURL提供的Callback联系上。
aos_http_transport_t *aos_curl_http_transport_create(aos_pool_t *p)
{
aos_func_u func;
aos_curl_http_transport_t *t;
t = (aos_curl_http_transport_t *)aos_pcalloc(p, sizeof(aos_curl_http_transport_t));
t->pool = p;
t->options = aos_default_http_transport_options;
t->cleanup = aos_fstack_create(p, 5);
func.func1 = (aos_func1_pt)aos_transport_cleanup;
aos_fstack_push(t->cleanup, t, func, 1);
t->curl = aos_request_get();
func.func1 = (aos_func1_pt)request_release;
aos_fstack_push(t->cleanup, t->curl, func, 1);
t->header_callback = aos_curl_default_header_callback;
t->read_callback = aos_curl_default_read_callback;
t->write_callback = aos_curl_default_write_callback;
return (aos_http_transport_t *)t;
}
size_t aos_curl_default_read_callback(char *buffer, size_t size, size_t nitems, void *instream)
{
int len;
int bytes;
aos_curl_http_transport_t *t;
t = (aos_curl_http_transport_t *)(instream);
len = size * nitems;
if (t->controller->error_code != AOSE_OK) {
aos_debug_log("abort read callback.");
return CURL_READFUNC_ABORT;
}
if ((bytes = t->req->read_body(t->req, buffer, len)) < 0) {
aos_debug_log("read body failure, %d.", bytes);
t->controller->error_code = AOSE_READ_BODY_ERROR;
t->controller->reason = "read body failure.";
return CURL_READFUNC_ABORT;
}
aos_move_transport_state(t, TRANS_STATE_BODY_OUT);
return bytes;
}
int aos_curl_transport_setup(aos_curl_http_transport_t *t)
{
...
curl_easy_setopt_safe(CURLOPT_READFUNCTION, t->read_callback);
...
}
红色部分基本已经说明了OSS C SDK如何调用CURL提供的CURLOPT_READFUNCTION了。参考OSS C SDK上传数据的实现大概也就知道如何对上传数据做一些简单的自定义了。
2、下载数据
下载数据常见的方式有两种：下载数据到memory、下载数据到文件。以下载数据到文件中为例，OSS C SDK使用CURL提供的Callback方式实现了数据下载。首先，熟悉一下OSS C SDK中使用的一些数据结构：
struct aos_http_response_s {
int status;
aos_table_t *headers;
aos_list_t body;
int64_t body_len;
char *file_path;
aos_file_buf_t* file_buf;
int64_t content_length;
aos_pool_t *pool;
void *user_data;
aos_write_http_body_pt write_body;
aos_http_body_type_e type;
};
typedef enum {
BODY_IN_MEMORY = 0,
BODY_IN_FILE,
BODY_IN_CALLBACK
} aos_http_body_type_e;
typedef int (*aos_write_http_body_pt)(aos_http_response_t *resp, const char *buffer, int len);
OSS C SDK提供了oss_get_object_to_file接口用于将OSS上的数据下载到本地文件中。重点看一下这个函数中oss_init_read_response_body_to_file函数的实现：
int oss_init_read_response_body_to_file(aos_pool_t *p, const aos_string_t *filename, aos_http_response_t *resp)
{
int res = AOSE_OK;
aos_file_buf_t *fb = aos_create_file_buf(p);
res = aos_open_file_for_write(p, filename->data, fb);
if (res != AOSE_OK) {
aos_error_log("Open write file fail, filename:%s\n", filename->data);
return res;
}
resp->file_path = filename->data;
resp->file_buf = fb;
resp->write_body = aos_write_http_body_file;
resp->type = BODY_IN_FILE;
return res;
}
这部分和上面的aos_http_response_s数据结构联系起来，可以设置将下载数据写入本地文件。接下来看一下这些设置如何和CURL提供的Callback联系上。
aos_http_transport_t *aos_curl_http_transport_create(aos_pool_t *p) {
aos_func_u func;
aos_curl_http_transport_t *t;
t = (aos_curl_http_transport_t *)aos_pcalloc(p, sizeof(aos_curl_http_transport_t));
t->pool = p;
t->options = aos_default_http_transport_options;
t->cleanup = aos_fstack_create(p, 5);
func.func1 = (aos_func1_pt)aos_transport_cleanup;
aos_fstack_push(t->cleanup, t, func, 1);
t->curl = aos_request_get();
func.func1 = (aos_func1_pt)request_release;
aos_fstack_push(t->cleanup, t->curl, func, 1);
t->header_callback = aos_curl_default_header_callback;
t->read_callback = aos_curl_default_read_callback;
t->write_callback = aos_curl_default_write_callback;
return (aos_http_transport_t *)t;
}
size_t aos_curl_default_write_callback(char *ptr, size_t size, size_t nmemb, void *userdata){
int len; int bytes;
aos_curl_http_transport_t *t;
t = (aos_curl_http_transport_t *)(userdata);
len = size * nmemb;
if (t->controller->first_byte_time == 0) {
t->controller->first_byte_time = apr_time_now();
}
aos_curl_transport_headers_done(t);
if (t->controller->error_code != AOSE_OK) {
aos_debug_log("write callback abort");
return 0;
}
if (t->resp->status < 200 || t->resp->status > 299) {
bytes = aos_write_http_body_memory(t->resp, ptr, len);
assert(bytes == len);
aos_move_transport_state(t, TRANS_STATE_BODY_IN);
return bytes;
}
if (t->resp->type == BODY_IN_MEMORY && t->resp->body_len >= (int64_t)t->controller->options->max_memory_size) {
t->controller->reason = apr_psprintf(t->pool,
"receive body too big, current body size: %" APR_INT64_T_FMT ", max memory size: %" APR_INT64_T_FMT, t->resp->body_len, t->controller->options->max_memory_size);
t->controller->error_code = AOSE_OVER_MEMORY;
aos_error_log("error reason:%s, ", t->controller->reason);
return 0;
}
if ((bytes = t->resp->write_body(t->resp, ptr, len)) < 0) {
aos_debug_log("write body failure, %d.", bytes);
t->controller->error_code = AOSE_WRITE_BODY_ERROR;
t->controller->reason = "write body failure.";
return 0;
}
aos_move_transport_state(t, TRANS_STATE_BODY_IN);
return bytes;
}
int aos_curl_transport_setup(aos_curl_http_transport_t *t) {
...
curl_easy_setopt_safe(CURLOPT_WRITEFUNCTION, t->write_callback);
...
}
红色部分基本说明了oss c sdk如何调用CURL提供的CURLOPT_WRITEFUNCTION了。
参考OSS C SDK使用CURL进行数据上传和下载的实现，能够根据自己的需求自定义数据上传和下载的Callback，对数据进行一些预处理。