package main
import (
"context"
"flag"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss"
"github.com/aliyun/alibabacloud-oss-go-sdk-v2/oss/credentials"
)
// 定义全局变量
var (
region string // 存储区域
bucketName string // 存储空间名称
prefix string // 对象前缀（文件夹路径）
localDir string // 本地下载目录
maxWorkers int // 最大并发数
maxKeys int // 每次列举的最大对象数
)
// DownloadTask 下载任务结构
type DownloadTask struct {
ObjectKey string
LocalPath string
Size int64
}
// DownloadResult 下载结果结构
type DownloadResult struct {
ObjectKey string
Success bool
Error error
Size int64
}
// init函数用于初始化命令行参数
func init() {
flag.StringVar(®ion, "region", "", "The region in which the bucket is located.")
flag.StringVar(&bucketName, "bucket", "", "The name of the bucket.")
flag.StringVar(&prefix, "prefix", "", "The prefix (folder path) to download.")
flag.StringVar(&localDir, "local-dir", "./downloads", "Local directory to save downloaded files.")
flag.IntVar(&maxWorkers, "workers", 5, "Maximum number of concurrent downloads.")
flag.IntVar(&maxKeys, "max-keys", 1000, "Maximum number of objects to list at once.")
}
func main() {
// 解析命令行参数
flag.Parse()
// 检查必要参数
if len(bucketName) == 0 {
flag.PrintDefaults()
log.Fatalf("invalid parameters, bucket name required")
}
if len(region) == 0 {
flag.PrintDefaults()
log.Fatalf("invalid parameters, region required")
}
// 确保前缀以/结尾（如果不是空字符串）
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
// 创建本地下载目录
if err := os.MkdirAll(localDir, 0755); err != nil {
log.Fatalf("failed to create local directory: %v", err)
}
// 加载默认配置并设置凭证提供者和区域
cfg := oss.LoadDefaultConfig().
WithCredentialsProvider(credentials.NewEnvironmentVariableCredentialsProvider()).
WithRegion(region)
// 创建OSS客户端
client := oss.NewClient(cfg)
fmt.Printf("开始批量下载，存储空间: %s, 前缀: %s, 本地目录: %s\n", bucketName, prefix, localDir)
// 列举所有需要下载的对象
tasks, err := listObjects(client, bucketName, prefix)
if err != nil {
log.Fatalf("failed to list objects: %v", err)
}
if len(tasks) == 0 {
fmt.Println("没有找到需要下载的文件")
return
}
fmt.Printf("找到 %d 个文件需要下载\n", len(tasks))
// 执行批量下载
results := batchDownload(client, tasks, maxWorkers)
// 统计下载结果
var successCount, failCount int
var totalSize int64
for _, result := range results {
if result.Success {
successCount++
totalSize += result.Size
} else {
failCount++
fmt.Printf("下载失败: %s, 错误: %v\n", result.ObjectKey, result.Error)
}
}
fmt.Printf("\n下载完成! 成功: %d, 失败: %d, 总大小: %s\n",
successCount, failCount, formatBytes(totalSize))
}
// listObjects 列举存储空间中指定前缀的所有对象
func listObjects(client *oss.Client, bucketName, prefix string) ([]DownloadTask, error) {
var tasks []DownloadTask
var continuationToken *string
for {
// 创建列举对象请求
request := &oss.ListObjectsV2Request{
Bucket: oss.Ptr(bucketName),
Prefix: oss.Ptr(prefix),
MaxKeys: int32(maxKeys),
ContinuationToken: continuationToken,
}
// 执行列举操作
result, err := client.ListObjectsV2(context.TODO(), request)
if err != nil {
return nil, fmt.Errorf("failed to list objects: %w", err)
}
// 处理列举结果
for _, obj := range result.Contents {
// 跳过文件夹对象（以/结尾且大小为0）
if strings.HasSuffix(*obj.Key, "/") && obj.Size == 0 {
continue
}
// 计算本地文件路径
relativePath := strings.TrimPrefix(*obj.Key, prefix)
localPath := filepath.Join(localDir, relativePath)
tasks = append(tasks, DownloadTask{
ObjectKey: *obj.Key,
LocalPath: localPath,
Size: obj.Size,
})
}
// 检查是否还有更多对象
if result.NextContinuationToken == nil {
break
}
continuationToken = result.NextContinuationToken
}
return tasks, nil
}
// batchDownload 执行批量下载
func batchDownload(client *oss.Client, tasks []DownloadTask, maxWorkers int) []DownloadResult {
taskChan := make(chan DownloadTask, len(tasks))
resultChan := make(chan DownloadResult, len(tasks))
// 启动工作协程
var wg sync.WaitGroup
for i := 0; i < maxWorkers; i++ {
wg.Add(1)
go downloadWorker(client, bucketName, taskChan, resultChan, &wg)
}
// 发送下载任务
go func() {
for _, task := range tasks {
taskChan <- task
}
close(taskChan)
}()
// 等待所有工作协程完成
go func() {
wg.Wait()
close(resultChan)
}()
// 收集结果并显示进度
var results []DownloadResult
completed := 0
total := len(tasks)
for result := range resultChan {
results = append(results, result)
completed++
if result.Success {
fmt.Printf("✓ [%d/%d] %s (%s)\n",
completed, total, result.ObjectKey, formatBytes(result.Size))
} else {
fmt.Printf("✗ [%d/%d] %s - 错误: %v\n",
completed, total, result.ObjectKey, result.Error)
}
}
return results
}
// downloadWorker 下载工作协程
func downloadWorker(client *oss.Client, bucketName string, taskChan <-chan DownloadTask,
resultChan chan<- DownloadResult, wg *sync.WaitGroup) {
defer wg.Done()
for task := range taskChan {
result := DownloadResult{
ObjectKey: task.ObjectKey,
Size: task.Size,
}
// 创建本地文件目录
if err := os.MkdirAll(filepath.Dir(task.LocalPath), 0755); err != nil {
result.Error = fmt.Errorf("failed to create directory: %w", err)
resultChan <- result
continue
}
// 检查文件是否已存在且大小一致
if fileInfo, err := os.Stat(task.LocalPath); err == nil {
if fileInfo.Size() == task.Size {
result.Success = true
resultChan <- result
continue
}
}
// 创建下载请求
getRequest := &oss.GetObjectRequest{
Bucket: oss.Ptr(bucketName),
Key: oss.Ptr(task.ObjectKey),
}
// 执行下载
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
_, err := client.GetObjectToFile(ctx, getRequest, task.LocalPath)
cancel()
if err != nil {
result.Error = err
} else {
result.Success = true
}
resultChan <- result
}
}
// formatBytes 格式化字节数为可读格式
func formatBytes(bytes int64) string {
const unit = 1024
if bytes < unit {
return fmt.Sprintf("%d B", bytes)
}
div, exp := int64(unit), 0
for n := bytes / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp])
}
使用方法
# 编译测试代码为可执行文件
go build -o oss-batch-download main.go
# 下载指定文件夹
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix images/2024/
# 自定义本地目录和并发数
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix documents/ -local-dir ./downloads -workers 10
# 下载整个存储空间
./oss-batch-download -region oss-cn-hangzhou -bucket my-bucket -prefix ""
输出示例
程序运行时会显示详细的下载进度：
开始批量下载，存储空间: my-bucket, 前缀: images/2024/, 本地目录: ./downloads
找到 150 个文件需要下载
✓ [1/150] images/2024/photo1.jpg (2.3 MB)
✓ [2/150] images/2024/photo2.png (1.8 MB)
...
下载完成! 成功: 148, 失败: 2, 总大小: 1.2 GB