« back to all changes in this revision
Viewing changes to storage.go
- browse files at revision 240
- compare with another revision
- download diff
- download tarball
- view history from revision 240
- Committer: Ian Booth
- Date: 2014-12-02 00:36:45 UTC
- Revision ID: ian.booth@canonical.com-20141202003645-ye8a5akifuf2wjk3
Ensure all regions have costs and remove custom formatting
- files removed:
- utilities
- files modified:
- Makefile
added
removed
storage.go
7
7
// file storage API.
8
8
9
9
import (
10
"bytes"
11
"encoding/base64"
12
"fmt"
13
"io"
14
. "launchpad.net/gwacl/logging"
15
"strconv"
16
"strings"
10
"bytes"
11
"encoding/base64"
12
"fmt"
13
"io"
14
. "launchpad.net/gwacl/logging"
15
"strconv"
16
"strings"
17
17
)
18
18
19
19
// UploadBlockBlob uses PutBlock and PutBlockList API operations to upload
20
20
// arbitrarily large files, 1MB at a time.
21
21
func (context *StorageContext) UploadBlockBlob(
22
container, filename string, data io.Reader) error {
23
24
buffer := make([]byte, 1024*1024) // 1MB buffer
25
blockList := &BlockList{}
26
27
// Upload the file in chunks.
28
for blockNum := int64(0); ; blockNum++ {
29
blockSize, err := data.Read(buffer)
30
if err == io.EOF {
31
break
32
}
33
if err != nil {
34
return err
35
}
36
block := bytes.NewReader(buffer[:blockSize])
37
blockID := strconv.FormatInt(blockNum, 36)
38
// Block IDs must be a consistent length, so pad it out.
39
blockID = fmt.Sprintf("%030s", blockID)
40
Debugf("Uploading block %d (size=%d, id=%s).\n",
41
blockNum, blockSize, blockID)
42
err = context.PutBlock(container, filename, blockID, block)
43
if err != nil {
44
return err
45
}
46
blockList.Add(BlockListLatest, blockID)
47
}
48
49
// Commit those blocks by writing the block list.
50
Debugf("Committing %d blocks.\n", len(blockList.Items))
51
return context.PutBlockList(container, filename, blockList)
22
container, filename string, data io.Reader) error {
23
24
buffer := make([]byte, 1024*1024) // 1MB buffer
25
blockList := &BlockList{}
26
27
// Upload the file in chunks.
28
for blockNum := int64(0); ; blockNum++ {
29
blockSize, err := data.Read(buffer)
30
if err == io.EOF {
31
break
32
}
33
if err != nil {
34
return err
35
}
36
block := bytes.NewReader(buffer[:blockSize])
37
blockID := strconv.FormatInt(blockNum, 36)
38
// Block IDs must be a consistent length, so pad it out.
39
blockID = fmt.Sprintf("%030s", blockID)
40
Debugf("Uploading block %d (size=%d, id=%s).\n",
41
blockNum, blockSize, blockID)
42
err = context.PutBlock(container, filename, blockID, block)
43
if err != nil {
44
return err
45
}
46
blockList.Add(BlockListLatest, blockID)
47
}
48
49
// Commit those blocks by writing the block list.
50
Debugf("Committing %d blocks.\n", len(blockList.Items))
51
return context.PutBlockList(container, filename, blockList)
52
52
}
53
53
54
54
// ListAllBlobs requests from the API a list of blobs in a container.
55
55
func (context *StorageContext) ListAllBlobs(request *ListBlobsRequest) (*BlobEnumerationResults, error) {
56
blobs := make([]Blob, 0)
57
var batch *BlobEnumerationResults
58
59
// Request the initial result, using the empty marker. Then, for as long
60
// as the result has a nonempty NextMarker, request the next batch using
61
// that marker.
62
// This loop is very similar to the one in ListAllContainers().
63
for marker, nextMarker := "", "x"; nextMarker != ""; marker = nextMarker {
64
var err error
65
// Don't use := here or you'll shadow variables from the function's
66
// outer scopes.
67
request.Marker = marker
68
batch, err = context.ListBlobs(request)
69
if err != nil {
70
return nil, err
71
}
72
// The response may contain a NextMarker field, to let us request a
73
// subsequent batch of results. The XML parser won't trim whitespace out
74
// of the marker tag, so we do that here.
75
nextMarker = strings.TrimSpace(batch.NextMarker)
76
blobs = append(blobs, batch.Blobs...)
77
}
78
79
// There's more in a BlobsEnumerationResults than just the blobs.
80
// Return the latest batch, but give it the full cumulative blobs list
81
// instead of just the last batch.
82
// To the caller, this will look like they made one call to Azure's
83
// List Blobs method, but batch size was unlimited.
84
batch.Blobs = blobs
85
return batch, nil
56
blobs := make([]Blob, 0)
57
var batch *BlobEnumerationResults
58
59
// Request the initial result, using the empty marker. Then, for as long
60
// as the result has a nonempty NextMarker, request the next batch using
61
// that marker.
62
// This loop is very similar to the one in ListAllContainers().
63
for marker, nextMarker := "", "x"; nextMarker != ""; marker = nextMarker {
64
var err error
65
// Don't use := here or you'll shadow variables from the function's
66
// outer scopes.
67
request.Marker = marker
68
batch, err = context.ListBlobs(request)
69
if err != nil {
70
return nil, err
71
}
72
// The response may contain a NextMarker field, to let us request a
73
// subsequent batch of results. The XML parser won't trim whitespace out
74
// of the marker tag, so we do that here.
75
nextMarker = strings.TrimSpace(batch.NextMarker)
76
blobs = append(blobs, batch.Blobs...)
77
}
78
79
// There's more in a BlobsEnumerationResults than just the blobs.
80
// Return the latest batch, but give it the full cumulative blobs list
81
// instead of just the last batch.
82
// To the caller, this will look like they made one call to Azure's
83
// List Blobs method, but batch size was unlimited.
84
batch.Blobs = blobs
85
return batch, nil
86
86
}
87
87
88
88
type DeleteAllBlobsRequest struct {
89
Container string
90
// Other params possible, add later.
89
Container string
90
// Other params possible, add later.
91
91
}
92
92
93
93
// RemoveAllBlobs requests a deletion of all the blobs in a container.
94
94
// The blobs are not deleted immediately, so when this call returns they
95
95
// may still be present for a while.
96
96
func (context *StorageContext) DeleteAllBlobs(request *DeleteAllBlobsRequest) error {
97
blobs, err := context.ListAllBlobs(&ListBlobsRequest{
98
Container: request.Container})
99
if err != nil {
100
return err
101
}
102
103
for _, blob := range blobs.Blobs {
104
err := context.DeleteBlob(request.Container, blob.Name)
105
if err != nil {
106
return err
107
}
108
}
109
110
return nil
97
blobs, err := context.ListAllBlobs(&ListBlobsRequest{
98
Container: request.Container})
99
if err != nil {
100
return err
101
}
102
103
for _, blob := range blobs.Blobs {
104
err := context.DeleteBlob(request.Container, blob.Name)
105
if err != nil {
106
return err
107
}
108
}
109
110
return nil
111
111
}
112
112
113
113
// ListAllContainers requests from the storage service a list of containers
114
114
// in the storage account.
115
115
func (context *StorageContext) ListAllContainers() (*ContainerEnumerationResults, error) {
116
containers := make([]Container, 0)
117
var batch *ContainerEnumerationResults
118
119
// Request the initial result, using the empty marker. Then, for as long
120
// as the result has a nonempty NextMarker, request the next batch using
121
// that marker.
122
for marker, nextMarker := "", "x"; nextMarker != ""; marker = nextMarker {
123
var err error
124
// Don't use := here or you'll shadow variables from the function's
125
// outer scopes.
126
request := &ListContainersRequest{Marker: marker}
127
batch, err = context.ListContainers(request)
128
if err != nil {
129
return nil, err
130
}
131
// The response may contain a NextMarker field, to let us request a
132
// subsequent batch of results. The XML parser won't trim whitespace out
133
// of the marker tag, so we do that here.
134
nextMarker = strings.TrimSpace(batch.NextMarker)
135
containers = append(containers, batch.Containers...)
136
}
137
138
// There's more in a ContainerEnumerationResults than just the containers.
139
// Return the latest batch, but give it the full cumulative containers list
140
// instead of just the last batch.
141
// To the caller, this will look like they made one call to Azure's
142
// List Containers method, but batch size was unlimited.
143
batch.Containers = containers
144
return batch, nil
116
containers := make([]Container, 0)
117
var batch *ContainerEnumerationResults
118
119
// Request the initial result, using the empty marker. Then, for as long
120
// as the result has a nonempty NextMarker, request the next batch using
121
// that marker.
122
for marker, nextMarker := "", "x"; nextMarker != ""; marker = nextMarker {
123
var err error
124
// Don't use := here or you'll shadow variables from the function's
125
// outer scopes.
126
request := &ListContainersRequest{Marker: marker}
127
batch, err = context.ListContainers(request)
128
if err != nil {
129
return nil, err
130
}
131
// The response may contain a NextMarker field, to let us request a
132
// subsequent batch of results. The XML parser won't trim whitespace out
133
// of the marker tag, so we do that here.
134
nextMarker = strings.TrimSpace(batch.NextMarker)
135
containers = append(containers, batch.Containers...)
136
}
137
138
// There's more in a ContainerEnumerationResults than just the containers.
139
// Return the latest batch, but give it the full cumulative containers list
140
// instead of just the last batch.
141
// To the caller, this will look like they made one call to Azure's
142
// List Containers method, but batch size was unlimited.
143
batch.Containers = containers
144
return batch, nil
145
145
}
146
146
147
147
type CreateVHDRequest struct {
148
Container string // Container name in the storage account
149
Filename string // Specify the filename in which to store the VHD
150
FilesystemData io.Reader // A formatted filesystem, e.g. iso9660.
151
Size int // How many bytes from the Filesystem data to upload. *Must* be a multiple of 512.
148
Container string // Container name in the storage account
149
Filename string // Specify the filename in which to store the VHD
150
FilesystemData io.Reader // A formatted filesystem, e.g. iso9660.
151
Size int // How many bytes from the Filesystem data to upload. *Must* be a multiple of 512.
152
152
}
153
153
154
154
// CreateInstanceDataVHD will take the supplied filesystem data and create an
157
157
// arbitrary data to a new instance - create a disk here and then attach it to
158
158
// the new instance.
159
159
func (context *StorageContext) CreateInstanceDataVHD(req *CreateVHDRequest) error {
160
// We need several steps:
161
// 1. Create an empty page blob of exactly VHD_SIZE bytes (see
162
// vhd_footer.go)
163
// 2. Upload VHD_FOOTER to the last page of the blob.
164
// 3. Upload the supplied FilesystemData from the start of the blob.
165
166
var err error
167
168
if req.Size%512 != 0 {
169
return fmt.Errorf("Size must be a multiple of 512")
170
}
171
if req.Size > VHD_SIZE-512 {
172
// Protect against writing over the VHD footer.
173
return fmt.Errorf("Size cannot be bigger than %d", VHD_SIZE-512)
174
}
175
176
// Step 1.
177
err = context.PutBlob(&PutBlobRequest{
178
Container: req.Container,
179
BlobType: "page",
180
Filename: req.Filename,
181
Size: VHD_SIZE,
182
})
183
184
if err != nil {
185
return err
186
}
187
188
// Step 2.
189
data, err := base64.StdEncoding.DecodeString(VHD_FOOTER)
190
if err != nil {
191
// This really shouldn't ever happen since there's a test to make sure
192
// it can be decoded.
193
panic(err)
194
}
195
dataReader := bytes.NewReader(data)
196
err = context.PutPage(&PutPageRequest{
197
Container: req.Container,
198
Filename: req.Filename,
199
StartRange: VHD_SIZE - 512, // last page of the blob
200
EndRange: VHD_SIZE - 1,
201
Data: dataReader,
202
})
203
204
if err != nil {
205
return err
206
}
207
208
// Step 3.
209
err = context.PutPage(&PutPageRequest{
210
Container: req.Container,
211
Filename: req.Filename,
212
StartRange: 0,
213
EndRange: req.Size - 1,
214
Data: req.FilesystemData,
215
})
216
217
if err != nil {
218
return err
219
}
220
221
return nil
160
// We need several steps:
161
// 1. Create an empty page blob of exactly VHD_SIZE bytes (see
162
// vhd_footer.go)
163
// 2. Upload VHD_FOOTER to the last page of the blob.
164
// 3. Upload the supplied FilesystemData from the start of the blob.
165
166
var err error
167
168
if req.Size%512 != 0 {
169
return fmt.Errorf("Size must be a multiple of 512")
170
}
171
if req.Size > VHD_SIZE-512 {
172
// Protect against writing over the VHD footer.
173
return fmt.Errorf("Size cannot be bigger than %d", VHD_SIZE-512)
174
}
175
176
// Step 1.
177
err = context.PutBlob(&PutBlobRequest{
178
Container: req.Container,
179
BlobType: "page",
180
Filename: req.Filename,
181
Size: VHD_SIZE,
182
})
183
184
if err != nil {
185
return err
186
}
187
188
// Step 2.
189
data, err := base64.StdEncoding.DecodeString(VHD_FOOTER)
190
if err != nil {
191
// This really shouldn't ever happen since there's a test to make sure
192
// it can be decoded.
193
panic(err)
194
}
195
dataReader := bytes.NewReader(data)
196
err = context.PutPage(&PutPageRequest{
197
Container: req.Container,
198
Filename: req.Filename,
199
StartRange: VHD_SIZE - 512, // last page of the blob
200
EndRange: VHD_SIZE - 1,
201
Data: dataReader,
202
})
203
204
if err != nil {
205
return err
206
}
207
208
// Step 3.
209
err = context.PutPage(&PutPageRequest{
210
Container: req.Container,
211
Filename: req.Filename,
212
StartRange: 0,
213
EndRange: req.Size - 1,
214
Data: req.FilesystemData,
215
})
216
217
if err != nil {
218
return err
219
}
220
221
return nil
222
222
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.1