Object Storage & Blob Storage

1// ============================================
2// Object Storage — Production Patterns
3// ============================================
4
5const AWS = require('aws-sdk');
6const s3 = new AWS.S3();
7
8// ---------- Pre-signed URL Upload Flow ----------
9
10// ✅ GOOD: Client uploads directly to S3 via pre-signed URL
11// Server never handles the large file!
12
13async function getUploadUrl(req, res) {
14  const { fileName, fileType } = req.body;
15  const key = `uploads/\${req.user.id}/\${Date.now()}-\${fileName}`;
16
17  // Generate a URL that allows direct upload to S3
18  const presignedUrl = await s3.getSignedUrlPromise('putObject', {
19    Bucket: 'my-app-uploads',
20    Key: key,
21    ContentType: fileType,
22    Expires: 300, // URL valid for 5 minutes
23    Conditions: [
24      ['content-length-range', 0, 10 * 1024 * 1024], // Max 10MB
25    ],
26  });
27
28  // Save metadata to database
29  await db.insert('files', {
30    userId: req.user.id,
31    s3Key: key,
32    fileName,
33    fileType,
34    status: 'pending_upload',
35  });
36
37  res.json({
38    uploadUrl: presignedUrl,
39    key,
40    expiresIn: 300,
41  });
42}
43
44// Client-side upload (browser):
45// const response = await fetch(uploadUrl, {
46//   method: 'PUT',
47//   body: file,
48//   headers: { 'Content-Type': file.type }
49// });
50
51// ---------- Pre-signed URL Download Flow ----------
52
53async function getDownloadUrl(req, res) {
54  const { fileId } = req.params;
55  const file = await db.findFile(fileId);
56
57  if (!file) return res.status(404).json({ error: 'File not found' });
58
59  // Check access permissions
60  if (file.userId !== req.user.id) {
61    return res.status(403).json({ error: 'Not authorized' });
62  }
63
64  const downloadUrl = await s3.getSignedUrlPromise('getObject', {
65    Bucket: 'my-app-uploads',
66    Key: file.s3Key,
67    Expires: 3600, // URL valid for 1 hour
68    ResponseContentDisposition:
69      `attachment; filename="\${file.fileName}"`,
70  });
71
72  res.json({ downloadUrl, expiresIn: 3600 });
73}
74
75// ---------- Image Upload with Processing Pipeline ----------
76
77// ❌ BAD: Process images synchronously in the API handler
78async function badImageUpload(req, res) {
79  const image = req.file;
80  const thumbnail = await sharp(image.buffer).resize(150, 150).toBuffer();
81  const medium = await sharp(image.buffer).resize(600, 600).toBuffer();
82  const large = await sharp(image.buffer).resize(1200, 1200).toBuffer();
83
84  await Promise.all([
85    s3Upload('thumbnails/', thumbnail),
86    s3Upload('medium/', medium),
87    s3Upload('large/', large),
88    s3Upload('original/', image.buffer),
89  ]);
90  // User waits for ALL processing + 4 uploads = 5-15 seconds!
91  res.json({ success: true });
92}
93
94// ✅ GOOD: Upload original, process asynchronously
95async function goodImageUpload(req, res) {
96  const key = `originals/\${req.user.id}/\${Date.now()}.jpg`;
97
98  // Upload original only
99  await s3.putObject({
100    Bucket: 'my-app-images',
101    Key: key,
102    Body: req.file.buffer,
103    ContentType: 'image/jpeg',
104  }).promise();
105
106  // Trigger async processing (S3 event → Lambda/SQS)
107  // Lambda will generate thumbnails and multiple sizes
108  await db.insert('images', {
109    userId: req.user.id,
110    originalKey: key,
111    status: 'processing', // Will become 'ready' after Lambda runs
112  });
113
114  // User gets immediate response
115  res.status(202).json({
116    message: 'Image uploaded, processing in background',
117    imageId: 'img_123',
118  });
119}
120
121// Lambda function triggered by S3 upload event:
122async function processImageLambda(event) {
123  const bucket = event.Records[0].s3.bucket.name;
124  const key = event.Records[0].s3.object.key;
125
126  const original = await s3.getObject({ Bucket: bucket, Key: key }).promise();
127  const sharp = require('sharp');
128
129  const sizes = [
130    { name: 'thumbnail', width: 150, height: 150 },
131    { name: 'medium', width: 600, height: 600 },
132    { name: 'large', width: 1200, height: 1200 },
133  ];
134
135  for (const size of sizes) {
136    const resized = await sharp(original.Body)
137      .resize(size.width, size.height, { fit: 'cover' })
138      .jpeg({ quality: 85 })
139      .toBuffer();
140
141    await s3.putObject({
142      Bucket: bucket,
143      Key: key.replace('originals/', `\${size.name}/`),
144      Body: resized,
145      ContentType: 'image/jpeg',
146      CacheControl: 'public, max-age=31536000', // Cache for 1 year
147    }).promise();
148  }
149
150  // Update database status
151  await db.update('images', { originalKey: key }, { status: 'ready' });
152}
153
154// ---------- Storage Class Selection ----------
155
156const storageClasses = {
157  'S3 Standard': {
158    cost: '$0.023/GB/month',
159    useCase: 'Frequently accessed data (user profiles, active content)',
160    retrieval: 'Instant',
161  },
162  'S3 Intelligent-Tiering': {
163    cost: '$0.023/GB/month (auto-optimized)',
164    useCase: 'Unknown access patterns',
165    retrieval: 'Instant',
166  },
167  'S3 Infrequent Access': {
168    cost: '$0.0125/GB/month',
169    useCase: 'Accessed less than once a month (old data)',
170    retrieval: 'Instant (per-request fee)',
171  },
172  'S3 Glacier': {
173    cost: '$0.004/GB/month',
174    useCase: 'Archival (compliance, backups)',
175    retrieval: 'Minutes to hours',
176  },
177  'S3 Glacier Deep Archive': {
178    cost: '$0.00099/GB/month',
179    useCase: 'Long-term archival (7+ years retention)',
180    retrieval: '12-48 hours',
181  },
182};
183
184async function s3Upload(prefix, buffer) {
185  // helper stub
186}
187
188console.log('Storage classes:', storageClasses);