Repository navigation
Expand file tree
/
Copy pathbulk_log.js
More file actions
72 lines (57 loc) · 2.1 KB
/
Copy pathbulk_log.js
File metadata and controls
72 lines (57 loc) · 2.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
/**
The bulk logger writes the task stream directly to disk then uploads that file
to s3 after the task has completed running.
*/
const Debug = require('debug');
const fs = require('mz/fs');
const streamClosed = require('../stream_closed');
const { tmpdir } = require('os');
const { mkdtemp, rm } = require('fs/promises');
const { join, sep } = require('path');
const uploadToS3 = require('../upload_to_s3');
const zlib = require('zlib');
let debug = Debug('taskcluster-docker-worker:features:bulk_log');
let ARTIFACT_NAME = 'public/logs/terminal_bulk.log.gz';
class BulkLog {
constructor(artifact) {
this.featureName = 'bulkLogHandler';
this.artifactName = artifact || ARTIFACT_NAME;
}
async created(task) {
this.tmpDir = await mkdtemp(`${tmpdir()}${sep}`);
this.tmpFile = 'bulk_log';
this.filePath = join(this.tmpDir, this.tmpFile);
debug('Created BulkLog using tempfile: ' + this.filePath);
// Eventually we want to save the content as gzip on s3 or azure so we
// incrementally compress it via streams.
let gzip = zlib.createGzip();
// Pipe the task stream to a temp file on disk.
this.stream = fs.createWriteStream(this.filePath);
task.stream.pipe(gzip).pipe(this.stream);
}
async killed(task) {
if (task.isCanceled()) {return;}
//this.stream.end();
// Ensure the stream is completely written prior to uploading the temp file.
await streamClosed(this.stream);
// Open a new stream to read the entire log from disk (this in theory could
// be a huge file).
let diskStream = fs.createReadStream(this.filePath);
// expire the log when the task expires
let expiration = new Date(task.task.expires);
try {
await uploadToS3(task.queue, task.status.taskId, task.runId,
diskStream, this.artifactName, expiration, {
'content-type': 'text/plain; charset=utf-8',
'content-encoding': 'gzip',
});
} catch (err) {
debug(err);
throw err;
}
// Remove temporary directory
await rm(this.tmpDir, { recursive: true });
return this.artifactName;
}
}
module.exports = BulkLog;