Skip to content

Upload all content to s3 as public and immutable #1871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 22, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/bin/render-readmes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,14 @@ fn main() {
let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
config
.uploader
.upload(&client, &readme_path, content, content_length, "text/html")
.upload(
&client,
&readme_path,
content,
content_length,
"text/html",
None,
)
.unwrap_or_else(|_| {
panic!(
"[{}-{}] Couldn't upload file to S3",
Expand Down
2 changes: 2 additions & 0 deletions src/s3/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ impl Bucket {
content: R,
content_length: u64,
content_type: &str,
extra_headers: Option<header::HeaderMap>,
) -> reqwest::Result<reqwest::Response> {
let path = if path.starts_with('/') {
&path[1..]
Expand All @@ -60,6 +61,7 @@ impl Bucket {
.header(header::AUTHORIZATION, auth)
.header(header::CONTENT_TYPE, content_type)
.header(header::DATE, date)
.headers(extra_headers.unwrap_or_else(header::HeaderMap::new))
.body(reqwest::Body::sized(content, content_length))
.send()?
.error_for_status()
Expand Down
1 change: 1 addition & 0 deletions src/tasks/dump_db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ impl DumpTarball {
tarfile,
content_length,
"application/gzip",
None,
)
.map_err(std_error_no_send)?;
Ok(())
Expand Down
2 changes: 1 addition & 1 deletion src/tests/record.rs
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ fn replay_http(
mut exchange: Exchange,
stdout: &mut dyn Write,
) -> Box<dyn Future<Item = hyper::Response<hyper::Body>, Error = hyper::Error> + Send> {
static IGNORED_HEADERS: &[&str] = &["authorization", "date", "user-agent"];
static IGNORED_HEADERS: &[&str] = &["authorization", "date", "user-agent", "cache-control"];

assert_eq!(req.uri().to_string(), exchange.request.uri);
assert_eq!(req.method().to_string(), exchange.request.method);
Expand Down
32 changes: 28 additions & 4 deletions src/uploaders.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use conduit::Request;
use flate2::read::GzDecoder;
use openssl::hash::{Hasher, MessageDigest};
use reqwest::header;

use crate::util::LimitErrorReader;
use crate::util::{human, internal, CargoResult, ChainError, Maximums};
Expand All @@ -13,6 +14,8 @@ use std::sync::Arc;
use crate::middleware::app::RequestApp;
use crate::models::Crate;

pub const CACHE_CONTROL_IMMUTABLE: &str = "public,max-age=31536000,immutable";

#[derive(Clone, Debug)]
pub enum Uploader {
/// For production usage, uploads and redirects to s3.
Expand Down Expand Up @@ -70,13 +73,13 @@ impl Uploader {
}
}

/// Returns the interna path of an uploaded crate's version archive.
/// Returns the internal path of an uploaded crate's version archive.
fn crate_path(name: &str, version: &str) -> String {
// No slash in front so we can use join
format!("crates/{}/{}-{}.crate", name, name, version)
}

/// Returns the interna path of an uploaded crate's version readme.
/// Returns the internal path of an uploaded crate's version readme.
fn readme_path(name: &str, version: &str) -> String {
format!("readmes/{}/{}-{}.html", name, name, version)
}
Expand All @@ -91,11 +94,19 @@ impl Uploader {
mut content: R,
content_length: u64,
content_type: &str,
extra_headers: Option<header::HeaderMap>,
) -> CargoResult<Option<String>> {
match *self {
Uploader::S3 { ref bucket, .. } => {
bucket
.put(client, path, content, content_length, content_type)
.put(
client,
path,
content,
content_length,
content_type,
extra_headers,
)
.map_err(|e| internal(&format_args!("failed to upload to S3: {}", e)))?;
Ok(Some(String::from(path)))
}
Expand Down Expand Up @@ -126,12 +137,18 @@ impl Uploader {
let checksum = hash(&body);
let content_length = body.len() as u64;
let content = Cursor::new(body);
let mut extra_headers = header::HeaderMap::new();
extra_headers.insert(
header::CACHE_CONTROL,
CACHE_CONTROL_IMMUTABLE.parse().unwrap(),
);
self.upload(
app.http_client(),
&path,
content,
content_length,
"application/x-tar",
Some(extra_headers),
)?;
Ok(checksum)
}
Expand All @@ -146,7 +163,14 @@ impl Uploader {
let path = Uploader::readme_path(crate_name, vers);
let content_length = readme.len() as u64;
let content = Cursor::new(readme);
self.upload(http_client, &path, content, content_length, "text/html")?;
self.upload(
http_client,
&path,
content,
content_length,
"text/html",
None,
)?;
Ok(())
}
}
Expand Down