diff --git a/Cargo.lock b/Cargo.lock index b0125b482..fa1ead6c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,6 +42,20 @@ dependencies = [ "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "arc-swap" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "argon2rs" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "arrayref" version = "0.3.5" @@ -127,6 +141,15 @@ name = "bitflags" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "blake2-rfc" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "block-buffer" version = "0.3.3" @@ -178,7 +201,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "bytes" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -258,6 +281,18 @@ name = "cfg-if" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "chrono" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "clap" version = "2.32.0" @@ -373,7 +408,7 @@ source = "git+https://github.com/rust-lang/cargo.git#545f354259be4e9745ea00a524c dependencies = [ "curl 0.4.21 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", @@ -405,6 +440,9 @@ dependencies = [ "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "router 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_core 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_credential 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_s3 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", "sass-rs 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -529,6 +567,16 @@ dependencies = [ "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "dtoa" version = "0.2.2" @@ -781,10 +829,10 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -861,10 +909,10 @@ dependencies = [ [[package]] name = "http" -version = "0.1.13" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -902,14 +950,14 @@ dependencies = [ [[package]] name = "hyper" -version = "0.12.11" +version = "0.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -921,6 +969,7 @@ dependencies = [ "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-threadpool 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", "want 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -930,9 +979,9 @@ name = "hyper-tls" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.19 (registry+https://github.com/rust-lang/crates.io-index)", "native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1279,6 +1328,17 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "mio-named-pipes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mio-uds" version = "0.6.7" @@ -1635,7 +1695,7 @@ name = "postgres" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "fallible-iterator 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "postgres-protocol 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1650,7 +1710,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "fallible-iterator 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "hmac 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1895,6 +1955,17 @@ dependencies = [ "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "redox_users" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "regex" version = "0.2.11" @@ -1949,11 +2020,11 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "encoding_rs 0.8.10 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", - "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.19 (registry+https://github.com/rust-lang/crates.io-index)", "hyper-tls 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "libflate 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1984,6 +2055,64 @@ dependencies = [ "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rusoto_core" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hmac 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.19 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-tls 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "md5 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_credential 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "xml-rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rusoto_credential" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.19 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-process 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rusoto_s3" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "rusoto_core 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)", + "xml-rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustc-demangle" version = "0.1.9" @@ -2106,6 +2235,11 @@ dependencies = [ "schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "scopeguard" version = "0.3.3" @@ -2229,6 +2363,29 @@ name = "shell-escape" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "signal-hook" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "signal-hook-registry 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "signal-hook-registry" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arc-swap 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "siphasher" version = "0.2.3" @@ -2497,7 +2654,7 @@ name = "tokio" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2518,7 +2675,7 @@ name = "tokio-codec" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2555,11 +2712,26 @@ name = "tokio-io" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio-process" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", + "mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-signal 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tokio-reactor" version = "0.1.6" @@ -2577,12 +2749,28 @@ dependencies = [ "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio-signal" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", + "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "signal-hook 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tokio-tcp" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2620,7 +2808,7 @@ name = "tokio-udp" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2634,7 +2822,7 @@ name = "tokio-uds" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2942,6 +3130,14 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "xml-rs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [metadata] "checksum adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7e522997b529f05601e05166c07ed17789691f562762c7f3b987263d2dedee5c" "checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a" @@ -2949,6 +3145,8 @@ dependencies = [ "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" "checksum approx 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f71f10b5c4946a64aad7b8cf65e3406cd3da22fc448595991d22423cf6db67b4" +"checksum arc-swap 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "bc4662175ead9cd84451d5c35070517777949a2ed84551764129cedb88384841" +"checksum argon2rs 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3f67b0b6a86dae6e67ff4ca2b6201396074996379fba2b92ff649126f37cb392" "checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" @@ -2959,6 +3157,7 @@ dependencies = [ "checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +"checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" "checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" "checksum bodyparser 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6928e817538b74a73d1dd6e9a942a2a35c632a597b6bb14fd009480f859a6bf5" "checksum bstr 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "59604ece62a407dc9164732e5adea02467898954c3a5811fd2dc140af14ef15b" @@ -2966,11 +3165,12 @@ dependencies = [ "checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" "checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" "checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" -"checksum bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0ce55bd354b095246fc34caf4e9e242f5297a7fd938b090cadfea6eee614aa62" +"checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" "checksum bytesize 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "716960a18f978640f25101b5cbf1c6f6b0d3192fab36a2d98ca96f0ecbe41010" "checksum cargo 0.38.0 (git+https://github.com/rust-lang/cargo.git)" = "" "checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" "checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3" +"checksum chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "77d81f58b7301084de3b958691458a53c3f7e0b1d702f77e550b6a88e3a88abe" "checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum commoncrypto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d056a8586ba25a1e4d61cb090900e495952c7886786fc55f909ab2f819b69007" @@ -2995,6 +3195,7 @@ dependencies = [ "checksum curl 0.4.21 (registry+https://github.com/rust-lang/crates.io-index)" = "a85f2f95f2bd277d316d1aa8a477687ab4a6942258c7db7c89c187534669979c" "checksum curl-sys 0.4.18 (registry+https://github.com/rust-lang/crates.io-index)" = "9d91a0052d5b982887d8e829bee0faffc7218ea3c6ebd3d6c2c8f678a93c9a42" "checksum digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90" +"checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" "checksum dtoa 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0dd841b58510c9618291ffa448da2e4e0f699d984d436122372f446dae62263d" "checksum dtoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6d301140eb411af13d3115f9a562c85cc6b541ade9dfa314132244aaee7489dd" "checksum encoding_rs 0.8.10 (registry+https://github.com/rust-lang/crates.io-index)" = "065f4d0c826fdaef059ac45487169d918558e3cf86c9d89f6e81cf52369126e5" @@ -3033,11 +3234,11 @@ dependencies = [ "checksum hmac 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44f3bdb08579d99d7dc761c0e266f13b5f2ab8c8c703b9fc9ef333cd8f48f55e" "checksum home 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "80dff82fb58cfbbc617fb9a9184b010be0529201553cda50ad04372bc2333aff" "checksum html5ever 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b04478cf718862650a0bf66acaf8f2f8c906fbc703f35c916c1f4211b069a364" -"checksum http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "24f58e8c2d8e886055c3ead7b28793e1455270b5fb39650984c224bc538ba581" +"checksum http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "eed324f0f0daf6ec10c474f150505af2c143f251722bf9dbd1261bd1f2ee2c1a" "checksum httparse 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e8734b0cfd3bc3e101ec59100e101c2eecd19282202e87808b3037b442777a83" "checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" "checksum hyper 0.10.13 (registry+https://github.com/rust-lang/crates.io-index)" = "368cb56b2740ebf4230520e2b90ebb0461e69034d85d1945febd9b3971426db2" -"checksum hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)" = "78d50abbd1790e0f4c74cb1d4a2211b439bac661d54107ad5564c55e77906762" +"checksum hyper 0.12.19 (registry+https://github.com/rust-lang/crates.io-index)" = "f1ebec079129e43af5e234ef36ee3d7e6085687d145b7ea653b262d16c6b65f1" "checksum hyper-tls 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "32cd73f14ad370d3b4d4b7dce08f69b81536c82e39fcc89731930fe5788cd661" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" "checksum ignore 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8dc57fa12805f367736a38541ac1a9fc6a52812a0ca959b1d4d4b640a89eb002" @@ -3078,6 +3279,7 @@ dependencies = [ "checksum mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)" = "30de2e4613efcba1ec63d8133f344076952090c122992a903359be5a4f99c3ed" "checksum miniz-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "609ce024854aeb19a0ef7567d348aaa5a746b32fb72e336df7fcc16869d7e2b4" "checksum mio 0.6.16 (registry+https://github.com/rust-lang/crates.io-index)" = "71646331f2619b1026cc302f87a2b8b648d5c6dd6937846a16cc8ce0f347f432" +"checksum mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" "checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum miow 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "396aa0f2003d7df8395cb93e09871561ccc3e785f0acb369170e8cc74ddf9226" @@ -3146,6 +3348,7 @@ dependencies = [ "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +"checksum redox_users 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3fe5204c3a17e97dde73f285d49be585df59ed84b50a872baf416e73b62c3828" "checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" "checksum regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0b2f0808e7d7e4fb1cb07feb6ff2f4bc827938f24f8c2e6a3beb7370af544bdd" "checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" @@ -3154,6 +3357,9 @@ dependencies = [ "checksum reqwest 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1d68c7bf0b1dc3860b80c6d31d05808bf54cdc1bfc70a4680893791becd083ae" "checksum route-recognizer 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3255338088df8146ba63d60a9b8e3556f1146ce2973bc05a75181a42ce2256" "checksum router 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9b1797ff166029cb632237bb5542696e54961b4cf75a324c6f05c9cf0584e4e" +"checksum rusoto_core 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1a1069ba04874a485528d1602fab4569f2434a5547614428e2cc22b91bfb71" +"checksum rusoto_credential 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b0d6cc3a602f01b9c5a04c8ed4ee281b789c5b2692d93202367c9b99ebc022ed" +"checksum rusoto_s3 0.40.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4da6eac54781d2aac517a99f1d85d0d6a78674543f8d122d884628c1ff21b495" "checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" "checksum rustc-workspace-hack 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc71d2faa173b74b232dedc235e3ee1696581bb132fc116fa3626d6151a1a8fb" @@ -3170,6 +3376,7 @@ dependencies = [ "checksum scheduled-thread-pool 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a2ff3fc5223829be817806c6441279c676e454cc7da608faf03b0ccc09d3889" "checksum schemamama 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1f726d3b10198a91b545c12e55775ddf4abb681056aa62adf75ed00b68855ef9" "checksum schemamama_postgres 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9a69defe7b625fa5c4bfda0a1525c9729baef68f620e505464b7bf0a4d1697f6" +"checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum security-framework 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "697d3f3c23a618272ead9e1fb259c1411102b31c6af8b93f1d64cca9c3b0e8e0" "checksum security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab01dfbe5756785b5b4d46e0289e5a18071dfa9a7c2b24213ea00b9ef9b665bf" @@ -3185,6 +3392,9 @@ dependencies = [ "checksum serde_urlencoded 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "aaed41d9fb1e2f587201b863356590c90c1157495d811430a0c0325fe8169650" "checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" "checksum shell-escape 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "170a13e64f2a51b77a45702ba77287f5c6829375b04a69cf2222acd17d0cfab9" +"checksum shlex 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +"checksum signal-hook 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "72ab58f1fda436857e6337dcb6a5aaa34f16c5ddc87b3a8b6ef7a212f90b9c5a" +"checksum signal-hook-registry 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cded4ffa32146722ec54ab1f16320568465aa922aa9ab4708129599740da85d7" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum sized-chunks 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a2a2eb3fe454976eefb479f78f9b394d34d661b647c6326a3a6e66f68bb12c26" "checksum slab 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5f9776d6b986f77b35c6cf846c11ad986ff128fe0b2b63a3628e3755e8d3102d" @@ -3221,7 +3431,9 @@ dependencies = [ "checksum tokio-executor 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c117b6cf86bb730aab4834f10df96e4dd586eff2c3c27d3781348da49e255bde" "checksum tokio-fs 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b5cbe4ca6e71cb0b62a66e4e6f53a8c06a6eefe46cc5f665ad6f274c9906f135" "checksum tokio-io 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "8b8a85fffbec3c5ab1ab62324570230dcd37ee5996a7859da5caf7b9d45e3e8c" +"checksum tokio-process 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "88e1281e412013f1ff5787def044a9577a0bed059f451e835f1643201f8b777d" "checksum tokio-reactor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4b26fd37f1125738b2170c80b551f69ff6fecb277e6e5ca885e53eec2b005018" +"checksum tokio-signal 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "dd6dc5276ea05ce379a16de90083ec80836440d5ef8a6a39545a3207373b8296" "checksum tokio-tcp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7ad235e9dadd126b2d47f6736f65aa1fdcd6420e66ca63f44177bc78df89f912" "checksum tokio-threadpool 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "bbd8a8b911301c60cbfaa2a6588fb210e5c1038375b8bdecc47aa09a94c3c05f" "checksum tokio-timer 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3a52f00c97fedb6d535d27f65cccb7181c8dd4c6edc3eda9ea93f6d45d05168e" @@ -3271,3 +3483,4 @@ dependencies = [ "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +"checksum xml-rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c1cb601d29fe2c2ac60a2b2e5e293994d87a1f6fa9687a31a15270f909be9c2" diff --git a/Cargo.toml b/Cargo.toml index 2e22748df..9a6bb376a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,9 @@ html5ever = "0.22" cargo = { git = "https://github.com/rust-lang/cargo.git" } schemamama = "0.3" schemamama_postgres = "0.2" +rusoto_s3 = "0.40" +rusoto_core = "0.40" +rusoto_credential = "0.40" # iron dependencies diff --git a/src/db/file.rs b/src/db/file.rs index d22a2c38d..fdcc43b72 100644 --- a/src/db/file.rs +++ b/src/db/file.rs @@ -5,26 +5,20 @@ //! filesystem. This module is adding files into database and retrieving them. -use std::path::Path; +use std::path::{PathBuf, Path}; use postgres::Connection; use rustc_serialize::json::{Json, ToJson}; -use std::fs::File; +use std::fs; use std::io::Read; use error::Result; use failure::err_msg; - - -fn file_path(prefix: &str, name: &str) -> String { - match prefix.is_empty() { - true => name.to_owned(), - false => format!("{}/{}", prefix, name), - } -} +use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client}; +use rusoto_core::region::Region; +use rusoto_credential::EnvironmentProvider; fn get_file_list_from_dir>(path: P, - prefix: &str, - files: &mut Vec) + files: &mut Vec) -> Result<()> { let path = path.as_ref(); @@ -32,11 +26,9 @@ fn get_file_list_from_dir>(path: P, let file = try!(file); if try!(file.file_type()).is_file() { - file.file_name().to_str().map(|name| files.push(file_path(prefix, name))); + files.push(file.path()); } else if try!(file.file_type()).is_dir() { - file.file_name() - .to_str() - .map(|name| get_file_list_from_dir(file.path(), &file_path(prefix, name), files)); + try!(get_file_list_from_dir(file.path(), files)); } } @@ -44,23 +36,80 @@ fn get_file_list_from_dir>(path: P, } -pub fn get_file_list>(path: P) -> Result> { +pub fn get_file_list>(path: P) -> Result> { let path = path.as_ref(); - let mut files: Vec = Vec::new(); + let mut files = Vec::new(); if !path.exists() { return Err(err_msg("File not found")); } else if path.is_file() { - path.file_name() - .and_then(|name| name.to_str()) - .map(|name| files.push(format!("{}", name))); + files.push(PathBuf::from(path.file_name().unwrap())); } else if path.is_dir() { - try!(get_file_list_from_dir(path, "", &mut files)); + try!(get_file_list_from_dir(path, &mut files)); + for file_path in &mut files { + // We want the paths in this list to not be {path}/bar.txt but just bar.txt + *file_path = PathBuf::from(file_path.strip_prefix(path).unwrap()); + } } Ok(files) } +pub struct Blob { + pub path: String, + pub mime: String, + pub date_updated: time::Timespec, + pub content: Vec, +} + +pub fn get_path(conn: &Connection, path: &str) -> Option { + let rows = conn.query("SELECT path, mime, date_updated, content + FROM files + WHERE path = $1", &[&path]).unwrap(); + + if rows.len() == 0 { + None + } else { + let row = rows.get(0); + let mut content = row.get(3); + if content == b"in-s3" { + let client = s3_client(); + content = client.and_then(|c| c.get_object(GetObjectRequest { + bucket: "rust-docs-rs".into(), + key: path.into(), + ..Default::default() + }).sync().ok()).and_then(|r| r.body).map(|b| { + let mut b = b.into_blocking_read(); + let mut content = Vec::new(); + b.read_to_end(&mut content).unwrap(); + content + }).unwrap(); + }; + + Some(Blob { + path: row.get(0), + mime: row.get(1), + date_updated: row.get(2), + content, + }) + } +} + +fn s3_client() -> Option { + // If AWS keys aren't configured, then presume we should use the DB exclusively + // for file storage. + if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() { + return None; + } + Some(S3Client::new_with( + rusoto_core::request::HttpClient::new().unwrap(), + EnvironmentProvider::default(), + std::env::var("S3_ENDPOINT").ok().map(|e| Region::Custom { + name: "us-west-1".to_owned(), + endpoint: e, + }).unwrap_or(Region::UsWest1), + )) +} /// Adds files into database and returns list of files with their mime type in Json pub fn add_path_into_database>(conn: &Connection, @@ -72,20 +121,23 @@ pub fn add_path_into_database>(conn: &Connection, try!(cookie.load::<&str>(&[])); let trans = try!(conn.transaction()); + let client = s3_client(); + let mut file_list_with_mimes: Vec<(String, PathBuf)> = Vec::new(); - let mut file_list_with_mimes: Vec<(String, String)> = Vec::new(); - - for file_path_str in try!(get_file_list(&path)) { + for file_path in try!(get_file_list(&path)) { let (path, content, mime) = { - let path = Path::new(path.as_ref()).join(&file_path_str); + let path = Path::new(path.as_ref()).join(&file_path); // Some files have insufficient permissions (like .lock file created by cargo in // documentation directory). We are skipping this files. - let mut file = match File::open(path) { + let mut file = match fs::File::open(path) { Ok(f) => f, Err(_) => continue, }; let mut content: Vec = Vec::new(); try!(file.read_to_end(&mut content)); + let bucket_path = Path::new(prefix).join(&file_path) + .into_os_string().into_string().unwrap(); + let mime = { let mime = try!(cookie.buffer(&content)); // css's are causing some problem in browsers @@ -93,9 +145,10 @@ pub fn add_path_into_database>(conn: &Connection, // convert them to text/css // do the same for javascript files if mime == "text/plain" { - if file_path_str.ends_with(".css") { + let e = file_path.extension().unwrap_or_default(); + if e == "css" { "text/css".to_owned() - } else if file_path_str.ends_with(".js") { + } else if e == "js" { "application/javascript".to_owned() } else { mime.to_owned() @@ -105,14 +158,42 @@ pub fn add_path_into_database>(conn: &Connection, } }; - file_list_with_mimes.push((mime.clone(), file_path_str.clone())); + let content: Option> = if let Some(client) = &client { + let s3_res = client.put_object(PutObjectRequest { + acl: Some("public-read".into()), + bucket: "rust-docs-rs".into(), + key: bucket_path.clone(), + body: Some(content.clone().into()), + content_type: Some(mime.clone()), + ..Default::default() + }).sync(); + match s3_res { + // we've successfully uploaded the content, so steal it; + // we don't want to put it in the DB + Ok(_) => None, + // Since s3 was configured, we want to panic on failure to upload. + Err(e) => { + panic!("failed to upload to {}: {:?}", bucket_path, e) + }, + } + } else { + Some(content.clone().into()) + }; + + file_list_with_mimes.push((mime.clone(), file_path.clone())); - (file_path(prefix, &file_path_str), content, mime) + ( + bucket_path, + content, + mime, + ) }; // check if file already exists in database let rows = try!(conn.query("SELECT COUNT(*) FROM files WHERE path = $1", &[&path])); + let content = content.unwrap_or_else(|| "in-s3".to_owned().into()); + if rows.get(0).get::(0) == 0 { try!(trans.query("INSERT INTO files (path, mime, content) VALUES ($1, $2, $3)", &[&path, &mime, &content])); @@ -130,14 +211,14 @@ pub fn add_path_into_database>(conn: &Connection, -fn file_list_to_json(file_list: Vec<(String, String)>) -> Result { +fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result { let mut file_list_json: Vec = Vec::new(); for file in file_list { let mut v: Vec = Vec::new(); v.push(file.0.clone()); - v.push(file.1.clone()); + v.push(file.1.into_os_string().into_string().unwrap()); file_list_json.push(v.to_json()); } @@ -150,8 +231,7 @@ fn file_list_to_json(file_list: Vec<(String, String)>) -> Result { mod test { extern crate env_logger; use std::env; - use super::{get_file_list, add_path_into_database}; - use super::super::connect_db; + use super::get_file_list; #[test] fn test_get_file_list() { @@ -162,16 +242,6 @@ mod test { assert!(files.unwrap().len() > 0); let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap(); - assert_eq!(files[0], "Cargo.toml"); - } - - #[test] - #[ignore] - fn test_add_path_into_database() { - let _ = env_logger::try_init(); - - let conn = connect_db().unwrap(); - let res = add_path_into_database(&conn, "example", env::current_dir().unwrap().join("src")); - assert!(res.is_ok()); + assert_eq!(files[0], std::path::Path::new("Cargo.toml")); } } diff --git a/src/db/mod.rs b/src/db/mod.rs index 372a35bb5..7c51c9f48 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -12,7 +12,7 @@ use r2d2; use r2d2_postgres; mod add_package; -mod file; +pub mod file; mod migrate; diff --git a/src/docbuilder/chroot_builder.rs b/src/docbuilder/chroot_builder.rs index 02295e57a..c9a90665e 100644 --- a/src/docbuilder/chroot_builder.rs +++ b/src/docbuilder/chroot_builder.rs @@ -244,7 +244,6 @@ impl DocBuilder { /// Remove documentation, build directory and sources directory of a package fn clean(&self, package: &Package) -> Result<()> { debug!("Cleaning package"); - use std::fs::remove_dir_all; let documentation_path = PathBuf::from(&self.options.destination) .join(package.manifest().name().as_str()); let source_path = source_path(&package).unwrap(); diff --git a/src/lib.rs b/src/lib.rs index 9e5cd4f68..bc9b41f51 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,6 +31,9 @@ extern crate toml; extern crate html5ever; extern crate schemamama; extern crate schemamama_postgres; +extern crate rusoto_s3; +extern crate rusoto_core; +extern crate rusoto_credential; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::ChrootBuilderResult; diff --git a/src/web/file.rs b/src/web/file.rs index 8cdbbca31..987f4f1d9 100644 --- a/src/web/file.rs +++ b/src/web/file.rs @@ -5,39 +5,15 @@ use time; use postgres::Connection; use iron::{Handler, Request, IronResult, Response, IronError}; use iron::status; +use crate::db; -pub struct File { - pub path: String, - pub mime: String, - pub date_added: time::Timespec, - pub date_updated: time::Timespec, - pub content: Vec, -} - +pub struct File(pub db::file::Blob); impl File { /// Gets file from database pub fn from_path(conn: &Connection, path: &str) -> Option { - - let rows = conn.query("SELECT path, mime, date_added, date_updated, content - FROM files - WHERE path = $1", - &[&path]) - .unwrap(); - - if rows.len() == 0 { - None - } else { - let row = rows.get(0); - Some(File { - path: row.get(0), - mime: row.get(1), - date_added: row.get(2), - date_updated: row.get(3), - content: row.get(4), - }) - } + Some(File(db::file::get_path(conn, path)?)) } @@ -45,19 +21,19 @@ impl File { pub fn serve(self) -> Response { use iron::headers::{CacheControl, LastModified, CacheDirective, HttpDate, ContentType}; - let mut response = Response::with((status::Ok, self.content)); + let mut response = Response::with((status::Ok, self.0.content)); let cache = vec![CacheDirective::Public, CacheDirective::MaxAge(super::STATIC_FILE_CACHE_DURATION as u32)]; - response.headers.set(ContentType(self.mime.parse().unwrap())); + response.headers.set(ContentType(self.0.mime.parse().unwrap())); response.headers.set(CacheControl(cache)); - response.headers.set(LastModified(HttpDate(time::at(self.date_updated)))); + response.headers.set(LastModified(HttpDate(time::at(self.0.date_updated)))); response } /// Checks if mime type of file is "application/x-empty" pub fn is_empty(&self) -> bool { - self.mime == "application/x-empty" + self.0.mime == "application/x-empty" } } diff --git a/src/web/mod.rs b/src/web/mod.rs index 5faeaa8c2..31950ac8a 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -504,8 +504,6 @@ fn opensearch_xml_handler(_: &mut Request) -> IronResult { } fn ico_handler(req: &mut Request) -> IronResult { - use iron::Url; - if let Some(&"favicon.ico") = req.url.path().last() { // if we're looking for exactly "favicon.ico", we need to defer to the handler that loads // from `public_html`, so return a 404 here to make the main handler carry on diff --git a/src/web/releases.rs b/src/web/releases.rs index 2247a8f39..74e536848 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -476,7 +476,6 @@ pub fn search_handler(req: &mut Request) -> IronResult { let mut resp = Response::with((status::Found, Redirect(url))); use iron::headers::{Expires, HttpDate}; - use time; resp.headers.set(Expires(HttpDate(time::now()))); return Ok(resp); } @@ -515,7 +514,6 @@ pub fn search_handler(req: &mut Request) -> IronResult { let mut resp = Response::with((status::Found, Redirect(url))); use iron::headers::{Expires, HttpDate}; - use time; resp.headers.set(Expires(HttpDate(time::now()))); return Ok(resp); } diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 702a3e452..a7fe8f8c0 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -222,7 +222,7 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { let mut content = RustdocPage::default(); - let file_content = ctry!(String::from_utf8(file.content)); + let file_content = ctry!(String::from_utf8(file.0.content)); let (head, body, mut body_class) = ctry!(utils::extract_head_and_body(&file_content)); content.head = head; diff --git a/src/web/source.rs b/src/web/source.rs index ea8679610..c824339e1 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -222,10 +222,10 @@ pub fn source_browser_handler(req: &mut Request) -> IronResult { let (content, is_rust_source) = if let Some(file) = file { // serve the file with DatabaseFileHandler if file isn't text and not empty - if !file.mime.starts_with("text") && !file.is_empty() { + if !file.0.mime.starts_with("text") && !file.is_empty() { return Ok(file.serve()); - } else if file.mime.starts_with("text") && !file.is_empty() { - (String::from_utf8(file.content).ok(), file.path.ends_with(".rs")) + } else if file.0.mime.starts_with("text") && !file.is_empty() { + (String::from_utf8(file.0.content).ok(), file.0.path.ends_with(".rs")) } else { (None, false) }