diff --git a/Manifest.toml b/Manifest.toml index af053bf85..756ad5414 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,7 +1,12 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.7.3" +julia_version = "1.9.2" manifest_format = "2.0" +project_hash = "e70b5d720da0bc43b48ac1bdf723e65efb73e2c0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -9,27 +14,20 @@ uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -[[deps.ChainRulesCore]] -deps = ["Compat", "LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "e30f2f4e20f7f186dc36529910beaedc60cfa644" -uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" -version = "1.16.0" - -[[deps.ChangesOfVariables]] -deps = ["InverseFunctions", "LinearAlgebra", "Test"] -git-tree-sha1 = "2fba81a302a7be671aefe194f0525ef231104e7f" -uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" -version = "0.1.8" - [[deps.Compat]] -deps = ["Dates", "LinearAlgebra", "UUIDs"] +deps = ["UUIDs"] git-tree-sha1 = "e460f044ca8b99be31d35fe54fc33a5c33dd8ed7" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" version = "4.9.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.5+0" [[deps.ContextVariablesX]] deps = ["Compat", "Logging", "UUIDs"] @@ -62,38 +60,65 @@ git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" version = "0.9.3" +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -[[deps.InverseFunctions]] -deps = ["Test"] -git-tree-sha1 = "68772f49f54b479fa88ace904f6127f0a3bb2e46" -uuid = "3587e190-3f89-42d0-90ee-14403ec27112" -version = "0.1.12" - [[deps.IrrationalConstants]] git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" version = "0.2.2" +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" + [[deps.LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [[deps.LinearAlgebra]] -deps = ["Libdl", "libblastrampoline_jll"] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.LogExpFunctions]] -deps = ["ChainRulesCore", "ChangesOfVariables", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] git-tree-sha1 = "c3ce8e7420b3a6e071e0fe4745f5d4300e37b13f" uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" version = "0.3.24" + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -107,6 +132,11 @@ version = "0.5.10" deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" + [[deps.MemPool]] deps = ["DataStructures", "Distributed", "Mmap", "Random", "Serialization", "Sockets"] git-tree-sha1 = "b9c1a032c3c1310a857c061ce487c632eaa1faa4" @@ -122,18 +152,29 @@ version = "1.1.0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" + [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.21+4" [[deps.OrderedCollections]] git-tree-sha1 = "2e73fe17cac3c62ad1aebe70d44c963c3cfdc3e3" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" version = "1.6.2" +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.2" + [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" @@ -142,6 +183,10 @@ uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" deps = ["Printf"] uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + [[deps.Random]] deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -154,6 +199,7 @@ version = "1.3.0" [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -172,12 +218,13 @@ uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" version = "1.1.1" [[deps.SparseArrays]] -deps = ["LinearAlgebra", "Random"] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [[deps.Statistics]] deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.9.0" [[deps.StatsAPI]] deps = ["LinearAlgebra"] @@ -191,9 +238,20 @@ git-tree-sha1 = "75ebe04c5bed70b91614d684259b661c9e6274a4" uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" version = "0.34.0" -[[deps.Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+6" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" [[deps.TimespanLogging]] deps = ["Distributed", "Profile"] @@ -208,6 +266,22 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" + [[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl", "OpenBLAS_jll"] +deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/Project.toml b/Project.toml index b7efa5b7e..aa1289192 100644 --- a/Project.toml +++ b/Project.toml @@ -20,11 +20,30 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" TimespanLogging = "a526e669-04d3-4846-9525-c66122c55f63" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +[weakdeps] +Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" +FFMPEG = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" +Luxor = "ae8d54c2-7ccd-5906-9d76-62fc9837b5bc" +Mux = "a975b10e-0019-58db-a62f-e48ff68538c9" +ProfileSVG = "132c30aa-f267-4189-9183-c8a63c7e05e6" + +[extensions] +DaggerColorsExt = "Colors" +DaggerFFMPEGExt = "FFMPEG" +DaggerLuxorExt = ["Colors", "Luxor"] +DaggerMuxExt = "Mux" +DaggerProfileSVGExt = "ProfileSVG" + [compat] +Colors = "0.12" ContextVariablesX = "0.1" DataStructures = "0.18" +FFMPEG = "0.4" +Luxor = "3" MacroTools = "0.5" MemPool = "0.4.4" +Mux = "1" +ProfileSVG = "0.2" Requires = "1" StatsBase = "0.28, 0.29, 0.30, 0.31, 0.32, 0.33, 0.34" TimespanLogging = "0.1" @@ -32,7 +51,11 @@ julia = "1.7" [extras] Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" +FFMPEG = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" +Luxor = "ae8d54c2-7ccd-5906-9d76-62fc9837b5bc" +Mux = "a975b10e-0019-58db-a62f-e48ff68538c9" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +ProfileSVG = "132c30aa-f267-4189-9183-c8a63c7e05e6" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] diff --git a/src/ui/graph.jl b/ext/DaggerColorsExt.jl similarity index 96% rename from src/ui/graph.jl rename to ext/DaggerColorsExt.jl index fdd63ba50..9cfc85ac9 100644 --- a/src/ui/graph.jl +++ b/ext/DaggerColorsExt.jl @@ -1,8 +1,17 @@ -import .Colors +module DaggerColorsExt + +@static if isdefined(Base, :get_extension) + import Colors +else + import .Colors +end + +import Dagger +import Dagger: show_plan + +using Dagger.TimespanLogging: Timespan +using Dagger: Chunk, Thunk, Processor -import .Dagger -import .Dagger: show_plan -import .Dagger.TimespanLogging: Timespan ### DAG-based graphing @@ -288,3 +297,5 @@ function showfn(io, f::Function) end end showfn(io, f) = show(io, f) + +end # module DaggerColorsExt diff --git a/src/ui/video.jl b/ext/DaggerFFMPEGExt.jl similarity index 75% rename from src/ui/video.jl rename to ext/DaggerFFMPEGExt.jl index 6c4257d30..2952fdac5 100644 --- a/src/ui/video.jl +++ b/ext/DaggerFFMPEGExt.jl @@ -1,4 +1,12 @@ -import .FFMPEG, .FileIO +module DaggerFFMPEGExt + +@static if isdefined(Base, :get_extension) + import FFMPEG +else + import .FFMPEG +end + +import Dagger function combine_images(path, delay) fr = 1 / delay @@ -16,9 +24,11 @@ function combine_images(path, delay) "$path/final.mp4") end -function combine_gantt_images(ctx, svg_path::String, prof_path::String, delay) +function Dagger._combine_gantt_images(ctx, svg_path::String, prof_path::String, delay) combine_images(svg_path, delay) ctx.profile && combine_images(prof_path, delay) return (gantt="$svg_path/final.mp4", profile="$prof_path/final.mp4") end + +end # module DaggerFFMPEGExt diff --git a/src/ui/gantt-luxor.jl b/ext/DaggerLuxorExt.jl similarity index 90% rename from src/ui/gantt-luxor.jl rename to ext/DaggerLuxorExt.jl index 26e327e50..8c930fb16 100644 --- a/src/ui/gantt-luxor.jl +++ b/ext/DaggerLuxorExt.jl @@ -1,8 +1,14 @@ -import .Luxor: Drawing, finish, Point, background, sethue, fontsize, rect, text +module DaggerLuxorExt -prof_to_svg(::Any, ::Any, ::Any; kwargs...) = nothing +@static if isdefined(Base, :get_extension) + import Colors + import Luxor: Drawing, finish, Point, background, sethue, fontsize, rect, text +else + import .Colors + import .Luxor: Drawing, finish, Point, background, sethue, fontsize, rect, text +end -combine_gantt_images(::Any, ::Any, ::Any, ::Any) = ("", "") +import Dagger function draw_gantt(ctx, svg_path, prof_path; delay=2, width=1000, height=640, window_length=20) root_time = time_ns() @@ -33,8 +39,8 @@ function draw_gantt(ctx, svg_path, prof_path; delay=2, width=1000, height=640, w # Concatenate and render profile data if ctx.profile - prof_data, prof_lidict = logs_to_stackframes(window_logs) - prof_to_svg(prof_path, prof_data, prof_lidict, image_idx; width=width) + prof_data, prof_lidict = Dagger.logs_to_stackframes(window_logs) + Dagger._prof_to_svg(prof_path, prof_data, prof_lidict, image_idx; width=width) end for proc in unique(map(x->x[1].timeline[2], filter(x->x[1].category==:compute, window_logs))) @@ -98,7 +104,7 @@ function draw_gantt(ctx, svg_path, prof_path; delay=2, width=1000, height=640, w end if isdir(svg_path) final_paths = try - combine_gantt_images(ctx, svg_path, prof_path, delay) + Dagger._combine_gantt_images(ctx, svg_path, prof_path, delay) catch err @error "Image-to-video failed" exception=err ("", "") @@ -143,3 +149,5 @@ function show_gantt(ctx; delay=2, port=8000, width=1000, height=640, window_leng serve_gantt(svg_path, prof_path; port=port, delay=delay) end end + +end # module DaggerLuxorExt diff --git a/src/ui/gantt-mux.jl b/ext/DaggerMuxExt.jl similarity index 92% rename from src/ui/gantt-mux.jl rename to ext/DaggerMuxExt.jl index 7c37b2463..79128ee75 100644 --- a/src/ui/gantt-mux.jl +++ b/ext/DaggerMuxExt.jl @@ -1,4 +1,12 @@ -using .Mux +module DaggerMuxExt + +@static if isdefined(Base, :get_extension) + using Mux +else + using .Mux +end + +using Dagger function serve_gantt(svg_path, prof_path; port=8000, delay=5) # Setup Mux app @@ -65,3 +73,5 @@ function serve_gantt(svg_path, prof_path; port=8000, delay=5) end end end + +end # module DaggerMuxExt diff --git a/ext/DaggerProfileSVGExt.jl b/ext/DaggerProfileSVGExt.jl new file mode 100644 index 000000000..7760187fc --- /dev/null +++ b/ext/DaggerProfileSVGExt.jl @@ -0,0 +1,21 @@ +module DaggerProfileSVGExt + +@static if isdefined(Base, :get_extension) + import ProfileSVG +else + import .ProfileSVG +end + +import Dagger + +function Dagger._prof_to_svg(path::String, pr, lidict, image_idx; width=1000) + length(pr) > 0 || return + if isdir(path) + path = joinpath(path, repr(image_idx) * ".svg") + end + open(path, "w") do io + ProfileSVG.save(io, pr; lidict=lidict, width=width) + end +end + +end # module DaggerProfileSVGExt diff --git a/src/Dagger.jl b/src/Dagger.jl index 6bf708b6d..440e87908 100644 --- a/src/Dagger.jl +++ b/src/Dagger.jl @@ -62,31 +62,43 @@ include("ui/gantt-text.jl") # Logging include("lib/logging-events.jl") +# For interoperation of gantt-luxor and video: +_combine_gantt_images(::Any, ::Any, ::Any, ::Any) = ("", "") + +# For interoperation of gantt-luxor and profilesvg: +_prof_to_svg(::Any, ::Any, ::Any; kwargs...) = nothing + +@static if !isdefined(Base, :get_extension) + using Requires +end + function __init__() # Initialize system UUID system_uuid() - @require Colors="5ae59095-9a9b-59fe-a467-6f913c188581" begin - include("ui/graph.jl") - @require Luxor="ae8d54c2-7ccd-5906-9d76-62fc9837b5bc" begin - # Gantt chart renderer - include("ui/gantt-luxor.jl") + @static if !isdefined(Base, :get_extension) + @require Colors="5ae59095-9a9b-59fe-a467-6f913c188581" begin + include("../ext/DaggerColorsExt.jl") + @require Luxor="ae8d54c2-7ccd-5906-9d76-62fc9837b5bc" begin + # Gantt chart renderer + include("../ext/DaggerLuxorExt.jl") + end end @require Mux="a975b10e-0019-58db-a62f-e48ff68538c9" begin # Gantt chart HTTP server - include("ui/gantt-mux.jl") + include("../ext/DaggerMuxExt.jl") end - end - @require ProfileSVG="132c30aa-f267-4189-9183-c8a63c7e05e6" begin - # Profile renderer - include("ui/profile-profilesvg.jl") - end - @require FFMPEG="c87230d0-a227-11e9-1b43-d7ebe4e7570a" begin - @require FileIO="5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" begin + + @require ProfileSVG="132c30aa-f267-4189-9183-c8a63c7e05e6" begin + # Profile renderer + include("../ext/DaggerProfileSVGExt.jl") + end + @require FFMPEG="c87230d0-a227-11e9-1b43-d7ebe4e7570a" begin # Video generator - include("ui/video.jl") + include("../ext/DaggerFFMPEGExt.jl") end end + for tid in 1:Threads.nthreads() add_processor_callback!("__cpu_thread_$(tid)__") do ThreadProc(myid(), tid) diff --git a/src/ui/gantt-text.jl b/src/ui/gantt-text.jl index f2356c36f..2390c3fe1 100644 --- a/src/ui/gantt-text.jl +++ b/src/ui/gantt-text.jl @@ -22,7 +22,7 @@ function draw_gantt_text(ctx; delay=2, width=40, height=20, window_length=20) #= TODO: Concatenate and render profile data if ctx.profile prof_data, prof_lidict = logs_to_stackframes(window_logs) - prof_to_svg(prof_path, prof_data, prof_lidict, image_idx; width=width) + _prof_to_svg(prof_path, prof_data, prof_lidict, image_idx; width=width) end =# diff --git a/src/ui/profile-profilesvg.jl b/src/ui/profile-profilesvg.jl deleted file mode 100644 index 4bb1c381d..000000000 --- a/src/ui/profile-profilesvg.jl +++ /dev/null @@ -1,11 +0,0 @@ -import .ProfileSVG - -function prof_to_svg(path::String, pr, lidict, image_idx; width=1000) - length(pr) > 0 || return - if isdir(path) - path = joinpath(path, repr(image_idx) * ".svg") - end - open(path, "w") do io - ProfileSVG.save(io, pr; lidict=lidict, width=width) - end -end