diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bf1380f5a07bc..3df89f0f4d096 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,3 +4,11 @@ CODEOWNERS @JuliaLang/github-actions /.github/workflows/rerun_failed.yml @DilumAluthge /.github/workflows/statuses.yml @DilumAluthge +/.github/workflows/PrAssignee.yml @LilithHafner @DilumAluthge +/base/special/ @oscardssmith +/base/sort.jl @LilithHafner +/test/sorting.jl @LilithHafner +/stdlib/*_jll @giordano +/base/binaryplatforms.jl @giordano +/src/julia_gcext.h @fingolfin +/test/gcext/gcext.c @fingolfin diff --git a/.github/workflows/PrAssignee.yml b/.github/workflows/PrAssignee.yml new file mode 100644 index 0000000000000..074e7a5528ce8 --- /dev/null +++ b/.github/workflows/PrAssignee.yml @@ -0,0 +1,210 @@ +name: PR Assignee +on: + # Important security note: Do NOT use `actions/checkout` + # or any other method for checking out the pull request's source code. + # This is because the pull request's source code is untrusted, but the + # GITHUB_TOKEN has write permissions (because of the `on: pull_request_target` event). + # + # Quoting from the GitHub Docs: + # > For workflows that are triggered by the pull_request_target event, the GITHUB_TOKEN is granted + # > read/write repository permission unless the permissions key is specified and the workflow can access secrets, + # > even when it is triggered from a fork. + # > + # > Although the workflow runs in the context of the base of the pull request, + # > you should make sure that you do not check out, build, or run untrusted code from the pull request with this event. + # + # Source: https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request_target + # + # See also: https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/ + pull_request_target: + types: [opened, reopened, ready_for_review] + +# Permissions for the `GITHUB_TOKEN`: +permissions: + pull-requests: write # Needed in order to assign a user as the PR assignee + +jobs: + pr-assignee: + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.draft != true }} + steps: + # Important security note: As discussed above, do NOT use `actions/checkout` + # or any other method for checking out the pull request's source code. + # This is because the pull request's source code is untrusted, but the + # GITHUB_TOKEN has write permissions (because of the `on: pull_request_target` event). + - name: Add Assignee + # We pin all third-party actions to a full length commit SHA + # https://docs.github.com/en/actions/security-for-github-actions/security-guides/security-hardening-for-github-actions#using-third-party-actions + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + retries: 5 # retry GitHub API requests up to 5 times, with exponential backoff + retry-exempt-status-codes: 404 + # Don't retry 404 because we will hit a 404 when the PR author is a committer. + # This 404 is normal and expected. + # Do retry 400 and other 4xx errors because github sometimes (erroneously) + # returns a 4xx error code due to server errors. + script: | + const oldPrAssignees = context.payload.pull_request.assignees + .map(obj => obj.login) + console.log('oldPrAssignees: ', oldPrAssignees); + const prAuthor = context.payload.pull_request.user.login; + + // Check if the PR is opened by a collaborator on the repo, aka someone with write (commit) permissions or higher. + const relevantPerms = [ + // 'triage', // Uncomment this line if you don't want PRs from triagers to get auto-assignees. + 'push', + 'maintain', + 'admin', + ] + const allCollaboratorsNestedPromises = relevantPerms.map( + (perm) => github.paginate( + // We use the `/repos/{owner}/{repo}/collaborators` endpoint to avoid needing org scope permissions: + '/repos/{owner}/{repo}/collaborators', + { + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100, + permission: perm, + }, + (response) => response.data.map((collaboratorInfo) => collaboratorInfo.login), + ) + ) + const allCollaboratorsNested = await Promise.all(allCollaboratorsNestedPromises); + const allCollaboratorsFlattened = allCollaboratorsNested.flat(); + + // Skip BumpStdlibs.jl PRs + allCollaboratorsFlattened.push('DilumAluthgeBot'); + // Skip Dependabot PRs + allCollaboratorsFlattened.push('dependabot'); + + const isCollaborator = allCollaboratorsFlattened.includes(prAuthor); + + console.log('prAuthor: ', prAuthor); + console.log('isCollaborator: ', isCollaborator); + + // Load the list of assignable reviewers from the JuliaLang/pr-assignment repo at: + // https://github.com/JuliaLang/pr-assignment/blob/main/users.txt + // + // NOTE to JuliaLang committers: If you want to be assigned to new PRs, please add your + // GitHub username to that file. + + // Load file contents + const { data: fileContentsObj } = await github.rest.repos.getContent({ + owner: 'JuliaLang', + repo: 'pr-assignment', + path: 'users.txt', + ref: 'main', + }); + + const fileContentsBufferObj = Buffer.from(fileContentsObj.content, "base64"); + const fileContentsText = fileContentsBufferObj.toString("utf8"); + + // Find lines that match the following regex, and extract the usernames: + const regex = /^@([a-zA-Z0-9\-]+)(\s*?)?(#[\S]*?)?$/; + const assigneeCandidates = fileContentsText + .split('\n') + .map(line => line.trim()) + .map(line => line.match(regex)) + .filter(match => match !== null) + .map(match => match[1]); + + console.log('assigneeCandidates: ', assigneeCandidates); + if (assigneeCandidates.length < 1) { + const msg = 'ERROR: Could not find any assigneeCandidates'; + console.error(msg); + throw new Error(msg); + } + + if (oldPrAssignees.length >= 1) { + console.log('Skipping this PR, because it already has at least one assignee'); + return; + } + + + const RUNNER_DEBUG_original = process.env.RUNNER_DEBUG; + console.log('RUNNER_DEBUG_original: ', RUNNER_DEBUG_original); + if (RUNNER_DEBUG_original === undefined) { + var thisIsActionsRunnerDebugMode = false; + } else { + const RUNNER_DEBUG_trimmed = RUNNER_DEBUG_original.trim().toLowerCase() + if (RUNNER_DEBUG_trimmed.length < 1) { + var thisIsActionsRunnerDebugMode = false; + } else { + var thisIsActionsRunnerDebugMode = (RUNNER_DEBUG_trimmed == 'true') || (RUNNER_DEBUG_trimmed == '1'); + } + } + console.log('thisIsActionsRunnerDebugMode: ', thisIsActionsRunnerDebugMode); + + if (isCollaborator == true) { + + if (thisIsActionsRunnerDebugMode) { + // The PR author is a committer + // But thisIsActionsRunnerDebugMode is true, so we proceed to still run the rest of the script + console.log('PR is authored by JuliaLang committer, but thisIsActionsRunnerDebugMode is true, so we will still run the rest of the script: ', prAuthor); + } else { + // The PR author is a committer, so we skip assigning them + console.log('Skipping PR authored by JuliaLang committer: ', prAuthor); + console.log('Note: If you want to run the full script (even though the PR author is a committer), simply re-run this job with Actions debug logging enabled'); + return; + } + } + + var weDidEncounterError = false; + + // Assign random committer + const selectedAssignee = assigneeCandidates[Math.floor(Math.random()*assigneeCandidates.length)] + console.log('selectedAssignee: ', selectedAssignee); + console.log(`Attempting to assign @${selectedAssignee} to this PR...`); + await github.rest.issues.addAssignees({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + assignees: selectedAssignee, + }); + + // The following is commented out because the label only makes sense in the presence of a larger state machine + // // Add the "pr review" label + // const prReviewLabel = 'status: waiting for PR reviewer'; + // console.log('Attempting to add prReviewLabel to this PR...'); + // await github.rest.issues.addLabels({ + // owner: context.repo.owner, + // repo: context.repo.repo, + // issue_number: context.payload.pull_request.number, + // labels: [prReviewLabel], + // }); + + // Now get the updated PR info, and see if we were successful: + const updatedPrData = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + }); + const newPrAssignees = updatedPrData + .data + .assignees + .map(element => element.login) + console.log('newPrAssignees: ', newPrAssignees); + if (newPrAssignees.includes(selectedAssignee)) { + console.log(`Successfully assigned @${selectedAssignee}`); + } else { + weDidEncounterError = true; + console.log(`ERROR: Failed to assign @${selectedAssignee}`); + } + // const newPrLabels = updatedPrData + // .data + // .labels + // .map(element => element.name) + // console.log('newPrLabels: ', newPrLabels); + // if (newPrLabels.includes(prReviewLabel)) { + // console.log('Successfully added prReviewLabel'); + // } else { + // weDidEncounterError = true; + // console.log('ERROR: Failed to add add prReviewLabel'); + // } + + // Exit with error if any problems were encountered earlier + if (weDidEncounterError) { + const msg = 'ERROR: Encountered at least one problem while running the script'; + console.error(msg); + throw new Error(msg); + } diff --git a/.github/workflows/Typos.yml b/.github/workflows/Typos.yml index 6c9eeacc21800..4dcfdcf0095b9 100644 --- a/.github/workflows/Typos.yml +++ b/.github/workflows/Typos.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 5 steps: - name: Checkout the JuliaLang/julia repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Check spelling with typos diff --git a/.github/workflows/Whitespace.yml b/.github/workflows/Whitespace.yml index 37c9dbfd39a3c..7414365322292 100644 --- a/.github/workflows/Whitespace.yml +++ b/.github/workflows/Whitespace.yml @@ -15,10 +15,10 @@ jobs: timeout-minutes: 2 steps: - name: Checkout the JuliaLang/julia repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - - uses: julia-actions/setup-julia@9b79636afcfb07ab02c256cede01fe2db6ba808c # v2.6.0 + - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1 with: version: '1' - name: Check whitespace diff --git a/.github/workflows/cffconvert.yml b/.github/workflows/cffconvert.yml index 4c9debb246f3f..3e481f18e6f75 100644 --- a/.github/workflows/cffconvert.yml +++ b/.github/workflows/cffconvert.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out a copy of the repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false diff --git a/.gitignore b/.gitignore index 7975fa91a69d7..c4df2542005d4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,10 +9,12 @@ /usr-staging /Make.user /julia-* +/deps/jlutilities/depot /source-dist.tmp /source-dist.tmp1 /test/results_*.json /test/results_*.dat +/test/deps *.expmap *.exe diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000000..4dcaa2069fa1f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,115 @@ +# Information for AI agents + +## Module Organization +- `base/` - Core standard library (loaded at startup) +- `stdlib/` - Standard library packages (can be loaded independently) +- `Compiler/` - Julia compiler as a separate module (can be swapped) +- `src/` - C/C++ runtime and LLVM codegen +- `cli/` - Command-line interface and loader +- `doc/` - Documentation and User Manual + +## Running Julia + +You should have a recent binary copy of julia in your `$HOME/.juliaup/bin` directory. +You may use this julia executable for validation. +If a built version of Julia exists in the current source tree (at `usr/bin/julia`), +prefer that version. +Note that any changes you make to the source code after the binary is built +will not be reflected, unless you use `Revise`. + +## For all changes + +1. Run `make fix-whitespace` before creating the PR to make sure you're not committing any whitespace errors. + +## Building Julia + +If you made changes to the runtime (any files in `src/`), you will need to rebuild +julia. Run `make -j` to rebuild julia. This process may take up to 10 minutes +depending on your changes. + +After `make` run these static analysis checks: + - `make -C src clang-sa-` (replace `` with the basename of the file you modified) + - `make -C src clang-sagc-` which may require adding JL_GC_PUSH arguments, or JL_GC_PROMISE_ROOTED statements., or require fixing locks. Remember arguments are assumed rooted, so check the callers to make sure that is handled. If the value is being temporarily moved around in a struct or arraylist, `JL_GC_PROMISE_ROOTED(struct->field)` may be needed as a statement (it return void) immediately after reloading the struct before any use of struct. Put the promise as early in the code as is legal. + - `make -C src clang-tidy-` + +## Using Revise + +If you have made changes to files included in the system image (base/ or stdlib/), +and need to run code with these changes included, you can use `Revise`. +To do so, run `using Revise; Revise.track(Base)` (or Revise.track with the stdlib you modified). +The test system supports doing this automatically (see below). + +## Specific instructions for particular changes + +### Doctests + +#### Writing doctests + +If you are asked to write new doctests, first review `doc/src/devdocs/contributing/jldoctests.md` +for best practices. + +#### Verifying doctests +If you have changed any `jldoctest` code blocks you should take +the following steps to verify your work: +- Review `doc/src/devdocs/contributing/jldoctests.md`. In particular, determine + if any of the changed doctests require filters, labels or setup code. +- Run the doctests to verify that your change works: + - To run doctest with the pre-built juliaup: `make -C doc doctest=true revise=true JULIA_EXECUTABLE=$HOME/.juliaup/bin/julia` + - To run doctest with in-trr julia (preferred): `make -C doc doctest=true revise=true`. Do not pass any other options. + - IMPORTANT: The doctests may take up to 15 minutes. Do NOT terminate the doctests before completion. Do NOT use a timeout for doctests. + - If you are ChatGPT, you may have to increase yield_timeout_ms. + +Follow these steps for EVERY change you make in a doctest. + +### Test changes + +If you have changed a test (e.g. `foo`), you should run `make test-revise-foo` for the +corresponding test to ensure that the test is still passing with your changes. +- If you are adding a new test, add it to an existing test file. Do not create a new test file unless explicitly instructed. +- Write one comment at the top of the test to explain what is being tested. + Otherwise keep comments minimal. +- Use the environment variable `JULIA_TEST_FAILFAST=1` to make tests fail fast. + +### External dependencies + +When modifying external dependencies (patches in `deps/patches/` or version updates in `deps/`): + +1. Always test builds with `USE_BINARYBUILDER=0` to ensure source builds work correctly +2. For patches to external libraries: + - Verify the patch applies cleanly by running the extraction and patch steps + - Test the full build of the dependency: `make -C deps USE_BINARYBUILDER=0 compile-` + - Prefer using the full upstream commit in `git am` format (e.g., `git format-patch`) which includes proper commit metadata +3. When updating dependency versions, ensure all associated patches still apply + +### External JLLs + +To update a JLL to the latest version: +- Update the version number in the appropriate jll folder +- If the dependencies in the upstream jll changed, update the Project.toml +- Run `make -f contrib/refresh_checksums.mk ` to update the checksums. This may take a few minutes. + +### Writing code +After writing code, look up the docstring for each function you used. If there +are recommendations or additional considerations that apply to these functions, +make sure to take them into account. + +#### Specific instructions +- Do not `ccall` runtime C functions directly if there are existing wrappers for the function. +- Do not explicitly add a module prefix if the code you're adding is in the same module. E.g. do not use `Base.` for code in Base unless required. + +## Commit message formatting + +When writing commit messages, follow the format "component: Brief summary" for +the title. In the body of the commit message, provide a brief prose summary +of the purpose of the changes made. Do not specifically mention added tests, comments, +documentation, etc., unless this is the main purpose of the change. Do not mention +the test plan, unless it differs from what you were instructed to do in AGENTS.md. +If your change fixes one or more issues, use the syntax "Fixes #" at the end of the commit message, but do not include it in the title. + +When referencing external GitHub PRs or issues, use proper GitHub interlinking format (e.g., `owner/repo#123` for PRs/issues). +When fixing CI failures, include the link to the specific CI failure in the commit message. + +When creating pull requests, if the pull request consists of one commit only, +use the body of the commit for the body of the pull request. If there are multiple +commits in the pull request, follow the same guidelines for the pull request +as for the commit body. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000000000..47dc3e3d863cf --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c01890212df70..36ec53c6a181d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,25 +4,6 @@ Hi! If you are new to the Julia community: welcome, and thanks for trying Julia. If you are already familiar with Julia itself, this blog post by Katharine Hyatt on [Making your first Julia pull request](https://kshyatt.github.io/post/firstjuliapr/) is a great way to get started. - -# Table of Contents - -1. [Learning Julia](#learning-julia) -2. [Filing an issue](#filing-an-issue) - - [Before filing an issue](#before-filing-an-issue) - - [How to file a bug report](#how-to-file-a-bug-report) -3. [Submitting contributions](#submitting-contributions) - - [Contributor Checklist](#contributor-checklist) - - [Writing tests](#writing-tests) - - [Improving documentation](#improving-documentation) - - [Contributing to core functionality or base libraries](#contributing-to-core-functionality-or-base-libraries) - - [Contributing to the standard library](#contributing-to-the-standard-library) - - [Contributing to patch releases](#contributing-to-patch-releases) - - [Code Formatting Guidelines](#code-formatting-guidelines) - - [Git Recommendations For Pull Requests](#git-recommendations-for-pull-requests) -4. [Resources](#resources) - - ## Learning Julia [The learning page](https://julialang.org/learning) has a great list of resources for new and experienced users alike. @@ -73,315 +54,20 @@ A useful bug report filed as a GitHub issue provides information about how to re * Review discussions on the [Julia Discourse forum](https://discourse.julialang.org). -* For more detailed tips, read the [submission guide](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#submitting-contributions) below. +* If your pull request contains substantial contributions from a generative AI tool, please disclose so with details, and review all changes before opening. * Relax and have fun! -### Writing tests - -There are never enough tests. Track [code coverage at Codecov](https://codecov.io/github/JuliaLang/julia), and help improve it. - -1. Go visit https://codecov.io/github/JuliaLang/julia. - -2. Browse through the source files and find some untested functionality (highlighted in red) that you think you might be able to write a test for. - -3. Write a test that exercises this functionality---you can add your test to one of the existing files, or start a new one, whichever seems most appropriate to you. If you're adding a new test file, make sure you include it in the list of tests in `test/choosetests.jl`. https://docs.julialang.org/en/v1/stdlib/Test/ may be helpful in explaining how the testing infrastructure works. - -4. Run `make test-all` to rebuild Julia and run your new test(s). If you had to fix a bug or add functionality in `base`, this will ensure that your test passes and that you have not introduced extraneous whitespace. - -5. Submit the test as a pull request (PR). - -* Code for the buildbot configuration is maintained at: https://github.com/staticfloat/julia-buildbot -* You can see the current buildbot setup at: https://build.julialang.org/builders -* [Issue 9493](https://github.com/JuliaLang/julia/issues/9493) and [issue 11885](https://github.com/JuliaLang/julia/issues/11885) have more detailed discussion on code coverage. - -Code coverage shows functionality that still needs "proof of concept" tests. These are important, as are tests for tricky edge cases, such as converting between integer types when the number to convert is near the maximum of the range of one of the integer types. Even if a function already has some coverage on Codecov, it may still benefit from tests for edge cases. - -### Improving documentation - -*By contributing documentation to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* - -Julia's documentation source files are stored in the `doc/` directory and all docstrings are found in `base/`. Like everything else these can be modified using `git`. Documentation is built with [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl), which uses Markdown syntax. The HTML documentation can be built locally by running - -``` -make docs -``` - -from Julia's root directory. This will rebuild the Julia system image, then install or update the package dependencies required to build the documentation, and finally build the HTML documentation and place the resulting files in `doc/_build/html/`. - -> **Note** -> -> When making changes to any of Julia's documentation it is recommended that you run `make docs` to check that your changes are valid and do not produce any errors before opening a pull request. - -Below are outlined the three most common types of documentation changes and the steps required to perform them. Please note that the following instructions do not cover the full range of features provided by Documenter.jl. Refer to [Documenter's documentation](https://juliadocs.github.io/Documenter.jl/stable) if you encounter anything that is not covered by the sections below. - -#### Modifying files in `doc/src/` - -Most of the source text for the Julia Manual is located in `doc/src/`. To update or add new text to any one of the existing files the following steps should be followed: - -1. update the text in whichever `.md` files are applicable; -2. run `make docs` from the root directory; -3. check the output in `doc/_build/html/` to make sure the changes are correct; -4. commit your changes and open a pull request. - -> **Note** -> -> The contents of `doc/_build/` does **not** need to be committed when you make changes. - -To add a **new file** to `doc/src/` rather than updating a file replace step `1` above with - -1. add the file to the appropriate subdirectory in `doc/src/` and also add the file path to the `PAGES` vector in `doc/make.jl`. - -#### Modifying an existing docstring in `base/` - -All docstrings are written inline above the methods or types they are associated with and can be found by clicking on the `source` link that appears below each docstring in the HTML file. The steps needed to make a change to an existing docstring are listed below: - -1. find the docstring in `base/`; -2. update the text in the docstring; -3. run `make docs` from the root directory; -4. check the output in `doc/_build/html/` to make sure the changes are correct; -5. commit your changes and open a pull request. - -#### Adding a new docstring to `base/` - -The steps required to add a new docstring are listed below: - -1. find a suitable definition in `base/` that the docstring will be most applicable to; -2. add a docstring above the definition; -3. find a suitable `@docs` code block in one of the `doc/src/stdlib/` files where you would like the docstring to appear; -4. add the name of the definition to the `@docs` code block. For example, with a docstring added to a function `bar` - - ```julia - "..." - function bar(args...) - # ... - end - ``` - - you would add the name `bar` to a `@docs` block in `doc/src/stdlib/` - - ```@docs - foo - bar # <-- Added this one. - baz - ``` - -5. run `make docs` from the root directory; -6. check the output in `doc/_build/html` to make sure the changes are correct; -7. commit your changes and open a pull request. - -#### Doctests - -Examples written within docstrings can be used as testcases known as "doctests" by annotating code blocks with `jldoctest`. - - ```jldoctest - julia> uppercase("Docstring test") - "DOCSTRING TEST" - ``` - -A doctest needs to match an interactive REPL including the `julia>` prompt. It is recommended to add the header `# Examples` above the doctests. - -To run doctests you need to run `make -C doc doctest=true` from the root directory. You can use `make -C doc doctest=true revise=true` if you are modifying the doctests and don't want to rebuild Julia after each change (see details below about the Revise.jl workflow). - -#### News-worthy changes - -For new functionality and other substantial changes, add a brief summary to `NEWS.md`. The news item should cross reference the pull request (PR) parenthetically, in the form `([#pr])`. To add the PR reference number, first create the PR, then push an additional commit updating `NEWS.md` with the PR reference number. We periodically run `./julia doc/NEWS-update.jl` from the julia directory to update the cross-reference links, but this should not be done in a typical PR in order to avoid conflicting commits. - -#### Annotations for new features, deprecations and behavior changes - -API additions and deprecations, and minor behavior changes are allowed in minor version releases. -For documented features that are part of the public API, a compatibility note should be added into -the manual or the docstring. It should state the Julia minor version that changed the behavior -and have a brief message describing the change. - -At the moment, this should always be done with the following `compat` admonition -(so that it would be possible to programmatically find the annotations in the future): - - ``` - !!! compat "Julia 1.X" - This method was added in Julia 1.X. - ``` - -### Contributing to core functionality or base libraries - -*By contributing code to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* - -The Julia community uses [GitHub issues](https://github.com/JuliaLang/julia/issues) to track and discuss problems, feature requests, and pull requests (PR). - -Issues and pull requests should have self explanatory titles such that they can be understood from the list of PRs and Issues. -i.e. `Add {feature}` and `Fix {bug}` are good, `Fix #12345. Corrects the bug.` is bad. - -You can make pull requests for incomplete features to get code review. The convention is to open these as draft PRs and prefix -the pull request title with "WIP:" for Work In Progress, or "RFC:" for Request for Comments when work is completed and ready -for merging. This will prevent accidental merging of work that is in progress. - -Note: These instructions are for adding to or improving functionality in the base library. Before getting started, it can be helpful to discuss the proposed changes or additions on the [Julia Discourse forum](https://discourse.julialang.org) or in a GitHub issue---it's possible your proposed change belongs in a package rather than the core language. Also, keep in mind that changing stuff in the base can potentially break a lot of things. Finally, because of the time required to build Julia, note that it's usually faster to develop your code in stand-alone files, get it working, and then migrate it into the base libraries. - -Add new code to Julia's base libraries as follows (this is the "basic" approach; see a more efficient approach in the next section): - - 1. Edit the appropriate file in the `base/` directory, or add new files if necessary. Create tests for your functionality and add them to files in the `test/` directory. If you're editing C or Scheme code, most likely it lives in `src/` or one of its subdirectories, although some aspects of Julia's REPL initialization live in `cli/`. - - 2. Add any new files to `sysimg.jl` in order to build them into the Julia system image. - - 3. Add any necessary export symbols in `exports.jl`. - - 4. Include your tests in `test/Makefile` and `test/choosetests.jl`. - -Build as usual, and do `make clean testall` to test your contribution. If your contribution includes changes to Makefiles or external dependencies, make sure you can build Julia from a clean tree using `git clean -fdx` or equivalent (be careful – this command will delete any files lying around that aren't checked into git). - -#### Running specific tests - -There are `make` targets for running specific tests: - - make test-bitarray - -You can also use the `runtests.jl` script, e.g. to run `test/bitarray.jl` and `test/math.jl`: - - ./usr/bin/julia test/runtests.jl bitarray math - -#### Modifying base more efficiently with Revise.jl - -[Revise](https://github.com/timholy/Revise.jl) is a package that -tracks changes in source files and automatically updates function -definitions in your running Julia session. Using it, you can make -extensive changes to Base without needing to rebuild in order to test -your changes. - -Here is the standard procedure: - -1. If you are planning changes to any types or macros, make those - changes and build julia using `make`. (This is - necessary because `Revise` cannot handle changes to type - definitions or macros.) Unless it's - required to get Julia to build, you do not have to add any - functionality based on the new types, just the type definitions - themselves. - -2. Start a Julia REPL session. Then issue the following commands: - -```julia -using Revise # if you aren't launching it in your `.julia/config/startup.jl` -Revise.track(Base) -``` - -3. Edit files in `base/`, save your edits, and test the - functionality. - -If you need to restart your Julia session, just start at step 2 above. -`Revise.track(Base)` will note any changes from when Julia was last -built and incorporate them automatically. You only need to rebuild -Julia if you made code-changes that Revise cannot handle. - -For convenience, there are also `test-revise-*` targets for every [`test-*` -target](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#running-specific-tests) that use Revise to load any modifications to Base into the current -system image before running the corresponding test. This can be useful as a shortcut -on the command line (since tests aren't always designed to be run outside the -runtest harness). - -### Contributing to the standard library - -The standard library (stdlib) packages are baked into the Julia system image. -When running the ordinary test workflow on the stdlib packages, the system image -version overrides the version you are developing. -To test stdlib packages, you can do the following steps: - -1. Edit the UUID field of the `Project.toml` in the stdlib package -2. Change the current directory to the directory of the stdlib you are developing -3. Start julia with `julia --project=.` -4. You can now test the package by running `pkg> test` in Pkg mode. - -Because you changed the UUID, the package manager treats the stdlib package as -different from the one in the system image, and the system image version will -not override the package. - -Be sure to change the UUID value back before making the pull request. - -### Contributing to patch releases - -The process of [creating a patch release](https://docs.julialang.org/en/v1/devdocs/build/distributing/#Point-releasing-101) is roughly as follows: - -1. Create a new branch (e.g. `backports-release-1.10`) against the relevant minor release - branch (e.g. `release-1.10`). Usually a corresponding pull request is created as well. - -2. Add commits, nominally from `master` (hence "backports"), to that branch. - See below for more information on this process. - -3. Run the [BaseBenchmarks.jl](https://github.com/JuliaCI/BaseBenchmarks.jl) benchmark - suite and [PkgEval.jl](https://github.com/JuliaCI/PkgEval.jl) package ecosystem - exerciser against that branch. Nominally BaseBenchmarks.jl and PkgEval.jl are - invoked via [Nanosoldier.jl](https://github.com/JuliaCI/Nanosoldier.jl) from - the pull request associated with the backports branch. Fix any issues. - -4. Once all test and benchmark reports look good, merge the backports branch into - the corresponding release branch (e.g. merge `backports-release-1.10` into - `release-1.10`). - -5. Open a pull request that bumps the version of the relevant minor release to the - next patch version, e.g. as in [this pull request](https://github.com/JuliaLang/julia/pull/37718). - -6. Ping `@JuliaLang/releases` to tag the patch release and update the website. - -7. Open a pull request that bumps the version of the relevant minor release to the - next prerelease patch version, e.g. as in [this pull request](https://github.com/JuliaLang/julia/pull/37724). - -Step 2 above, i.e. backporting commits to the `backports-release-X.Y` branch, has largely -been automated via [`Backporter`](https://github.com/KristofferC/Backporter): Backporter -searches for merged pull requests with the relevant `backport-X.Y` tag, and attempts to -cherry-pick the commits from those pull requests onto the `backports-release-X.Y` branch. -Some commits apply successfully without intervention, others not so much. The latter -commits require "manual" backporting, with which help is generally much appreciated. -Backporter generates a report identifying those commits it managed to backport automatically -and those that require manual backporting; this report is usually copied into the first -post of the pull request associated with `backports-release-X.Y` and maintained as -additional commits are automatically and/or manually backported. - -When contributing a manual backport, if you have the necessary permissions, please push the -backport directly to the `backports-release-X.Y` branch. If you lack the relevant -permissions, please open a pull request against the `backports-release-X.Y` branch with the -manual backport. Once the manual backport is live on the `backports-release-X.Y` branch, -please remove the `backport-X.Y` tag from the originating pull request for the commits. - -### Code Formatting Guidelines - -#### General Formatting Guidelines for Julia code contributions - - - Follow the latest dev version of [Julia Style Guide](https://docs.julialang.org/en/v1/manual/style-guide/). - - use whitespace to make the code more readable - - no whitespace at the end of a line (trailing whitespace) - - comments are good, especially when they explain the algorithm - - try to adhere to a 92 character line length limit - - it is generally preferred to use ASCII operators and identifiers over - Unicode equivalents whenever possible - - in docstrings refer to the language as "Julia" and the executable as "`julia`" - -#### General Formatting Guidelines For C code contributions - - - 4 spaces per indentation level, no tabs - - space between `if` and `(` (`if (x) ...`) - - newline before opening `{` in function definitions - - `f(void)` for 0-argument function declarations - - newline between `}` and `else` instead of `} else {` - - if one part of an `if..else` chain uses `{ }` then all should - - no whitespace at the end of a line - -### Git Recommendations For Pull Requests - - - Avoid working from the `master` branch of your fork. Create a new branch as it will make it easier to update your pull request if Julia's `master` changes. - - Try to [squash](https://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) together small commits that make repeated changes to the same section of code, so your pull request is easier to review. A reasonable number of separate well-factored commits is fine, especially for larger changes. - - If any conflicts arise due to changes in Julia's `master`, prefer updating your pull request branch with `git rebase` versus `git merge` or `git pull`, since the latter will introduce merge commits that clutter the git history with noise that makes your changes more difficult to review. - - Descriptive commit messages are good. - - Using `git add -p` or `git add -i` can be useful to avoid accidentally committing unrelated changes. - - When linking to specific lines of code in discussion of an issue or pull request, hit the `y` key while viewing code on GitHub to reload the page with a URL that includes the specific version that you're viewing. That way any lines of code that you refer to will still make sense in the future, even if the content of the file changes. - - Whitespace can be automatically removed from existing commits with `git rebase`. - - To remove whitespace for the previous commit, run - `git rebase --whitespace=fix HEAD~1`. - - To remove whitespace relative to the `master` branch, run - `git rebase --whitespace=fix master`. - -#### Git Recommendations For Pull Request Reviewers +### Guidance for specific changes -- When merging, we generally like `squash+merge`. Unless it is the rare case of a PR with carefully staged individual commits that you want in the history separately, in which case `merge` is acceptable, but usually prefer `squash+merge`. +The julia project maintains a more in-depth `Contributor's Guide` as part of our +developer documentation. Here you can find more in-depth guidance for how to write +specific kinds of changes. In particular, you want want to read: +- [How to contribute code changes](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/code-changes.md) +- [How to contribute additional tests](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/tests.md) +- [How to work on documentation](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/documentation.md) +- [Workflow tips for working with git](https://github.com/JuliaLang/julia/blob/master/doc/src/devdocs/contributing/git-workflow.md) ## Resources diff --git a/Compiler/.gitignore b/Compiler/.gitignore new file mode 100644 index 0000000000000..ba39cc531edeb --- /dev/null +++ b/Compiler/.gitignore @@ -0,0 +1 @@ +Manifest.toml diff --git a/Compiler/LICENSE.md b/Compiler/LICENSE.md index 028a39923ef04..dbbcd7506fc1e 100644 --- a/Compiler/LICENSE.md +++ b/Compiler/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2009-2024: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, and other contributors: https://github.com/JuliaLang/julia/contributors +Copyright (c) 2009-2025: Jeff Bezanson, Stefan Karpinski, Viral B. Shah, and other contributors: https://github.com/JuliaLang/julia/contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Compiler/Project.toml b/Compiler/Project.toml index 994634f5a8b78..1a0cdf4abca39 100644 --- a/Compiler/Project.toml +++ b/Compiler/Project.toml @@ -1,6 +1,6 @@ name = "Compiler" uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" -version = "0.0.3" +version = "0.1.1" [compat] julia = "1.10" diff --git a/Compiler/README.md b/Compiler/README.md new file mode 100644 index 0000000000000..ae5aaa3f60792 --- /dev/null +++ b/Compiler/README.md @@ -0,0 +1,45 @@ +# The `Compiler` module + +This directory maintains the implementation of the Julia compiler. + +Through a bootstrapping process, it is bundled into the Julia runtime as `Base.Compiler`. + +You can also use this `Compiler` module as the `Compiler` standard library by following the steps below. + +## How to use + +To utilize this `Compiler.jl` standard library, you need to declare it as a dependency in +your `Project.toml` as follows: +> Project.toml +```toml +[deps] +Compiler = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" + +[compat] +Compiler = "0.1" +``` + +With the setup above, [the special placeholder version (v0.1)](https://github.com/JuliaLang/BaseCompiler.jl) +will be installed by default.[^1] + +[^1]: Currently, only version v0.1 is registered in the [General](https://github.com/JuliaRegistries/General) registry. + +If needed, you can switch to a custom implementation of the `Compiler` module by running +```julia-repl +pkg> dev /path/to/Compiler.jl # to use a local implementation +``` +or +```julia-repl +pkg> add https://url/of/Compiler/branch # to use a remote implementation +``` +This feature is particularly useful for developing or experimenting with alternative compiler implementations. + +> [!note] +> The Compiler.jl standard library is available starting from Julia v1.10. +> However, switching to a custom compiler implementation is supported only from +> Julia v1.12 onwards. + +> [!warning] +> When using a custom, non-`Base` version of `Compiler` implementation, it may be necessary +> to run `InteractiveUtils.@activate Compiler` to ensure proper functionality of certain +> reflection utilities. diff --git a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl index f430a4c84662f..ddf202f378fb5 100644 --- a/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl +++ b/Compiler/extras/CompilerDevTools/src/CompilerDevTools.jl @@ -45,27 +45,30 @@ function lookup_method_instance(f, args...) @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance} end -function Compiler.optimize(interp::SplitCacheInterp, opt::Compiler.OptimizationState, caller::Compiler.InferenceResult) - @invoke Compiler.optimize(interp::Compiler.AbstractInterpreter, opt::Compiler.OptimizationState, caller::Compiler.InferenceResult) - ir = opt.result.ir::Compiler.IRCode - override = GlobalRef(@__MODULE__(), :with_new_compiler) +function Compiler.transform_result_for_cache(interp::SplitCacheInterp, result::Compiler.InferenceResult, edges::Compiler.SimpleVector) + opt = result.src::Compiler.OptimizationState + ir = opt.optresult.ir::Compiler.IRCode + override = with_new_compiler for inst in ir.stmts stmt = inst[:stmt] isexpr(stmt, :call) || continue f = stmt.args[1] f === override && continue - if isa(f, GlobalRef) - T = widenconst(argextype(f, ir)) - T <: Core.Builtin && continue - end + T = widenconst(argextype(f, ir)) + T <: Core.Builtin && continue insert!(stmt.args, 1, override) insert!(stmt.args, 3, interp.owner) end + @invoke Compiler.transform_result_for_cache(interp::Compiler.AbstractInterpreter, result::Compiler.InferenceResult, edges::Compiler.SimpleVector) end with_new_compiler(f, args...; owner::SplitCacheOwner = SplitCacheOwner()) = with_new_compiler(f, owner, args...) function with_new_compiler(f, owner::SplitCacheOwner, args...) + # We try to avoid introducing `with_new_compiler` in the first place, + # but if we can't see the type, it's still possible to end up with a + # builtin here - simply forward to the ordinary builtin call. + isa(f, Core.Builtin) && return f(args...) mi = lookup_method_instance(f, args...) new_compiler_ci = Core.OptimizedGenerics.CompilerPlugins.typeinf( owner, mi, Compiler.SOURCE_MODE_ABI diff --git a/Compiler/extras/CompilerDevTools/test/runtests.jl b/Compiler/extras/CompilerDevTools/test/runtests.jl index e076de388f96b..89dc4696d9e1c 100644 --- a/Compiler/extras/CompilerDevTools/test/runtests.jl +++ b/Compiler/extras/CompilerDevTools/test/runtests.jl @@ -17,4 +17,8 @@ using CompilerDevTools: lookup_method_instance, SplitCacheInterp # required extra work to be cached under the same cache owner. mi = lookup_method_instance(do_work, 1, 2) @test haskey(cache, mi) + + # Should not error with a builtin whose type we do not know + f_unknown_builtin() = Base.compilerbarrier(:type, isa)(1, Int) + with_new_compiler(f_unknown_builtin, interp.owner) === true end; diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index 298ddd729ddd0..e1c167e57ed08 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -38,7 +38,7 @@ else using Core.Intrinsics, Core.IR using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstance, MethodMatch, - MethodTable, PartialOpaque, SimpleVector, TypeofVararg, + MethodTable, MethodCache, PartialOpaque, SimpleVector, TypeofVararg, _apply_iterate, apply_type, compilerbarrier, donotdelete, memoryref_isassigned, memoryrefget, memoryrefnew, memoryrefoffset, memoryrefset!, print, println, show, svec, typename, unsafe_write, write @@ -65,7 +65,9 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali structdiff, tls_world_age, unconstrain_vararg_length, unionlen, uniontype_layout, uniontypes, unsafe_convert, unwrap_unionall, unwrapva, vect, widen_diagonal, _uncompressed_ir, maybe_add_binding_backedge!, datatype_min_ninitialized, - partialstruct_init_undefs, fieldcount_noerror + partialstruct_init_undefs, fieldcount_noerror, _eval_import, _eval_using, + get_ci_mi + using Base.Order import Base: ==, _topmod, append!, convert, copy, copy!, findall, first, get, get!, @@ -120,6 +122,7 @@ function is_return_type(Core.@nospecialize(f)) return false end +include("timing.jl") include("sort.jl") # We don't include some.jl, but this definition is still useful. diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index f2b11996f672a..3552525f70cf3 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -363,15 +363,13 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: arg_n = split_argtypes[i]::Vector{Any} sig_n = argtypes_to_type(arg_n) sig_n === Bottom && continue - mt = ccall(:jl_method_table_for, Any, (Any,), sig_n) - mt === nothing && return FailedMethodMatch("Could not identify method table for call") - mt = mt::MethodTable thismatches = findall(sig_n, method_table(interp); limit = max_methods) if thismatches === nothing return FailedMethodMatch("For one of the union split cases, too many methods matched") end valid_worlds = intersect(valid_worlds, thismatches.valid_worlds) thisfullmatch = any(match::MethodMatch->match.fully_covers, thismatches) + mt = Core.methodtable thisinfo = MethodMatchInfo(thismatches, mt, sig_n, thisfullmatch) push!(infos, thisinfo) for idx = 1:length(thismatches) @@ -385,11 +383,6 @@ function find_union_split_method_matches(interp::AbstractInterpreter, argtypes:: end function find_simple_method_matches(interp::AbstractInterpreter, @nospecialize(atype), max_methods::Int) - mt = ccall(:jl_method_table_for, Any, (Any,), atype) - if mt === nothing - return FailedMethodMatch("Could not identify method table for call") - end - mt = mt::MethodTable matches = findall(atype, method_table(interp); limit = max_methods) if matches === nothing # this means too many methods matched @@ -397,6 +390,7 @@ function find_simple_method_matches(interp::AbstractInterpreter, @nospecialize(a return FailedMethodMatch("Too many methods matched") end fullmatch = any(match::MethodMatch->match.fully_covers, matches) + mt = Core.methodtable info = MethodMatchInfo(matches, mt, atype, fullmatch) applicable = MethodMatchTarget[MethodMatchTarget(matches[idx], info.edges, idx) for idx = 1:length(matches)] return MethodMatches(applicable, info, matches.valid_worlds) @@ -1402,8 +1396,9 @@ function matching_cache_argtypes(𝕃::AbstractLattice, mi::MethodInstance, if slotid !== nothing # using union-split signature, we may be able to narrow down `Conditional` sigt = widenconst(slotid > nargs ? argtypes[slotid] : cache_argtypes[slotid]) - thentype = tmeet(cnd.thentype, sigt) - elsetype = tmeet(cnd.elsetype, sigt) + ⊓ = meet(𝕃) + thentype = cnd.thentype ⊓ sigt + elsetype = cnd.elsetype ⊓ sigt if thentype === Bottom && elsetype === Bottom # we accidentally proved this method match is impossible # TODO bail out here immediately rather than just propagating Bottom ? @@ -2119,7 +2114,7 @@ function abstract_call_builtin(interp::AbstractInterpreter, f::Builtin, (; fargs else thentype = form_partially_defined_struct(𝕃ᵢ, argtype2, argtypes[3]) if thentype !== nothing - elsetype = argtype2 + elsetype = widenslotwrapper(argtype2) if rt === Const(false) thentype = Bottom elseif rt === Const(true) @@ -2214,18 +2209,12 @@ function abstract_call_unionall(interp::AbstractInterpreter, argtypes::Vector{An return CallMeta(ret, Any, Effects(EFFECTS_TOTAL; nothrow), call.info) end -function ci_abi(ci::CodeInstance) +function get_ci_abi(ci::CodeInstance) def = ci.def isa(def, ABIOverride) && return def.abi (def::MethodInstance).specTypes end -function get_ci_mi(ci::CodeInstance) - def = ci.def - isa(def, ABIOverride) && return def.def - return def::MethodInstance -end - function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::StmtInfo, sv::AbsIntState) argtypes = arginfo.argtypes ft′ = argtype_by_index(argtypes, 2) @@ -2237,7 +2226,7 @@ function abstract_invoke(interp::AbstractInterpreter, arginfo::ArgInfo, si::Stmt if isa(method_or_ci, CodeInstance) our_world = sv.world.this argtype = argtypes_to_type(pushfirst!(argtype_tail(argtypes, 4), ft)) - specsig = ci_abi(method_or_ci) + specsig = get_ci_abi(method_or_ci) defdef = get_ci_mi(method_or_ci).def exct = method_or_ci.exctype if !hasintersect(argtype, specsig) @@ -2425,11 +2414,15 @@ function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, s end function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) == 3 - return abstract_eval_getglobal(interp, sv, saw_latestworld, argtypes[2], argtypes[3]) - elseif length(argtypes) == 4 - return abstract_eval_getglobal(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 5 + if !isvarargtype(argtypes[end]) + if length(argtypes) == 3 + return abstract_eval_getglobal(interp, sv, saw_latestworld, argtypes[2], argtypes[3]) + elseif length(argtypes) == 4 + return abstract_eval_getglobal(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 5 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Any, generic_getglobal_exct, generic_getglobal_effects, NoCallInfo()) @@ -2437,6 +2430,7 @@ function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, s end @nospecs function abstract_eval_get_binding_type(interp::AbstractInterpreter, sv::AbsIntState, M, s) + @nospecialize M s ⊑ = partialorder(typeinf_lattice(interp)) if isa(M, Const) && isa(s, Const) (M, s) = (M.val, s.val) @@ -2444,7 +2438,7 @@ end return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) end gr = GlobalRef(M, s) - (valid_worlds, rt) = scan_leaf_partitions(interp, gr, sv.world) do interp, _, partition + (valid_worlds, rt) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, ::Core.Binding, partition::Core.BindingPartition local rt kind = binding_kind(partition) if is_some_guard(kind) || kind == PARTITION_KIND_DECLARED @@ -2469,12 +2463,17 @@ end end function abstract_eval_get_binding_type(interp::AbstractInterpreter, sv::AbsIntState, argtypes::Vector{Any}) - if length(argtypes) == 3 - return abstract_eval_get_binding_type(interp, sv, argtypes[2], argtypes[3]) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 4 + if !isvarargtype(argtypes[end]) + if length(argtypes) == 3 + return abstract_eval_get_binding_type(interp, sv, argtypes[2], argtypes[3]) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 4 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + else + return CallMeta(Type, Union{TypeError, ArgumentError}, EFFECTS_THROWS, NoCallInfo()) end - return CallMeta(Type, Union{TypeError, ArgumentError}, EFFECTS_THROWS, NoCallInfo()) end const setglobal!_effects = Effects(EFFECTS_TOTAL; effect_free=ALWAYS_FALSE, nothrow=false, inaccessiblememonly=ALWAYS_FALSE) @@ -2507,11 +2506,15 @@ end const generic_setglobal!_exct = Union{ArgumentError, TypeError, ErrorException, ConcurrencyViolationError} function abstract_eval_setglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) == 4 - return abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) - elseif length(argtypes) == 5 - return abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4], argtypes[5]) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 6 + if !isvarargtype(argtypes[end]) + if length(argtypes) == 4 + return abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) + elseif length(argtypes) == 5 + return abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4], argtypes[5]) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 6 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Any, generic_setglobal!_exct, setglobal!_effects, NoCallInfo()) @@ -2535,11 +2538,15 @@ function abstract_eval_swapglobal!(interp::AbstractInterpreter, sv::AbsIntState, end function abstract_eval_swapglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) == 4 - return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) - elseif length(argtypes) == 5 - return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4], argtypes[5]) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 6 + if !isvarargtype(argtypes[end]) + if length(argtypes) == 4 + return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) + elseif length(argtypes) == 5 + return abstract_eval_swapglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4], argtypes[5]) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 6 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Any, Union{generic_getglobal_exct,generic_setglobal!_exct}, setglobal!_effects, NoCallInfo()) @@ -2547,63 +2554,71 @@ function abstract_eval_swapglobal!(interp::AbstractInterpreter, sv::AbsIntState, end function abstract_eval_setglobalonce!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) in (4, 5, 6) - cm = abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) - if length(argtypes) >= 5 - goe = global_order_exct(argtypes[5], #=loading=#true, #=storing=#true) - cm = merge_exct(cm, goe) - end - if length(argtypes) == 6 - goe = global_order_exct(argtypes[6], #=loading=#true, #=storing=#false) - cm = merge_exct(cm, goe) - end - return CallMeta(Bool, cm.exct, cm.effects, cm.info) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 6 + if !isvarargtype(argtypes[end]) + if length(argtypes) in (4, 5, 6) + cm = abstract_eval_setglobal!(interp, sv, saw_latestworld, argtypes[2], argtypes[3], argtypes[4]) + if length(argtypes) >= 5 + goe = global_order_exct(argtypes[5], #=loading=#true, #=storing=#true) + cm = merge_exct(cm, goe) + end + if length(argtypes) == 6 + goe = global_order_exct(argtypes[6], #=loading=#true, #=storing=#false) + cm = merge_exct(cm, goe) + end + return CallMeta(Bool, cm.exct, cm.effects, cm.info) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 7 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Bool, generic_setglobal!_exct, setglobal!_effects, NoCallInfo()) end end - function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) - if length(argtypes) in (5, 6, 7) - (M, s, x, v) = argtypes[2], argtypes[3], argtypes[4], argtypes[5] - T = nothing - if isa(M, Const) && isa(s, Const) - M, s = M.val, s.val - M isa Module || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) - s isa Symbol || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) - gr = GlobalRef(M, s) - (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp, _, partition - partition_T = nothing - partition_rte = abstract_eval_partition_load(interp, partition) - if binding_kind(partition) == PARTITION_KIND_GLOBAL - partition_T = partition_restriction(partition) + if !isvarargtype(argtypes[end]) + if length(argtypes) in (5, 6, 7) + (M, s, x, v) = argtypes[2], argtypes[3], argtypes[4], argtypes[5] + T = nothing + if isa(M, Const) && isa(s, Const) + M, s = M.val, s.val + M isa Module || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) + s isa Symbol || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) + gr = GlobalRef(M, s) + v′ = RefValue{Any}(v) + (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp::AbstractInterpreter, binding::Core.Binding, partition::Core.BindingPartition + partition_T = nothing + partition_rte = abstract_eval_partition_load(interp, binding, partition) + if binding_kind(partition) == PARTITION_KIND_GLOBAL + partition_T = partition_restriction(partition) + end + partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v′[])[2]} + partition_rte = RTEffects(partition_rte.rt, partition_exct, partition_rte.effects) + Pair{RTEffects, Any}(partition_rte, partition_T) end - partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v)[2]} - partition_rte = RTEffects(partition_rte.rt, partition_exct, partition_rte.effects) - Pair{RTEffects, Any}(partition_rte, partition_T) + update_valid_age!(sv, valid_worlds) + effects = merge_effects(rte.effects, Effects(setglobal!_effects, nothrow=rte.exct===Bottom)) + sg = CallMeta(Any, rte.exct, effects, GlobalAccessInfo(convert(Core.Binding, gr))) + else + sg = abstract_eval_setglobal!(interp, sv, saw_latestworld, M, s, v) + end + if length(argtypes) >= 6 + goe = global_order_exct(argtypes[6], #=loading=#true, #=storing=#true) + sg = merge_exct(sg, goe) + end + if length(argtypes) == 7 + goe = global_order_exct(argtypes[7], #=loading=#true, #=storing=#false) + sg = merge_exct(sg, goe) end - update_valid_age!(sv, valid_worlds) - effects = merge_effects(rte.effects, Effects(setglobal!_effects, nothrow=rte.exct===Bottom)) - sg = CallMeta(Any, rte.exct, effects, GlobalAccessInfo(convert(Core.Binding, gr))) + rt = T === nothing ? + ccall(:jl_apply_cmpswap_type, Any, (Any,), S) where S : + ccall(:jl_apply_cmpswap_type, Any, (Any,), T) + return CallMeta(rt, sg.exct, sg.effects, sg.info) else - sg = abstract_eval_setglobal!(interp, sv, saw_latestworld, M, s, v) + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) end - if length(argtypes) >= 6 - goe = global_order_exct(argtypes[6], #=loading=#true, #=storing=#true) - sg = merge_exct(sg, goe) - end - if length(argtypes) == 7 - goe = global_order_exct(argtypes[7], #=loading=#true, #=storing=#false) - sg = merge_exct(sg, goe) - end - rt = T === nothing ? - ccall(:jl_apply_cmpswap_type, Any, (Any,), S) where S : - ccall(:jl_apply_cmpswap_type, Any, (Any,), T) - return CallMeta(rt, sg.exct, sg.effects, sg.info) - elseif !isvarargtype(argtypes[end]) || length(argtypes) > 8 + elseif length(argtypes) > 8 return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) else return CallMeta(Any, Union{generic_getglobal_exct,generic_setglobal!_exct}, setglobal!_effects, NoCallInfo()) @@ -2660,11 +2675,8 @@ function abstract_call_known(interp::AbstractInterpreter, @nospecialize(f), return Future(abstract_eval_isdefinedglobal(interp, argtypes[2], argtypes[3], Const(true), length(argtypes) == 4 ? argtypes[4] : Const(:unordered), si.saw_latestworld, sv)) - elseif f === Core.isdefinedglobal && 3 <= length(argtypes) <= 5 - return Future(abstract_eval_isdefinedglobal(interp, argtypes[2], argtypes[3], - length(argtypes) >= 4 ? argtypes[4] : Const(true), - length(argtypes) >= 5 ? argtypes[5] : Const(:unordered), - si.saw_latestworld, sv)) + elseif f === Core.isdefinedglobal + return Future(abstract_eval_isdefinedglobal(interp, sv, si.saw_latestworld, argtypes)) elseif f === Core.get_binding_type return Future(abstract_eval_get_binding_type(interp, sv, argtypes)) end @@ -3243,7 +3255,7 @@ function abstract_eval_copyast(interp::AbstractInterpreter, e::Expr, sstate::Sta return RTEffects(rt, Any, effects) end -function abstract_eval_isdefined_expr(interp::AbstractInterpreter, e::Expr, sstate::StatementState, +function abstract_eval_isdefined_expr(::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) sym = e.args[1] if isa(sym, SlotNumber) && sstate.vtypes !== nothing @@ -3253,7 +3265,7 @@ function abstract_eval_isdefined_expr(interp::AbstractInterpreter, e::Expr, ssta elseif !vtyp.undef rt = Const(true) # definitely assigned previously else # form `Conditional` to refine `vtyp.undef` in the then branch - rt = Conditional(sym, vtyp.typ, vtyp.typ; isdefined=true) + rt = Conditional(sym, widenslotwrapper(vtyp.typ), widenslotwrapper(vtyp.typ); isdefined=true) end return RTEffects(rt, Union{}, EFFECTS_TOTAL) end @@ -3345,6 +3357,23 @@ function abstract_eval_isdefinedglobal(interp::AbstractInterpreter, @nospecializ return CallMeta(Bool, Union{exct, TypeError, UndefVarError}, generic_isdefinedglobal_effects, NoCallInfo()) end +function abstract_eval_isdefinedglobal(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) + if !isvarargtype(argtypes[end]) + if 3 <= length(argtypes) <= 5 + return abstract_eval_isdefinedglobal(interp, argtypes[2], argtypes[3], + length(argtypes) >= 4 ? argtypes[4] : Const(true), + length(argtypes) >= 5 ? argtypes[5] : Const(:unordered), + saw_latestworld, sv) + else + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + end + elseif length(argtypes) > 6 + return CallMeta(Union{}, ArgumentError, EFFECTS_THROWS, NoCallInfo()) + else + return CallMeta(Bool, Union{ConcurrencyViolationError, TypeError, UndefVarError}, generic_isdefinedglobal_effects, NoCallInfo()) + end +end + function abstract_eval_throw_undef_if_not(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) condt = abstract_eval_value(interp, e.args[2], sstate, sv) condval = maybe_extract_const_bool(condt) @@ -3410,7 +3439,7 @@ function abstract_eval_statement_expr(interp::AbstractInterpreter, e::Expr, ssta elseif ehead === :cfunction return abstract_eval_cfunction(interp, e, sstate, sv) elseif ehead === :method - rt = (length(e.args) == 1) ? Any : Nothing + rt = (length(e.args) == 1) ? Any : Method return RTEffects(rt, Any, EFFECTS_UNKNOWN) elseif ehead === :copyast return abstract_eval_copyast(interp, e, sstate, sv) @@ -3456,7 +3485,59 @@ function refine_partial_type(@nospecialize t) return t end +abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::IRInterpretationState) = nothing + +function abstract_eval_nonlinearized_foreigncall_name(interp::AbstractInterpreter, e, sstate::StatementState, sv::AbsIntState) + if isexpr(e, :call) + n = length(e.args) + argtypes = Vector{Any}(undef, n) + callresult = Future{CallMeta}() + i::Int = 1 + nextstate::UInt8 = 0x0 + local ai, res + function evalargs(interp, sv) + if nextstate === 0x1 + @goto state1 + elseif nextstate === 0x2 + @goto state2 + end + while i <= n + ai = abstract_eval_nonlinearized_foreigncall_name(interp, e.args[i], sstate, sv) + if !isready(ai) + nextstate = 0x1 + return false + @label state1 + end + argtypes[i] = ai[].rt + i += 1 + end + res = abstract_call(interp, ArgInfo(e.args, argtypes), sstate, sv) + if !isready(res) + nextstate = 0x2 + return false + @label state2 + end + callresult[] = res[] + return true + end + evalargs(interp, sv) || push!(sv.tasks, evalargs) + return callresult + else + return Future(abstract_eval_basic_statement(interp, e, sstate, sv)) + end +end + function abstract_eval_foreigncall(interp::AbstractInterpreter, e::Expr, sstate::StatementState, sv::AbsIntState) + callee = e.args[1] + if isexpr(callee, :call) && length(callee.args) > 1 && callee.args[1] == GlobalRef(Core, :tuple) + # NOTE these expressions are not properly linearized + abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[2], sstate, sv) + if length(callee.args) > 2 + abstract_eval_nonlinearized_foreigncall_name(interp, callee.args[3], sstate, sv) + end + else + abstract_eval_value(interp, callee, sstate, sv) + end mi = frame_instance(sv) t = sp_type_rewrap(e.args[2], mi, true) for i = 3:length(e.args) @@ -3559,13 +3640,11 @@ function abstract_eval_binding_partition!(interp::AbstractInterpreter, g::Global return partition end -abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, ::Core.Binding, partition::Core.BindingPartition) = - abstract_eval_partition_load(interp, partition) -function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, partition::Core.BindingPartition) +function abstract_eval_partition_load(interp::Union{AbstractInterpreter,Nothing}, binding::Core.Binding, partition::Core.BindingPartition) kind = binding_kind(partition) isdepwarn = (partition.kind & PARTITION_FLAG_DEPWARN) != 0 local_getglobal_effects = Effects(generic_getglobal_effects, effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE) - if is_some_guard(kind) || kind == PARTITION_KIND_UNDEF_CONST + if is_some_guard(kind) if interp !== nothing && InferenceParams(interp).assume_bindings_static return RTEffects(Union{}, UndefVarError, EFFECTS_THROWS) else @@ -3591,15 +3670,27 @@ function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing # Could be replaced by a backdated const which has an effect, so we can't assume it won't. # Besides, we would prefer not to merge the world range for this into the world range for # _GLOBAL, because that would pessimize codegen. - local_getglobal_effects = Effects(local_getglobal_effects, effect_free=ALWAYS_FALSE) + effects = Effects(local_getglobal_effects, effect_free=ALWAYS_FALSE) rt = Any else rt = partition_restriction(partition) + effects = local_getglobal_effects end - return RTEffects(rt, UndefVarError, local_getglobal_effects) + if (interp !== nothing && InferenceParams(interp).assume_bindings_static && + kind in (PARTITION_KIND_GLOBAL, PARTITION_KIND_DECLARED) && + isdefined(binding, :value)) + exct = Union{} + effects = Effects(generic_getglobal_effects; nothrow=true) + else + # We do not assume in general that assigned global bindings remain assigned. + # The existence of pkgimages allows them to revert in practice. + exct = UndefVarError + end + return RTEffects(rt, exct, effects) end -function scan_specified_partitions(query::Function, walk_binding_partition::Function, interp, g::GlobalRef, wwr::WorldWithRange) +function scan_specified_partitions(query::F1, walk_binding_partition::F2, + interp::Union{AbstractInterpreter,Nothing}, g::GlobalRef, wwr::WorldWithRange) where {F1,F2} local total_validity, rte, binding_partition binding = convert(Core.Binding, g) lookup_world = max_world(wwr.valid_worlds) @@ -3632,48 +3723,44 @@ function scan_specified_partitions(query::Function, walk_binding_partition::Func return Pair{WorldRange, typeof(rte)}(total_validity, rte) end -scan_leaf_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) = +scan_leaf_partitions(query::F, ::Nothing, g::GlobalRef, wwr::WorldWithRange) where F = + scan_specified_partitions(query, walk_binding_partition, nothing, g, wwr) +scan_leaf_partitions(query::F, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) where F = scan_specified_partitions(query, walk_binding_partition, interp, g, wwr) -scan_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) = - scan_specified_partitions(query, - (b::Core.Binding, bpart::Core.BindingPartition, world::UInt)-> - Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}(WorldRange(bpart.min_world, bpart.max_world), b=>bpart), - interp, g, wwr) +function scan_partitions(query::F, interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) where F + walk_binding_partition = function (b::Core.Binding, partition::Core.BindingPartition, world::UInt) + Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}( + WorldRange(partition.min_world, partition.max_world), b=>partition) + end + return scan_specified_partitions(query, walk_binding_partition, interp, g, wwr) +end -abstract_load_all_consistent_leaf_partitions(interp, g::GlobalRef, wwr::WorldWithRange) = +abstract_load_all_consistent_leaf_partitions(interp::AbstractInterpreter, g::GlobalRef, wwr::WorldWithRange) = scan_leaf_partitions(abstract_eval_partition_load, interp, g, wwr) +abstract_load_all_consistent_leaf_partitions(::Nothing, g::GlobalRef, wwr::WorldWithRange) = + scan_leaf_partitions(abstract_eval_partition_load, nothing, g, wwr) -function abstract_eval_globalref_partition(interp, binding::Core.Binding, partition::Core.BindingPartition) - # For inference purposes, we don't particularly care which global binding we end up loading, we only - # care about its type. However, we would still like to terminate the world range for the particular - # binding we end up reaching such that codegen can emit a simpler pointer load. - Pair{RTEffects, Union{Nothing, Core.Binding}}( - abstract_eval_partition_load(interp, partition), - binding_kind(partition) in (PARTITION_KIND_GLOBAL, PARTITION_KIND_DECLARED) ? binding : nothing) -end - -function abstract_eval_globalref(interp, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) +function abstract_eval_globalref(interp::AbstractInterpreter, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) if saw_latestworld return RTEffects(Any, Any, generic_getglobal_effects) end - (valid_worlds, (ret, binding_if_global)) = scan_leaf_partitions(abstract_eval_globalref_partition, interp, g, sv.world) + # For inference purposes, we don't particularly care which global binding we end up loading, we only + # care about its type. However, we would still like to terminate the world range for the particular + # binding we end up reaching such that codegen can emit a simpler pointer load. + (valid_worlds, ret) = scan_leaf_partitions(abstract_eval_partition_load, interp, g, sv.world) update_valid_age!(sv, valid_worlds) - if ret.rt !== Union{} && ret.exct === UndefVarError && binding_if_global !== nothing && InferenceParams(interp).assume_bindings_static - if isdefined(binding_if_global, :value) - ret = RTEffects(ret.rt, Union{}, Effects(generic_getglobal_effects, nothrow=true)) - end - # We do not assume in general that assigned global bindings remain assigned. - # The existence of pkgimages allows them to revert in practice. - end return ret end function global_assignment_rt_exct(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, g::GlobalRef, @nospecialize(newty)) if saw_latestworld - return Pair{Any,Any}(newty, Union{ErrorException, TypeError}) + return Pair{Any,Any}(newty, ErrorException) + end + newty′ = RefValue{Any}(newty) + (valid_worlds, ret) = scan_partitions(interp, g, sv.world) do interp::AbstractInterpreter, ::Core.Binding, partition::Core.BindingPartition + global_assignment_binding_rt_exct(interp, partition, newty′[]) end - (valid_worlds, ret) = scan_partitions((interp, _, partition)->global_assignment_binding_rt_exct(interp, partition, newty), interp, g, sv.world) update_valid_age!(sv, valid_worlds) return ret end @@ -3688,10 +3775,10 @@ function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partitio ty = kind == PARTITION_KIND_DECLARED ? Any : partition_restriction(partition) wnewty = widenconst(newty) if !hasintersect(wnewty, ty) - return Pair{Any,Any}(Bottom, TypeError) + return Pair{Any,Any}(Bottom, ErrorException) elseif !(wnewty <: ty) retty = tmeet(typeinf_lattice(interp), newty, ty) - return Pair{Any,Any}(retty, TypeError) + return Pair{Any,Any}(retty, ErrorException) end return Pair{Any,Any}(newty, Bottom) end @@ -3707,6 +3794,108 @@ function abstract_eval_ssavalue(s::SSAValue, ssavaluetypes::Vector{Any}) return typ end +struct AbstractEvalBasicStatementResult + rt + exct + effects::Union{Nothing,Effects} + changes::Union{Nothing,StateUpdate} + refinements # ::Union{Nothing,SlotRefinement,Vector{Any}} + currsaw_latestworld::Bool + function AbstractEvalBasicStatementResult(rt, exct, effects::Union{Nothing,Effects}, + changes::Union{Nothing,StateUpdate}, refinements, currsaw_latestworld::Bool) + @nospecialize rt exct refinements + return new(rt, exct, effects, changes, refinements, currsaw_latestworld) + end +end + +@inline function abstract_eval_basic_statement( + interp::AbstractInterpreter, @nospecialize(stmt), sstate::StatementState, frame::InferenceState, + result::Union{Nothing,Future{RTEffects}}=nothing) + rt = nothing + exct = Bottom + changes = nothing + refinements = nothing + effects = nothing + currsaw_latestworld = sstate.saw_latestworld + if result !== nothing + @goto injectresult + end + if isa(stmt, NewvarNode) + changes = StateUpdate(stmt.slot, VarState(Bottom, true)) + elseif isa(stmt, PhiNode) + add_curr_ssaflag!(frame, IR_FLAGS_REMOVABLE) + # Implement convergence for PhiNodes. In particular, PhiNodes need to tmerge over + # the incoming values from all iterations, but `abstract_eval_phi` will only tmerge + # over the first and last iterations. By tmerging in the current old_rt, we ensure that + # we will not lose an intermediate value. + rt = abstract_eval_phi(interp, stmt, sstate, frame) + old_rt = frame.ssavaluetypes[frame.currpc] + rt = old_rt === NOT_FOUND ? rt : tmerge(typeinf_lattice(interp), old_rt, rt) + else + lhs = nothing + if isexpr(stmt, :(=)) + lhs = stmt.args[1] + stmt = stmt.args[2] + end + if !isa(stmt, Expr) + (; rt, exct, effects, refinements) = abstract_eval_special_value(interp, stmt, sstate, frame) + else + hd = stmt.head + if hd === :method + fname = stmt.args[1] + if isa(fname, SlotNumber) + changes = StateUpdate(fname, VarState(Any, false)) + end + elseif (hd === :code_coverage_effect || + # :boundscheck can be narrowed to Bool + (hd !== :boundscheck && is_meta_expr(stmt))) + rt = Nothing + elseif hd === :latestworld + currsaw_latestworld = true + rt = Nothing + else + result = abstract_eval_statement_expr(interp, stmt, sstate, frame)::Future{RTEffects} + if !isready(result) || !isempty(frame.tasks) + return result + + @label injectresult + # reload local variables + lhs = nothing + if isexpr(stmt, :(=)) + lhs = stmt.args[1] + stmt = stmt.args[2] + end + end + result = result[] + (; rt, exct, effects, refinements) = result + if effects.noub === NOUB_IF_NOINBOUNDS + if has_curr_ssaflag(frame, IR_FLAG_INBOUNDS) + effects = Effects(effects; noub=ALWAYS_FALSE) + elseif !propagate_inbounds(frame) + # The callee read our inbounds flag, but unless we propagate inbounds, + # we ourselves don't read our parent's inbounds. + effects = Effects(effects; noub=ALWAYS_TRUE) + end + end + @assert !isa(rt, TypeVar) "unhandled TypeVar" + rt = maybe_singleton_const(rt) + if !isempty(frame.pclimitations) + if rt isa Const || rt === Union{} + empty!(frame.pclimitations) + else + rt = LimitedAccuracy(rt, frame.pclimitations) + frame.pclimitations = IdSet{InferenceState}() + end + end + end + end + if lhs !== nothing && rt !== Bottom + changes = StateUpdate(lhs::SlotNumber, VarState(rt, false)) + end + end + return AbstractEvalBasicStatementResult(rt, exct, effects, changes, refinements, currsaw_latestworld) +end + struct BestguessInfo{Interp<:AbstractInterpreter} interp::Interp bestguess @@ -3987,14 +4176,16 @@ end # make as much progress on `frame` as possible (without handling cycles) struct CurrentState - result::Future + result::Future{RTEffects} currstate::VarTable currsaw_latestworld::Bool bbstart::Int bbend::Int - CurrentState(result::Future, currstate::VarTable, currsaw_latestworld::Bool, bbstart::Int, bbend::Int) = new(result, currstate, currsaw_latestworld, bbstart, bbend) + CurrentState(result::Future{RTEffects}, currstate::VarTable, currsaw_latestworld::Bool, bbstart::Int, bbend::Int) = + new(result, currstate, currsaw_latestworld, bbstart, bbend) CurrentState() = new() end + function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextresult::CurrentState) @assert !is_inferred(frame) W = frame.ip @@ -4013,7 +4204,9 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr bbend = nextresult.bbend currstate = nextresult.currstate currsaw_latestworld = nextresult.currsaw_latestworld - @goto injectresult + stmt = frame.src.code[currpc] + result = abstract_eval_basic_statement(interp, stmt, StatementState(currstate, currsaw_latestworld), frame, nextresult.result) + @goto injected_result end if currbb != 1 @@ -4166,87 +4359,15 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState, nextr end # Process non control-flow statements @assert isempty(frame.tasks) - rt = nothing - exct = Bottom - changes = nothing - refinements = nothing - effects = nothing - if isa(stmt, NewvarNode) - changes = StateUpdate(stmt.slot, VarState(Bottom, true)) - elseif isa(stmt, PhiNode) - add_curr_ssaflag!(frame, IR_FLAGS_REMOVABLE) - # Implement convergence for PhiNodes. In particular, PhiNodes need to tmerge over - # the incoming values from all iterations, but `abstract_eval_phi` will only tmerge - # over the first and last iterations. By tmerging in the current old_rt, we ensure that - # we will not lose an intermediate value. - rt = abstract_eval_phi(interp, stmt, StatementState(currstate, currsaw_latestworld), frame) - old_rt = frame.ssavaluetypes[currpc] - rt = old_rt === NOT_FOUND ? rt : tmerge(typeinf_lattice(interp), old_rt, rt) + sstate = StatementState(currstate, currsaw_latestworld) + result = abstract_eval_basic_statement(interp, stmt, sstate, frame) + if result isa Future{RTEffects} + return CurrentState(result, currstate, currsaw_latestworld, bbstart, bbend) else - lhs = nothing - if isexpr(stmt, :(=)) - lhs = stmt.args[1] - stmt = stmt.args[2] - end - if !isa(stmt, Expr) - (; rt, exct, effects, refinements) = abstract_eval_special_value(interp, stmt, StatementState(currstate, currsaw_latestworld), frame) - else - hd = stmt.head - if hd === :method - fname = stmt.args[1] - if isa(fname, SlotNumber) - changes = StateUpdate(fname, VarState(Any, false)) - end - elseif (hd === :code_coverage_effect || ( - hd !== :boundscheck && # :boundscheck can be narrowed to Bool - is_meta_expr(stmt))) - rt = Nothing - elseif hd === :latestworld - currsaw_latestworld = true - rt = Nothing - else - result = abstract_eval_statement_expr(interp, stmt, StatementState(currstate, currsaw_latestworld), frame)::Future - if !isready(result) || !isempty(frame.tasks) - return CurrentState(result, currstate, currsaw_latestworld, bbstart, bbend) - @label injectresult - # reload local variables - stmt = frame.src.code[currpc] - changes = nothing - lhs = nothing - if isexpr(stmt, :(=)) - lhs = stmt.args[1] - stmt = stmt.args[2] - end - result = nextresult.result::Future{RTEffects} - end - result = result[] - (; rt, exct, effects, refinements) = result - if effects.noub === NOUB_IF_NOINBOUNDS - if has_curr_ssaflag(frame, IR_FLAG_INBOUNDS) - effects = Effects(effects; noub=ALWAYS_FALSE) - elseif !propagate_inbounds(frame) - # The callee read our inbounds flag, but unless we propagate inbounds, - # we ourselves don't read our parent's inbounds. - effects = Effects(effects; noub=ALWAYS_TRUE) - end - end - @assert !isa(rt, TypeVar) "unhandled TypeVar" - rt = maybe_singleton_const(rt) - if !isempty(frame.pclimitations) - if rt isa Const || rt === Union{} - empty!(frame.pclimitations) - else - rt = LimitedAccuracy(rt, frame.pclimitations) - frame.pclimitations = IdSet{InferenceState}() - end - end - end - end - effects === nothing || merge_override_effects!(interp, effects, frame) - if lhs !== nothing && rt !== Bottom - changes = StateUpdate(lhs::SlotNumber, VarState(rt, false)) - end + @label injected_result + (; rt, exct, effects, changes, refinements, currsaw_latestworld) = result end + effects === nothing || merge_override_effects!(interp, effects, frame) if !has_curr_ssaflag(frame, IR_FLAG_NOTHROW) if exct !== Union{} update_exc_bestguess!(interp, exct, frame) @@ -4345,7 +4466,7 @@ function conditional_change(𝕃ᵢ::AbstractLattice, currstate::VarTable, condt # "causes" since we ignored those in the comparison newtyp = tmerge(𝕃ᵢ, newtyp, LimitedAccuracy(Bottom, oldtyp.causes)) end - # if this `Conditional` is from from `@isdefined condt.slot`, refine its `undef` information + # if this `Conditional` is from `@isdefined condt.slot`, refine its `undef` information newundef = condt.isdefined ? !then_or_else : vtype.undef return StateUpdate(SlotNumber(condt.slot), VarState(newtyp, newundef), #=conditional=#true) end diff --git a/Compiler/src/abstractlattice.jl b/Compiler/src/abstractlattice.jl index f1514c6c9eed6..4d0accedfc765 100644 --- a/Compiler/src/abstractlattice.jl +++ b/Compiler/src/abstractlattice.jl @@ -227,9 +227,10 @@ that should be forwarded along with constant propagation. end @nospecializeinfer function is_const_prop_profitable_arg(𝕃::ConstsLattice, @nospecialize t) if isa(t, Const) - # don't consider mutable values useful constants + # don't consider mutable values useful constants unless they have const fields val = t.val - return isa(val, Symbol) || isa(val, Type) || isa(val, Method) || isa(val, CodeInstance) || !ismutable(val) + return isa(val, Symbol) || isa(val, Type) || isa(val, Method) || isa(val, CodeInstance) || + !ismutable(val) || (typeof(val).name.constfields != C_NULL) end isa(t, PartialTypeVar) && return false # this isn't forwardable return is_const_prop_profitable_arg(widenlattice(𝕃), t) diff --git a/Compiler/src/bootstrap.jl b/Compiler/src/bootstrap.jl index 2671ea114e818..74943fc765f17 100644 --- a/Compiler/src/bootstrap.jl +++ b/Compiler/src/bootstrap.jl @@ -10,7 +10,7 @@ function activate_codegen!() Core.eval(Compiler, quote let typeinf_world_age = Base.tls_world_age() @eval Core.OptimizedGenerics.CompilerPlugins.typeinf(::Nothing, mi::MethodInstance, source_mode::UInt8) = - Base.invoke_in_world($(Expr(:$, :typeinf_world_age)), typeinf_ext_toplevel, mi, Base.tls_world_age(), source_mode) + Base.invoke_in_world($(Expr(:$, :typeinf_world_age)), typeinf_ext_toplevel, mi, Base.tls_world_age(), source_mode, Compiler.TRIM_NO) end end) end @@ -67,17 +67,10 @@ function bootstrap!() end mi = specialize_method(m.method, Tuple{params...}, m.sparams) #isa_compileable_sig(mi) || println(stderr, "WARNING: inferring `", mi, "` which isn't expected to be called.") - push!(methods, mi) + typeinf_ext_toplevel(mi, world, isa_compileable_sig(mi) ? SOURCE_MODE_ABI : SOURCE_MODE_NOT_REQUIRED, TRIM_NO) end end end - codeinfos = typeinf_ext_toplevel(methods, [world], TRIM_NO) - for i = 1:2:length(codeinfos) - ci = codeinfos[i]::CodeInstance - src = codeinfos[i + 1]::CodeInfo - isa_compileable_sig(ci.def) || continue # println(stderr, "WARNING: compiling `", ci.def, "` which isn't expected to be called.") - ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), ci, src) - end endtime = time() println("Base.Compiler ──── ", sub_float(endtime,starttime), " seconds") end diff --git a/Compiler/src/cicache.jl b/Compiler/src/cicache.jl index 9c528bc0ae822..4d188dbbc3450 100644 --- a/Compiler/src/cicache.jl +++ b/Compiler/src/cicache.jl @@ -14,7 +14,7 @@ end function setindex!(cache::InternalCodeCache, ci::CodeInstance, mi::MethodInstance) @assert ci.owner === cache.owner m = mi.def - if isa(m, Method) && m.module != Core + if isa(m, Method) ccall(:jl_push_newly_inferred, Cvoid, (Any,), ci) end ccall(:jl_mi_cache_insert, Cvoid, (Any, Any), mi, ci) diff --git a/Compiler/src/inferencestate.jl b/Compiler/src/inferencestate.jl index 659b867128177..8a8e5354e3d59 100644 --- a/Compiler/src/inferencestate.jl +++ b/Compiler/src/inferencestate.jl @@ -188,7 +188,7 @@ mutable struct LazyGenericDomtree{IsPostDom} end function get!(x::LazyGenericDomtree{IsPostDom}) where {IsPostDom} isdefined(x, :domtree) && return x.domtree - return @timeit "domtree 2" x.domtree = IsPostDom ? + return @zone "CC: DOMTREE_2" x.domtree = IsPostDom ? construct_postdomtree(x.ir) : construct_domtree(x.ir) end @@ -575,21 +575,23 @@ function (::ComputeTryCatch{Handler})(code::Vector{Any}, bbs::Union{Vector{Basic end # check if coverage mode is enabled -function should_insert_coverage(mod::Module, debuginfo::DebugInfo) - coverage_enabled(mod) && return true - JLOptions().code_coverage == 3 || return false +should_insert_coverage(mod::Module, debuginfo::DebugInfo) = should_instrument(mod, debuginfo, true) + +function should_instrument(mod::Module, debuginfo::DebugInfo, only_if_affects_optimizer::Bool=false) + instrumentation_enabled(mod, only_if_affects_optimizer) && return true + JLOptions().code_coverage == 3 || JLOptions().malloc_log == 3 || return false # path-specific coverage mode: if any line falls in a tracked file enable coverage for all - return _should_insert_coverage(debuginfo) + return _should_instrument(debuginfo) end -_should_insert_coverage(mod::Symbol) = is_file_tracked(mod) -_should_insert_coverage(mod::Method) = _should_insert_coverage(mod.file) -_should_insert_coverage(mod::MethodInstance) = _should_insert_coverage(mod.def) -_should_insert_coverage(mod::Module) = false -function _should_insert_coverage(info::DebugInfo) +_should_instrument(loc::Symbol) = is_file_tracked(loc) +_should_instrument(loc::Method) = _should_instrument(loc.file) +_should_instrument(loc::MethodInstance) = _should_instrument(loc.def) +_should_instrument(loc::Module) = false +function _should_instrument(info::DebugInfo) linetable = info.linetable - linetable === nothing || (_should_insert_coverage(linetable) && return true) - _should_insert_coverage(info.def) && return true + linetable === nothing || (_should_instrument(linetable) && return true) + _should_instrument(info.def) && return true return false end diff --git a/Compiler/src/optimize.jl b/Compiler/src/optimize.jl index 99283c30b2744..27fb1e2168639 100644 --- a/Compiler/src/optimize.jl +++ b/Compiler/src/optimize.jl @@ -113,14 +113,19 @@ set_inlineable!(src::CodeInfo, val::Bool) = function inline_cost_clamp(x::Int) x > MAX_INLINE_COST && return MAX_INLINE_COST x < MIN_INLINE_COST && return MIN_INLINE_COST - return convert(InlineCostType, x) + x = ccall(:jl_encode_inlining_cost, UInt8, (InlineCostType,), x) + x = ccall(:jl_decode_inlining_cost, InlineCostType, (UInt8,), x) + return x end +const SRC_FLAG_DECLARED_INLINE = 0x1 +const SRC_FLAG_DECLARED_NOINLINE = 0x2 + is_declared_inline(@nospecialize src::MaybeCompressed) = - ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 1 + ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_INLINE is_declared_noinline(@nospecialize src::MaybeCompressed) = - ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == 2 + ccall(:jl_ir_flag_inlining, UInt8, (Any,), src) == SRC_FLAG_DECLARED_NOINLINE ##################### # OptimizationState # @@ -144,19 +149,50 @@ struct InliningState{Interp<:AbstractInterpreter} edges::Vector{Any} world::UInt interp::Interp + opt_cache::IdDict{MethodInstance,CodeInstance} +end +function InliningState(sv::InferenceState, interp::AbstractInterpreter, + opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) + return InliningState(sv.edges, frame_world(sv), interp, opt_cache) end -function InliningState(sv::InferenceState, interp::AbstractInterpreter) - return InliningState(sv.edges, frame_world(sv), interp) +function InliningState(interp::AbstractInterpreter, + opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) + return InliningState(Any[], get_inference_world(interp), interp, opt_cache) +end + +struct OptimizerCache{CodeCache} + wvc::WorldView{CodeCache} + owner + opt_cache::IdDict{MethodInstance,CodeInstance} + function OptimizerCache( + wvc::WorldView{CodeCache}, + @nospecialize(owner), + opt_cache::IdDict{MethodInstance,CodeInstance}) where CodeCache + new{CodeCache}(wvc, owner, opt_cache) + end end -function InliningState(interp::AbstractInterpreter) - return InliningState(Any[], get_inference_world(interp), interp) +function get((; wvc, owner, opt_cache)::OptimizerCache, mi::MethodInstance, default) + if haskey(opt_cache, mi) + codeinst = opt_cache[mi] + @assert codeinst.min_world ≤ wvc.worlds.min_world && + wvc.worlds.max_world ≤ codeinst.max_world && + codeinst.owner === owner + @assert isdefined(codeinst, :inferred) && codeinst.inferred === nothing + return codeinst + end + return get(wvc, mi, default) end # get `code_cache(::AbstractInterpreter)` from `state::InliningState` -code_cache(state::InliningState) = WorldView(code_cache(state.interp), state.world) +function code_cache(state::InliningState) + cache = WorldView(code_cache(state.interp), state.world) + owner = cache_owner(state.interp) + return OptimizerCache(cache, owner, state.opt_cache) +end mutable struct OptimizationResult ir::IRCode + inline_flag::UInt8 simplified::Bool # indicates whether the IR was processed with `cfg_simplify!` end @@ -168,7 +204,7 @@ end mutable struct OptimizationState{Interp<:AbstractInterpreter} linfo::MethodInstance src::CodeInfo - result::Union{Nothing, OptimizationResult} + optresult::Union{Nothing, OptimizationResult} stmt_info::Vector{CallInfo} mod::Module sptypes::Vector{VarState} @@ -179,13 +215,15 @@ mutable struct OptimizationState{Interp<:AbstractInterpreter} bb_vartables::Vector{Union{Nothing,VarTable}} insert_coverage::Bool end -function OptimizationState(sv::InferenceState, interp::AbstractInterpreter) - inlining = InliningState(sv, interp) +function OptimizationState(sv::InferenceState, interp::AbstractInterpreter, + opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) + inlining = InliningState(sv, interp, opt_cache) return OptimizationState(sv.linfo, sv.src, nothing, sv.stmt_info, sv.mod, sv.sptypes, sv.slottypes, inlining, sv.cfg, sv.unreachable, sv.bb_vartables, sv.insert_coverage) end -function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractInterpreter) +function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractInterpreter, + opt_cache::IdDict{MethodInstance,CodeInstance}=IdDict{MethodInstance,CodeInstance}()) # prepare src for running optimization passes if it isn't already nssavalues = src.ssavaluetypes if nssavalues isa Int @@ -205,7 +243,7 @@ function OptimizationState(mi::MethodInstance, src::CodeInfo, interp::AbstractIn mod = isa(def, Method) ? def.module : def # Allow using the global MI cache, but don't track edges. # This method is mostly used for unit testing the optimizer - inlining = InliningState(interp) + inlining = InliningState(interp, opt_cache) cfg = compute_basic_blocks(src.code) unreachable = BitSet() bb_vartables = Union{VarTable,Nothing}[] @@ -236,13 +274,29 @@ include("ssair/EscapeAnalysis.jl") include("ssair/passes.jl") include("ssair/irinterp.jl") +function ir_to_codeinf!(opt::OptimizationState, frame::InferenceState, edges::SimpleVector) + ir_to_codeinf!(opt, edges, compute_inlining_cost(frame.interp, frame.result, opt.optresult)) +end + +function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector, inlining_cost::InlineCostType) + src = ir_to_codeinf!(opt, edges) + src.inlining_cost = inlining_cost + src +end + +function ir_to_codeinf!(opt::OptimizationState, edges::SimpleVector) + src = ir_to_codeinf!(opt) + src.edges = edges + src +end + function ir_to_codeinf!(opt::OptimizationState) - (; linfo, src, result) = opt - if result === nothing + (; linfo, src, optresult) = opt + if optresult === nothing return src end - src = ir_to_codeinf!(src, result.ir) - opt.result = nothing + src = ir_to_codeinf!(src, optresult.ir) + opt.optresult = nothing opt.src = src maybe_validate_code(linfo, src, "optimized") return src @@ -485,63 +539,12 @@ end abstract_eval_ssavalue(s::SSAValue, src::Union{IRCode,IncrementalCompact}) = types(src)[s] """ - finish(interp::AbstractInterpreter, opt::OptimizationState, - ir::IRCode, caller::InferenceResult) + finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode) -Post-process information derived by Julia-level optimizations for later use. -In particular, this function determines the inlineability of the optimized code. +Called at the end of optimization to store the resulting IR back into the OptimizationState. """ -function finish(interp::AbstractInterpreter, opt::OptimizationState, - ir::IRCode, caller::InferenceResult) - (; src, linfo) = opt - (; def, specTypes) = linfo - - force_noinline = is_declared_noinline(src) - - # compute inlining and other related optimizations - result = caller.result - @assert !(result isa LimitedAccuracy) - result = widenslotwrapper(result) - - opt.result = OptimizationResult(ir, false) - - # determine and cache inlineability - if !force_noinline - sig = unwrap_unionall(specTypes) - if !(isa(sig, DataType) && sig.name === Tuple.name) - force_noinline = true - end - if !is_declared_inline(src) && result === Bottom - force_noinline = true - end - end - if force_noinline - set_inlineable!(src, false) - elseif isa(def, Method) - if is_declared_inline(src) && isdispatchtuple(specTypes) - # obey @inline declaration if a dispatch barrier would not help - set_inlineable!(src, true) - else - # compute the cost (size) of inlining this code - params = OptimizationParams(interp) - cost_threshold = default = params.inline_cost_threshold - if ⊑(optimizer_lattice(interp), result, Tuple) && !isconcretetype(widenconst(result)) - cost_threshold += params.inline_tupleret_bonus - end - # if the method is declared as `@inline`, increase the cost threshold 20x - if is_declared_inline(src) - cost_threshold += 19*default - end - # a few functions get special treatment - if def.module === _topmod(def.module) - name = def.name - if name === :iterate || name === :unsafe_convert || name === :cconvert - cost_threshold += 4*default - end - end - src.inlining_cost = inline_cost(ir, params, cost_threshold) - end - end +function finishopt!(interp::AbstractInterpreter, opt::OptimizationState, ir::IRCode) + opt.optresult = OptimizationResult(ir, ccall(:jl_ir_flag_inlining, UInt8, (Any,), opt.src), false) return nothing end @@ -717,7 +720,9 @@ function iscall_with_boundscheck(@nospecialize(stmt), sv::PostOptAnalysisState) f === nothing && return false if f === getfield nargs = 4 - elseif f === memoryrefnew || f === memoryrefget || f === memoryref_isassigned + elseif f === memoryrefnew + nargs= 3 + elseif f === memoryrefget || f === memoryref_isassigned nargs = 4 elseif f === memoryrefset! nargs = 5 @@ -1013,19 +1018,22 @@ end # run the optimization work function optimize(interp::AbstractInterpreter, opt::OptimizationState, caller::InferenceResult) - @timeit "optimizer" ir = run_passes_ipo_safe(opt.src, opt) + @zone "CC: OPTIMIZER" ir = run_passes_ipo_safe(opt.src, opt) ipo_dataflow_analysis!(interp, opt, ir, caller) - return finish(interp, opt, ir, caller) + finishopt!(interp, opt, ir) + return nothing end -macro pass(name, expr) +const ALL_PASS_NAMES = String[] +macro pass(name::String, expr) optimize_until = esc(:optimize_until) stage = esc(:__stage__) - macrocall = :(@timeit $(esc(name)) $(esc(expr))) + macrocall = :(@zone $name $(esc(expr))) macrocall.args[2] = __source__ # `@timeit` may want to use it + push!(ALL_PASS_NAMES, name) quote $macrocall - matchpass($optimize_until, ($stage += 1), $(esc(name))) && $(esc(:(@goto __done__))) + matchpass($optimize_until, ($stage += 1), $name) && $(esc(:(@goto __done__))) end end @@ -1036,24 +1044,29 @@ matchpass(::Nothing, _, _) = false function run_passes_ipo_safe( ci::CodeInfo, sv::OptimizationState, - optimize_until = nothing, # run all passes by default -) + optimize_until::Union{Nothing, Int, String} = nothing) # run all passes by default + if optimize_until isa String && !contains_is(ALL_PASS_NAMES, optimize_until) + error("invalid `optimize_until` argument, no such optimization pass") + elseif optimize_until isa Int && (optimize_until < 1 || optimize_until > length(ALL_PASS_NAMES)) + error("invalid `optimize_until` argument, no such optimization pass") + end + __stage__ = 0 # used by @pass # NOTE: The pass name MUST be unique for `optimize_until::String` to work - @pass "convert" ir = convert_to_ircode(ci, sv) - @pass "slot2reg" ir = slot2reg(ir, ci, sv) + @pass "CC: CONVERT" ir = convert_to_ircode(ci, sv) + @pass "CC: SLOT2REG" ir = slot2reg(ir, ci, sv) # TODO: Domsorting can produce an updated domtree - no need to recompute here - @pass "compact 1" ir = compact!(ir) - @pass "Inlining" ir = ssa_inlining_pass!(ir, sv.inlining, ci.propagate_inbounds) - # @timeit "verify 2" verify_ir(ir) - @pass "compact 2" ir = compact!(ir) - @pass "SROA" ir = sroa_pass!(ir, sv.inlining) - @pass "ADCE" (ir, made_changes) = adce_pass!(ir, sv.inlining) + @pass "CC: COMPACT_1" ir = compact!(ir) + @pass "CC: INLINING" ir = ssa_inlining_pass!(ir, sv.inlining, ci.propagate_inbounds) + # @zone "CC: VERIFY 2" verify_ir(ir) + @pass "CC: COMPACT_2" ir = compact!(ir) + @pass "CC: SROA" ir = sroa_pass!(ir, sv.inlining) + @pass "CC: ADCE" (ir, made_changes) = adce_pass!(ir, sv.inlining) if made_changes - @pass "compact 3" ir = compact!(ir, true) + @pass "CC: COMPACT_3" ir = compact!(ir, true) end if is_asserts() - @timeit "verify 3" begin + @zone "CC: VERIFY_3" begin verify_ir(ir, true, false, optimizer_lattice(sv.inlining.interp), sv.linfo) verify_linetable(ir.debuginfo, length(ir.stmts)) end @@ -1314,10 +1327,10 @@ end function slot2reg(ir::IRCode, ci::CodeInfo, sv::OptimizationState) # need `ci` for the slot metadata, IR for the code svdef = sv.linfo.def - @timeit "domtree 1" domtree = construct_domtree(ir) + @zone "CC: DOMTREE_1" domtree = construct_domtree(ir) defuse_insts = scan_slot_def_use(Int(ci.nargs), ci, ir.stmts.stmt) 𝕃ₒ = optimizer_lattice(sv.inlining.interp) - @timeit "construct_ssa" ir = construct_ssa!(ci, ir, sv, domtree, defuse_insts, 𝕃ₒ) # consumes `ir` + @zone "CC: CONSTRUCT_SSA" ir = construct_ssa!(ci, ir, sv, domtree, defuse_insts, 𝕃ₒ) # consumes `ir` # NOTE now we have converted `ir` to the SSA form and eliminated slots # let's resize `argtypes` now and remove unnecessary types for the eliminated slots resize!(ir.argtypes, ci.nargs) @@ -1459,7 +1472,7 @@ function statement_or_branch_cost(@nospecialize(stmt), line::Int, src::Union{Cod return thiscost end -function inline_cost(ir::IRCode, params::OptimizationParams, cost_threshold::Int) +function inline_cost_model(ir::IRCode, params::OptimizationParams, cost_threshold::Int) bodycost = 0 for i = 1:length(ir.stmts) stmt = ir[SSAValue(i)][:stmt] diff --git a/Compiler/src/ssair/inlining.jl b/Compiler/src/ssair/inlining.jl index 7b4ab34200764..251767d577157 100644 --- a/Compiler/src/ssair/inlining.jl +++ b/Compiler/src/ssair/inlining.jl @@ -73,10 +73,10 @@ add_inlining_edge!(et::InliningEdgeTracker, edge::MethodInstance) = add_inlining function ssa_inlining_pass!(ir::IRCode, state::InliningState, propagate_inbounds::Bool) # Go through the function, performing simple inlining (e.g. replacing call by constants # and analyzing legality of inlining). - @timeit "analysis" todo = assemble_inline_todo!(ir, state) + @zone "CC: ANALYSIS" todo = assemble_inline_todo!(ir, state) isempty(todo) && return ir # Do the actual inlining for every call we identified - @timeit "execution" ir = batch_inline!(ir, todo, propagate_inbounds, state.interp) + @zone "CC: EXECUTION" ir = batch_inline!(ir, todo, propagate_inbounds, state.interp) return ir end @@ -126,10 +126,11 @@ function cfg_inline_item!(ir::IRCode, idx::Int, todo::InliningTodo, state::CFGIn block = block_for_inst(ir, idx) inline_into_block!(state, block) - if !isempty(inlinee_cfg.blocks[1].preds) + if length(inlinee_cfg.blocks[1].preds) > 1 need_split_before = true + else + @assert inlinee_cfg.blocks[1].preds[1] == 0 end - last_block_idx = last(state.cfg.blocks[block].stmts) if false # TODO: ((idx+1) == last_block_idx && isa(ir[SSAValue(last_block_idx)], GotoNode)) need_split = false @@ -166,12 +167,18 @@ function cfg_inline_item!(ir::IRCode, idx::Int, todo::InliningTodo, state::CFGIn end new_block_range = (length(state.new_cfg_blocks)-length(inlinee_cfg.blocks)+1):length(state.new_cfg_blocks) - # Fixup the edges of the newely added blocks + # Fixup the edges of the newly added blocks for (old_block, new_block) in enumerate(bb_rename_range) if old_block != 1 || need_split_before p = state.new_cfg_blocks[new_block].preds let bb_rename_range = bb_rename_range map!(p, p) do old_pred_block + # the meaning of predecessor 0 depends on the block we encounter it: + # - in the first block, it represents the function entry and so needs to be re-mapped + if old_block == 1 && old_pred_block == 0 + return first(bb_rename_range) - 1 + end + # - elsewhere, it represents external control-flow from a caught exception which is un-affected by inlining return old_pred_block == 0 ? 0 : bb_rename_range[old_pred_block] end end @@ -186,10 +193,6 @@ function cfg_inline_item!(ir::IRCode, idx::Int, todo::InliningTodo, state::CFGIn end end - if need_split_before - push!(state.new_cfg_blocks[first(bb_rename_range)].preds, first(bb_rename_range)-1) - end - any_edges = false for (old_block, new_block) in enumerate(bb_rename_range) if (length(state.new_cfg_blocks[new_block].succs) == 0) @@ -399,7 +402,7 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector else bb_offset, post_bb_id = popfirst!(todo_bbs) # This implements the need_split_before flag above - need_split_before = !isempty(item.ir.cfg.blocks[1].preds) + need_split_before = length(item.ir.cfg.blocks[1].preds) > 1 if need_split_before finish_current_bb!(compact, 0) end @@ -976,7 +979,7 @@ function retrieve_ir_for_inlining(mi::MethodInstance, ir::IRCode, preserve_local return ir, spec_info, DebugInfo(ir.debuginfo, length(ir.stmts)) end function retrieve_ir_for_inlining(mi::MethodInstance, opt::OptimizationState, preserve_local_sources::Bool) - result = opt.result + result = opt.optresult if result !== nothing !result.simplified && simplify_ir!(result) return retrieve_ir_for_inlining(mi, result.ir, preserve_local_sources) @@ -1159,14 +1162,18 @@ function is_builtin(𝕃ₒ::AbstractLattice, s::Signature) end function handle_invoke_call!(todo::Vector{Pair{Int,Any}}, - ir::IRCode, idx::Int, stmt::Expr, info::InvokeCallInfo, flag::UInt32, + ir::IRCode, idx::Int, stmt::Expr, @nospecialize(info), flag::UInt32, sig::Signature, state::InliningState) - match = info.match + nspl = nsplit(info) + nspl == 0 && return nothing # e.g. InvokeCICallInfo + @assert nspl == 1 + mresult = getsplit(info, 1) + match = mresult.matches[1] if !match.fully_covers # TODO: We could union split out the signature check and continue on return nothing end - result = info.result + result = getresult(info, 1) if isa(result, ConcreteResult) item = concrete_result_item(result, info, state) elseif isa(result, SemiConcreteResult) @@ -1648,7 +1655,7 @@ function assemble_inline_todo!(ir::IRCode, state::InliningState) handle_opaque_closure_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, ModifyOpInfo) handle_modifyop!_call!(ir, idx, stmt, info, state) - elseif isa(info, InvokeCallInfo) + elseif sig.f === Core.invoke handle_invoke_call!(todo, ir, idx, stmt, info, flag, sig, state) elseif isa(info, FinalizerInfo) handle_finalizer_call!(ir, idx, stmt, info, state) diff --git a/Compiler/src/ssair/ir.jl b/Compiler/src/ssair/ir.jl index 8a317d2d8cc0d..e6a8ffe6d539e 100644 --- a/Compiler/src/ssair/ir.jl +++ b/Compiler/src/ssair/ir.jl @@ -105,6 +105,9 @@ function compute_basic_blocks(stmts::Vector{Any}) end # Compute successors/predecessors for (num, b) in enumerate(blocks) + if b.stmts.start == 1 + push!(b.preds, 0) # the entry block has a virtual predecessor + end terminator = stmts[last(b.stmts)] if isa(terminator, ReturnNode) # return never has any successors @@ -609,7 +612,7 @@ end elseif isa(stmt, EnterNode) op == 1 || throw(BoundsError()) stmt = EnterNode(stmt.catch_dest, v) - elseif isa(stmt, Union{AnySSAValue, GlobalRef}) + elseif isa(stmt, Union{AnySSAValue, Argument, GlobalRef}) op == 1 || throw(BoundsError()) stmt = v elseif isa(stmt, UpsilonNode) @@ -640,7 +643,7 @@ end function userefs(@nospecialize(x)) relevant = (isa(x, Expr) && is_relevant_expr(x)) || isa(x, GotoIfNot) || isa(x, ReturnNode) || isa(x, SSAValue) || isa(x, OldSSAValue) || isa(x, NewSSAValue) || - isa(x, PiNode) || isa(x, PhiNode) || isa(x, PhiCNode) || isa(x, UpsilonNode) || isa(x, EnterNode) + isa(x, PiNode) || isa(x, PhiNode) || isa(x, PhiCNode) || isa(x, UpsilonNode) || isa(x, EnterNode) || isa(x, Argument) return UseRefIterator(x, relevant) end @@ -810,7 +813,7 @@ end types(ir::Union{IRCode, IncrementalCompact}) = TypesView(ir) function getindex(compact::IncrementalCompact, ssa::SSAValue) - (1 ≤ ssa.id ≤ compact.result_idx) || throw(InvalidIRError()) + (1 ≤ ssa.id < compact.result_idx) || throw(InvalidIRError()) return compact.result[ssa.id] end diff --git a/Compiler/src/ssair/irinterp.jl b/Compiler/src/ssair/irinterp.jl index 084f28f0aa523..3d72da72625be 100644 --- a/Compiler/src/ssair/irinterp.jl +++ b/Compiler/src/ssair/irinterp.jl @@ -32,7 +32,7 @@ function concrete_eval_invoke(interp::AbstractInterpreter, ci::CodeInstance, arg end function abstract_eval_invoke_inst(interp::AbstractInterpreter, inst::Instruction, irsv::IRInterpretationState) - stmt = inst[:stmt] + stmt = inst[:stmt]::Expr ci = stmt.args[1] if ci isa MethodInstance world = frame_world(irsv) diff --git a/Compiler/src/ssair/passes.jl b/Compiler/src/ssair/passes.jl index 4251eecb23631..f16163554b75b 100644 --- a/Compiler/src/ssair/passes.jl +++ b/Compiler/src/ssair/passes.jl @@ -9,7 +9,9 @@ end function is_known_invoke_or_call(@nospecialize(x), @nospecialize(func), ir::Union{IRCode,IncrementalCompact}) isinvoke = isexpr(x, :invoke) (isinvoke || isexpr(x, :call)) || return false - ft = argextype(x.args[isinvoke ? 2 : 1], ir) + narg = isinvoke ? 2 : 1 + length(x.args) < narg && return false + ft = argextype(x.args[narg], ir) return singleton_type(ft) === func end @@ -183,7 +185,7 @@ function find_def_for_use( end function collect_leaves(compact::IncrementalCompact, @nospecialize(val), @nospecialize(typeconstraint), 𝕃ₒ::AbstractLattice, - predecessors = ((@nospecialize(def), compact::IncrementalCompact) -> isa(def, PhiNode) ? def.values : nothing)) + predecessors::Pre = ((@nospecialize(def), compact::IncrementalCompact) -> isa(def, PhiNode) ? def.values : nothing)) where {Pre} if isa(val, Union{OldSSAValue, SSAValue}) val, typeconstraint = simple_walk_constraint(compact, val, typeconstraint) end @@ -271,7 +273,7 @@ Starting at `val` walk use-def chains to get all the leaves feeding into this `v `predecessors(def, compact)` is a callback which should return the set of possible predecessors for a "phi-like" node (PhiNode or Core.ifelse) or `nothing` otherwise. """ -function walk_to_defs(compact::IncrementalCompact, @nospecialize(defssa), @nospecialize(typeconstraint), predecessors, 𝕃ₒ::AbstractLattice) +function walk_to_defs(compact::IncrementalCompact, @nospecialize(defssa), @nospecialize(typeconstraint), predecessors::Pre, 𝕃ₒ::AbstractLattice) where {Pre} visited_philikes = AnySSAValue[] isa(defssa, AnySSAValue) || return Any[defssa], visited_philikes def = compact[defssa][:stmt] diff --git a/Compiler/src/ssair/show.jl b/Compiler/src/ssair/show.jl index 0688c02eb6440..6731054ee8f30 100644 --- a/Compiler/src/ssair/show.jl +++ b/Compiler/src/ssair/show.jl @@ -12,7 +12,8 @@ using .Compiler: ALWAYS_FALSE, ALWAYS_TRUE, argextype, BasicBlock, block_for_ins CachedMethodTable, CFG, compute_basic_blocks, DebugInfoStream, Effects, EMPTY_SPTYPES, getdebugidx, IncrementalCompact, InferenceResult, InferenceState, InvalidIRError, IRCode, LimitedAccuracy, NativeInterpreter, scan_ssa_use!, - singleton_type, sptypes_from_meth_instance, StmtRange, Timings, VarState, widenconst + singleton_type, sptypes_from_meth_instance, StmtRange, Timings, VarState, widenconst, + get_ci_mi, get_ci_abi @nospecialize @@ -95,22 +96,39 @@ function print_stmt(io::IO, idx::Int, @nospecialize(stmt), code::Union{IRCode,Co elseif isexpr(stmt, :invoke) && length(stmt.args) >= 2 && isa(stmt.args[1], Union{MethodInstance,CodeInstance}) stmt = stmt::Expr # TODO: why is this here, and not in Base.show_unquoted - printstyled(io, " invoke "; color = :light_black) - mi = stmt.args[1] - if !(mi isa Core.MethodInstance) - mi = (mi::Core.CodeInstance).def - end - if isa(mi, Core.ABIOverride) - abi = mi.abi - mi = mi.def + ci = stmt.args[1] + if ci isa Core.CodeInstance + printstyled(io, " invoke "; color = :light_black) + mi = get_ci_mi(ci) + abi = get_ci_abi(ci) else - abi = mi.specTypes + printstyled(io, "dynamic invoke "; color = :yellow) + abi = (ci::Core.MethodInstance).specTypes end - show_unquoted(io, stmt.args[2], indent) - print(io, "(") # XXX: this is wrong if `sig` is not a concretetype method # more correct would be to use `fieldtype(sig, i)`, but that would obscure / discard Varargs information in show sig = abi == Tuple ? Core.svec() : Base.unwrap_unionall(abi).parameters::Core.SimpleVector + f = stmt.args[2] + ft = maybe_argextype(f, code, sptypes) + + # We can elide the type for arg0 if it... + skip_ftype = (length(sig) == 0) # doesn't exist... + skip_ftype = skip_ftype || ( + # ... or, f prints as a user-accessible value... + (f isa GlobalRef) && + # ... and matches the value of the singleton type of the invoked MethodInstance + (singleton_type(ft) === singleton_type(sig[1]) !== nothing) + ) + if skip_ftype + show_unquoted(io, f, indent) + else + print(io, "(") + show_unquoted(io, f, indent) + print(io, "::", sig[1], ")") + end + + # Print the remaining arguments (with type annotations from the invoked MethodInstance) + print(io, "(") print_arg(i) = sprint(; context=io) do io show_unquoted(io, stmt.args[i], indent) if (i - 1) <= length(sig) @@ -201,7 +219,7 @@ end end """ - Compute line number annotations for an IRCode + Compute line number annotations for an IRCode or CodeInfo. This functions compute three sets of annotations for each IR line. Take the following example (taken from `@code_typed sin(1.0)`): @@ -260,7 +278,7 @@ to catch up and print the intermediate scopes. Which scope is printed is indicat by the indentation of the method name and by an increased thickness of the appropriate line for the scope. """ -function compute_ir_line_annotations(code::IRCode) +function compute_ir_line_annotations(code::Union{IRCode,CodeInfo}) loc_annotations = String[] loc_methods = String[] loc_lineno = String[] @@ -270,7 +288,8 @@ function compute_ir_line_annotations(code::IRCode) last_printed_depth = 0 debuginfo = code.debuginfo def = :var"unknown scope" - for idx in 1:length(code.stmts) + n = isa(code, IRCode) ? length(code.stmts) : length(code.code) + for idx in 1:n buf = IOBuffer() print(buf, "│") stack = buildLineInfoNode(debuginfo, def, idx) @@ -329,7 +348,7 @@ function compute_ir_line_annotations(code::IRCode) loc_method = string(" "^printing_depth, loc_method) last_stack = stack end - push!(loc_annotations, String(take!(buf))) + push!(loc_annotations, takestring!(buf)) push!(loc_lineno, (lineno != 0 && lineno != last_lineno) ? string(lineno) : "") push!(loc_methods, loc_method) (lineno != 0) && (last_lineno = lineno) @@ -834,7 +853,7 @@ function new_nodes_iter(compact::IncrementalCompact) end # print only line numbers on the left, some of the method names and nesting depth on the right -function inline_linfo_printer(code::IRCode) +function inline_linfo_printer(code::Union{IRCode,CodeInfo}) loc_annotations, loc_methods, loc_lineno = compute_ir_line_annotations(code) max_loc_width = maximum(length, loc_annotations) max_lineno_width = maximum(length, loc_lineno) @@ -903,12 +922,15 @@ function stmts_used(::IO, code::CodeInfo) return used end -function default_config(code::IRCode; verbose_linetable=false) - return IRShowConfig(verbose_linetable ? statementidx_lineinfo_printer(code) - : inline_linfo_printer(code); - bb_color=:normal) +function default_config(code::IRCode; debuginfo = :source_inline) + return IRShowConfig(get_debuginfo_printer(code, debuginfo); bb_color=:normal) +end +default_config(code::CodeInfo; debuginfo = :source) = IRShowConfig(get_debuginfo_printer(code, debuginfo)) +function default_config(io::IO, src) + debuginfo = get(io, :debuginfo, nothing) + debuginfo !== nothing && return default_config(src; debuginfo) + return default_config(src) end -default_config(code::CodeInfo) = IRShowConfig(statementidx_lineinfo_printer(code)) function show_ir_stmts(io::IO, ir::Union{IRCode, CodeInfo, IncrementalCompact}, inds, config::IRShowConfig, sptypes::Vector{VarState}, used::BitSet, cfg::CFG, bb_idx::Int; pop_new_node! = Returns(nothing)) @@ -928,8 +950,7 @@ function finish_show_ir(io::IO, cfg::CFG, config::IRShowConfig) return nothing end -function show_ir(io::IO, ir::IRCode, config::IRShowConfig=default_config(ir); - pop_new_node! = new_nodes_iter(ir)) +function show_ir(io::IO, ir::IRCode, config::IRShowConfig=default_config(io, ir); pop_new_node! = new_nodes_iter(ir)) used = stmts_used(io, ir) cfg = ir.cfg maxssaid = length(ir.stmts) + length(ir.new_nodes) @@ -939,7 +960,7 @@ function show_ir(io::IO, ir::IRCode, config::IRShowConfig=default_config(ir); finish_show_ir(io, cfg, config) end -function show_ir(io::IO, ci::CodeInfo, config::IRShowConfig=default_config(ci); +function show_ir(io::IO, ci::CodeInfo, config::IRShowConfig=default_config(io, ci); pop_new_node! = Returns(nothing)) used = stmts_used(io, ci) cfg = compute_basic_blocks(ci.code) @@ -953,7 +974,7 @@ function show_ir(io::IO, ci::CodeInfo, config::IRShowConfig=default_config(ci); finish_show_ir(io, cfg, config) end -function show_ir(io::IO, compact::IncrementalCompact, config::IRShowConfig=default_config(compact.ir)) +function show_ir(io::IO, compact::IncrementalCompact, config::IRShowConfig=default_config(io, compact.ir)) cfg = compact.ir.cfg @@ -1155,3 +1176,35 @@ const __debuginfo = Dict{Symbol, Any}( ) const default_debuginfo = Ref{Symbol}(:none) debuginfo(sym) = sym === :default ? default_debuginfo[] : sym + +const __debuginfo = Dict{Symbol, Any}( + # :full => src -> statementidx_lineinfo_printer(src), # and add variable slot information + :source => src -> statementidx_lineinfo_printer(src), + :source_inline => src -> inline_linfo_printer(src), + # :oneliner => src -> statementidx_lineinfo_printer(PartialLineInfoPrinter, src), + :none => src -> lineinfo_disabled, + ) + +const debuginfo_modes = [:none, :source, :source_inline] +@assert Set(debuginfo_modes) == Set(keys(__debuginfo)) + +function validate_debuginfo_mode(mode::Symbol) + in(mode, debuginfo_modes) && return true + throw(ArgumentError("`debuginfo` must be one of the following: $(join([repr(mode) for mode in debuginfo_modes], ", "))")) +end + +const default_debuginfo_mode = Ref{Symbol}(:none) +function expand_debuginfo_mode(mode::Symbol, default = default_debuginfo_mode[]) + if mode === :default + mode = default + end + validate_debuginfo_mode(mode) + return mode +end + +function get_debuginfo_printer(mode::Symbol) + mode = expand_debuginfo_mode(mode) + return __debuginfo[mode] +end + +get_debuginfo_printer(src, mode::Symbol) = get_debuginfo_printer(mode)(src) diff --git a/Compiler/src/ssair/slot2ssa.jl b/Compiler/src/ssair/slot2ssa.jl index e0f3e207789a3..16a964b4d72f1 100644 --- a/Compiler/src/ssair/slot2ssa.jl +++ b/Compiler/src/ssair/slot2ssa.jl @@ -8,7 +8,7 @@ end SlotInfo() = SlotInfo(Int[], Int[], false) function scan_entry!(result::Vector{SlotInfo}, idx::Int, @nospecialize(stmt)) - # NewVarNodes count as defs for the purpose + # NewvarNodes count as defs for the purpose # of liveness analysis (i.e. they kill use chains) if isa(stmt, NewvarNode) result[slot_id(stmt.slot)].any_newvar = true @@ -574,7 +574,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, for (; leave_block) in catch_entry_blocks new_phic_nodes[leave_block] = NewPhiCNode2[] end - @timeit "idf" for (idx, slot) in Iterators.enumerate(defuses) + @zone "CC: IDF" for (idx, slot) in Iterators.enumerate(defuses) # No uses => no need for phi nodes isempty(slot.uses) && continue # TODO: Restore this optimization @@ -600,7 +600,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, continue end - @timeit "liveness" (live = compute_live_ins(cfg, slot)) + @zone "CC: LIVENESS" (live = compute_live_ins(cfg, slot)) for li in live.live_in_bbs push!(live_slots[li], idx) cidx = findfirst(x::TryCatchRegion->x.leave_block==li, catch_entry_blocks) @@ -671,7 +671,7 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, worklist = Tuple{Int, Int, Vector{Pair{Any, Any}}}[(1, 0, initial_incoming_vals)] visited = BitSet() new_nodes = ir.new_nodes - @timeit "SSA Rename" while !isempty(worklist) + @zone "CC: SSA_RENAME" while !isempty(worklist) (item, pred, incoming_vals) = pop!(worklist) if sv.bb_vartables[item] === nothing continue @@ -891,6 +891,6 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState, local node = new_nodes.stmts[i] node[:stmt] = new_to_regular(renumber_ssa!(node[:stmt], ssavalmap), nstmts) end - @timeit "domsort" ir = domsort_ssa!(ir, domtree) + @zone "CC: DOMSORT" ir = domsort_ssa!(ir, domtree) return ir end diff --git a/Compiler/src/ssair/verify.jl b/Compiler/src/ssair/verify.jl index 2b8f89173911a..3aa1e00e3f2d3 100644 --- a/Compiler/src/ssair/verify.jl +++ b/Compiler/src/ssair/verify.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -irshow_was_loaded() = invokelatest(isdefined, Compiler.IRShow, :debuginfo_firstline) +irshow_was_loaded() = invokelatest(isdefinedglobal, Compiler.IRShow, :debuginfo_firstline) function maybe_show_ir(ir::IRCode) if irshow_was_loaded() # ensure we use I/O that does not yield, as this gets called during compilation @@ -32,6 +32,10 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int, if isa(op, SSAValue) op.id > 0 || @verify_error "Def ($(op.id)) is invalid in final IR" if op.id > length(ir.stmts) + if op.id - length(ir.stmts) > length(ir.new_nodes.info) + @verify_error "Def ($(op.id)) points to non-existent new node" + raise_error() + end def_bb = block_for_inst(ir.cfg, ir.new_nodes.info[op.id - length(ir.stmts)].pos) else def_bb = block_for_inst(ir.cfg, op.id) diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index 6a85bc6605d3f..525fe0cc222b7 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -47,21 +47,20 @@ end add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo) = _add_edges_impl(edges, info) function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Bool=false) if !fully_covering(info) - # add legacy-style missing backedge info also exists = false for i in 2:length(edges) - if edges[i] === info.mt && edges[i-1] == info.atype + if edges[i] === Core.methodtable && edges[i-1] == info.atype exists = true break end end if !exists push!(edges, info.atype) - push!(edges, info.mt) + push!(edges, Core.methodtable) end end nmatches = length(info.results) - if nmatches == length(info.edges) == 1 + if nmatches == length(info.edges) == 1 && fully_covering(info) # try the optimized format for the representation, if possible and applicable # if this doesn't succeed, the backedge will be less precise, # but the forward edge will maintain the precision @@ -79,13 +78,15 @@ function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Boo end end # add check for whether this lookup already existed in the edges list + # encode nmatches as negative if fully_covers is false + encoded_nmatches = fully_covering(info) ? nmatches : -nmatches for i in 1:length(edges) - if edges[i] === nmatches && edges[i+1] == info.atype + if edges[i] === encoded_nmatches && edges[i+1] == info.atype # TODO: must also verify the CodeInstance match too return nothing end end - push!(edges, nmatches, info.atype) + push!(edges, encoded_nmatches, info.atype) for i = 1:nmatches edge = info.edges[i] m = info.results[i] @@ -102,7 +103,7 @@ function add_one_edge!(edges::Vector{Any}, edge::MethodInstance) i = 1 while i <= length(edges) edgeᵢ = edges[i] - edgeᵢ isa Int && (i += 2 + edgeᵢ; continue) + edgeᵢ isa Int && (i += 2 + abs(edgeᵢ); continue) edgeᵢ isa CodeInstance && (edgeᵢ = get_ci_mi(edgeᵢ)) edgeᵢ isa MethodInstance || (i += 1; continue) if edgeᵢ === edge && !(i > 1 && edges[i-1] isa Type) @@ -117,7 +118,7 @@ function add_one_edge!(edges::Vector{Any}, edge::CodeInstance) i = 1 while i <= length(edges) edgeᵢ_orig = edgeᵢ = edges[i] - edgeᵢ isa Int && (i += 2 + edgeᵢ; continue) + edgeᵢ isa Int && (i += 2 + abs(edgeᵢ); continue) edgeᵢ isa CodeInstance && (edgeᵢ = get_ci_mi(edgeᵢ)) edgeᵢ isa MethodInstance || (i += 1; continue) if edgeᵢ === edge.def && !(i > 1 && edges[i-1] isa Type) @@ -278,6 +279,7 @@ struct InvokeCICallInfo <: CallInfo end add_edges_impl(edges::Vector{Any}, info::InvokeCICallInfo) = add_inlining_edge!(edges, info.edge) +nsplit_impl(info::InvokeCICallInfo) = 0 """ info::InvokeCallInfo @@ -390,6 +392,11 @@ function add_inlining_edge!(edges::Vector{Any}, edge::CodeInstance) nothing end +nsplit_impl(info::InvokeCallInfo) = 1 +getsplit_impl(info::InvokeCallInfo, idx::Int) = (@assert idx == 1; MethodLookupResult(Core.MethodMatch[info.match], + WorldRange(typemin(UInt), typemax(UInt)), false)) +getresult_impl(info::InvokeCallInfo, idx::Int) = (@assert idx == 1; info.result) + """ info::OpaqueClosureCallInfo diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index e987ad3fd2b27..71719d75144b3 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -405,6 +405,9 @@ end return isdefined_tfunc(𝕃, arg1, sym) end @nospecs function isdefined_tfunc(𝕃::AbstractLattice, arg1, sym) + if arg1 isa MustAlias + arg1 = widenmustalias(arg1) + end arg1t = arg1 isa Const ? typeof(arg1.val) : isconstType(arg1) ? typeof(arg1.parameters[1]) : widenconst(arg1) a1 = unwrap_unionall(arg1t) if isa(a1, DataType) && !isabstracttype(a1) @@ -450,6 +453,10 @@ end return Const(true) end end + # datatype_fieldcount is what `fieldcount` uses internally + # and returns nothing (!==0) for non-definite field counts. + elseif datatype_fieldcount(a1) === 0 + return Const(false) end elseif isa(a1, Union) # Results can only be `Const` or `Bool` @@ -573,6 +580,16 @@ end add_tfunc(nfields, 1, 1, nfields_tfunc, 1) add_tfunc(Core._expr, 1, INT_INF, @nospecs((𝕃::AbstractLattice, args...)->Expr), 100) add_tfunc(svec, 0, INT_INF, @nospecs((𝕃::AbstractLattice, args...)->SimpleVector), 20) +@nospecs function _svec_ref_tfunc(𝕃::AbstractLattice, s, i) + if isa(s, Const) && isa(i, Const) + s, i = s.val, i.val + if isa(s, SimpleVector) && isa(i, Int) + return 1 ≤ i ≤ length(s) ? Const(s[i]) : Bottom + end + end + return Any +end +add_tfunc(Core._svec_ref, 2, 2, _svec_ref_tfunc, 1) @nospecs function typevar_tfunc(𝕃::AbstractLattice, n, lb_arg, ub_arg) lb = Union{} ub = Any @@ -2079,6 +2096,7 @@ end @nospecs function memoryref_tfunc(𝕃::AbstractLattice, ref, idx, boundscheck) memoryref_builtin_common_errorcheck(ref, Const(:not_atomic), boundscheck) || return Bottom hasintersect(widenconst(idx), Int) || return Bottom + hasintersect(widenconst(ref), GenericMemory) && return memoryref_tfunc(𝕃, ref) return ref end add_tfunc(memoryrefnew, 1, 3, memoryref_tfunc, 1) @@ -2090,7 +2108,7 @@ end add_tfunc(memoryrefoffset, 1, 1, memoryrefoffset_tfunc, 5) @nospecs function memoryref_builtin_common_errorcheck(mem, order, boundscheck) - hasintersect(widenconst(mem), GenericMemoryRef) || return false + hasintersect(widenconst(mem), Union{GenericMemory, GenericMemoryRef}) || return false hasintersect(widenconst(order), Symbol) || return false hasintersect(widenconst(unwrapva(boundscheck)), Bool) || return false return true @@ -2186,7 +2204,7 @@ function memoryref_builtin_common_nothrow(argtypes::Vector{Any}) idx = widenconst(argtypes[2]) idx ⊑ Int || return false boundscheck ⊑ Bool || return false - memtype ⊑ GenericMemoryRef || return false + memtype ⊑ Union{GenericMemory, GenericMemoryRef} || return false # If we have @inbounds (last argument is false), we're allowed to assume # we don't throw bounds errors. if isa(boundscheck, Const) @@ -2316,6 +2334,9 @@ function _builtin_nothrow(𝕃::AbstractLattice, @nospecialize(f::Builtin), argt elseif f === Core.compilerbarrier na == 2 || return false return compilerbarrier_nothrow(argtypes[1], nothing) + elseif f === Core._svec_ref + na == 2 || return false + return _svec_ref_tfunc(𝕃, argtypes[1], argtypes[2]) isa Const end return false end @@ -2346,7 +2367,9 @@ const _CONSISTENT_BUILTINS = Any[ throw, Core.throw_methoderror, setfield!, - donotdelete + donotdelete, + memoryrefnew, + memoryrefoffset, ] # known to be effect-free (but not necessarily nothrow) @@ -2371,6 +2394,7 @@ const _EFFECT_FREE_BUILTINS = [ Core.throw_methoderror, getglobal, compilerbarrier, + Core._svec_ref, ] const _INACCESSIBLEMEM_BUILTINS = Any[ @@ -2404,6 +2428,7 @@ const _ARGMEM_BUILTINS = Any[ replacefield!, setfield!, swapfield!, + Core._svec_ref, ] const _INCONSISTENT_INTRINSICS = Any[ @@ -2546,7 +2571,7 @@ const _EFFECTS_KNOWN_BUILTINS = Any[ # Core._primitivetype, # Core._setsuper!, # Core._structtype, - # Core._svec_ref, + Core._svec_ref, # Core._typebody!, Core._typevar, apply_type, @@ -2650,9 +2675,7 @@ function builtin_effects(𝕃::AbstractLattice, @nospecialize(f::Builtin), argty else if contains_is(_CONSISTENT_BUILTINS, f) consistent = ALWAYS_TRUE - elseif f === memoryrefnew || f === memoryrefoffset - consistent = ALWAYS_TRUE - elseif f === memoryrefget || f === memoryrefset! || f === memoryref_isassigned + elseif f === memoryrefget || f === memoryrefset! || f === memoryref_isassigned || f === Core._svec_ref consistent = CONSISTENT_IF_INACCESSIBLEMEMONLY elseif f === Core._typevar || f === Core.memorynew consistent = CONSISTENT_IF_NOTRETURNED @@ -2838,6 +2861,8 @@ _istypemin(@nospecialize x) = !_iszero(x) && Intrinsics.neg_int(x) === x function builtin_exct(𝕃::AbstractLattice, @nospecialize(f::Builtin), argtypes::Vector{Any}, @nospecialize(rt)) if isa(f, IntrinsicFunction) return intrinsic_exct(𝕃, f, argtypes) + elseif f === Core._svec_ref + return BoundsError end return Any end @@ -3178,15 +3203,12 @@ function _hasmethod_tfunc(interp::AbstractInterpreter, argtypes::Vector{Any}, sv isdispatchelem(ft) || return CallMeta(Bool, Any, Effects(), NoCallInfo()) # check that we might not have a subtype of `ft` at runtime, before doing supertype lookup below types = rewrap_unionall(Tuple{ft, unwrapped.parameters...}, types)::Type end - mt = ccall(:jl_method_table_for, Any, (Any,), types) - if !isa(mt, MethodTable) - return CallMeta(Bool, Any, EFFECTS_THROWS, NoCallInfo()) - end match, valid_worlds = findsup(types, method_table(interp)) update_valid_age!(sv, valid_worlds) if match === nothing rt = Const(false) vresults = MethodLookupResult(Any[], valid_worlds, true) + mt = Core.methodtable vinfo = MethodMatchInfo(vresults, mt, types, false) # XXX: this should actually be an info with invoke-type edge else rt = Const(true) diff --git a/Compiler/src/timing.jl b/Compiler/src/timing.jl new file mode 100644 index 0000000000000..8b8240b025a66 --- /dev/null +++ b/Compiler/src/timing.jl @@ -0,0 +1,38 @@ +if ccall(:jl_timing_enabled, Cint, ()) != 0 + function getzonedexpr(name::Union{Symbol, String}, ex::Expr, func::Symbol, file::Symbol, line::Integer, color::Integer) + event = RefValue{Ptr{Cvoid}}(C_NULL) + name = QuoteNode(Symbol(name)) + func = QuoteNode(func) + file = QuoteNode(file) + + # XXX: This buffer must be large enough to store any jl_timing_block_t (runtime-checked) + buffer = (0, 0, 0, 0, 0, 0, 0) + buffer_size = Core.sizeof(buffer) + return quote + if $event[] === C_NULL + $event[] = ccall(:_jl_timing_event_create, Ptr{Cvoid}, + (Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}, Ptr{UInt8}, Cint, Cint), + :CORE_COMPILER, $name, $func, $file, $line, $color) + end + timing_block = RefValue($buffer) + block_ptr = pointer_from_objref(timing_block) + $(Expr(:gc_preserve, quote + ccall(:_jl_timing_block_init, Cvoid, (Ptr{Cvoid}, Csize_t, Ptr{Cvoid}), block_ptr, $buffer_size, $event[]) + ccall(:_jl_timing_block_start, Cvoid, (Ptr{Cvoid},), block_ptr) + $(Expr(:tryfinally, + :($(Expr(:escape, ex))), + quote + ccall(:_jl_timing_block_end, Cvoid, (Ptr{Cvoid},), block_ptr) + end + )) + end, :timing_block)) + end + end + macro zone(name, ex::Expr) + return getzonedexpr(name, ex, :unknown_julia_function, __source__.file, __source__.line, 0) + end +else + macro zone(name, ex::Expr) + return esc(ex) + end +end diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 9a5acece139a3..9849d22a9ce67 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -104,9 +104,13 @@ end function finish!(interp::AbstractInterpreter, caller::InferenceState, validation_world::UInt, time_before::UInt64) result = caller.result #@assert last(result.valid_worlds) <= get_world_counter() || isempty(caller.edges) - if isdefined(result, :ci) + if caller.cache_mode === CACHE_MODE_LOCAL + @assert !isdefined(result, :ci) + result.src = transform_result_for_local_cache(interp, result) + elseif isdefined(result, :ci) edges = result_edges(interp, caller) ci = result.ci + mi = result.linfo # if we aren't cached, we don't need this edge # but our caller might, so let's just make it anyways if last(result.valid_worlds) >= validation_world @@ -115,48 +119,65 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation store_backedges(ci, edges) end inferred_result = nothing - uncompressed = inferred_result + uncompressed = result.src const_flag = is_result_constabi_eligible(result) + debuginfo = nothing discard_src = caller.cache_mode === CACHE_MODE_NULL || const_flag if !discard_src inferred_result = transform_result_for_cache(interp, result, edges) + if inferred_result !== nothing + uncompressed = inferred_result + debuginfo = get_debuginfo(inferred_result) + # Inlining may fast-path the global cache via `VolatileInferenceResult`, so store it back here + result.src = inferred_result + else + if isa(result.src, OptimizationState) + debuginfo = get_debuginfo(ir_to_codeinf!(result.src)) + elseif isa(result.src, CodeInfo) + debuginfo = get_debuginfo(result.src) + end + end # TODO: do we want to augment edges here with any :invoke targets that we got from inlining (such that we didn't have a direct edge to it already)? if inferred_result isa CodeInfo - result.src = inferred_result if may_compress(interp) nslots = length(inferred_result.slotflags) resize!(inferred_result.slottypes::Vector{Any}, nslots) resize!(inferred_result.slotnames, nslots) end - di = inferred_result.debuginfo - uncompressed = inferred_result - inferred_result = maybe_compress_codeinfo(interp, result.linfo, inferred_result) + inferred_result = maybe_compress_codeinfo(interp, mi, inferred_result) result.is_src_volatile = false elseif ci.owner === nothing # The global cache can only handle objects that codegen understands inferred_result = nothing end end - if !@isdefined di - di = DebugInfo(result.linfo) + if debuginfo === nothing + debuginfo = DebugInfo(mi) end + min_world, max_world = first(result.valid_worlds), last(result.valid_worlds) + ipo_effects = encode_effects(result.ipo_effects) time_now = _time_ns() time_self_ns = caller.time_self_ns + (time_now - time_before) time_total = (time_now - caller.time_start - caller.time_paused) * 1e-9 ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), - ci, inferred_result, const_flag, first(result.valid_worlds), last(result.valid_worlds), encode_effects(result.ipo_effects), - result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, di, edges) + ci, inferred_result, const_flag, min_world, max_world, ipo_effects, + result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, debuginfo, edges) + if is_cached(caller) # CACHE_MODE_GLOBAL + cache_result!(interp, result, ci) + end engine_reject(interp, ci) codegen = codegen_cache(interp) - if !discard_src && codegen !== nothing && uncompressed isa CodeInfo + if !discard_src && codegen !== nothing && (isa(uncompressed, CodeInfo) || isa(uncompressed, OptimizationState)) + if isa(uncompressed, OptimizationState) + uncompressed = ir_to_codeinf!(uncompressed, edges) + end # record that the caller could use this result to generate code when required, if desired, to avoid repeating n^2 work codegen[ci] = uncompressed if bootstrapping_compiler && inferred_result == nothing # This is necessary to get decent bootstrapping performance # when compiling the compiler to inject everything eagerly # where codegen can start finding and using it right away - mi = result.linfo - if mi.def isa Method && isa_compileable_sig(mi) + if mi.def isa Method && isa_compileable_sig(mi) && is_cached(caller) ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), ci, uncompressed) end end @@ -165,6 +186,11 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation return nothing end +function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) + mi = result.linfo + code_cache(interp)[mi] = ci +end + function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstance, src::CodeInfo) user_edges = src.edges edges = user_edges isa SimpleVector ? user_edges : user_edges === nothing ? Core.svec() : Core.svec(user_edges...) @@ -199,11 +225,13 @@ function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstan end function finish_nocycle(::AbstractInterpreter, frame::InferenceState, time_before::UInt64) - finishinfer!(frame, frame.interp, frame.cycleid) + opt_cache = IdDict{MethodInstance,CodeInstance}() + finishinfer!(frame, frame.interp, frame.cycleid, opt_cache) opt = frame.result.src if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` optimize(frame.interp, opt, frame.result) end + empty!(opt_cache) validation_world = get_world_counter() finish!(frame.interp, frame, validation_world, time_before) if isdefined(frame.result, :ci) @@ -233,10 +261,11 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei cycle_valid_worlds = intersect(cycle_valid_worlds, caller.world.valid_worlds) cycle_valid_effects = merge_effects(cycle_valid_effects, caller.ipo_effects) end + opt_cache = IdDict{MethodInstance,CodeInstance}() for frameid = cycleid:length(frames) caller = frames[frameid]::InferenceState adjust_cycle_frame!(caller, cycle_valid_worlds, cycle_valid_effects) - finishinfer!(caller, caller.interp, cycleid) + finishinfer!(caller, caller.interp, cycleid, opt_cache) time_now = _time_ns() caller.time_self_ns += (time_now - time_before) time_before = time_now @@ -257,6 +286,7 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei caller.time_paused = UInt64(0) caller.time_caches = 0.0 end + empty!(opt_cache) cycletop = frames[cycleid]::InferenceState time_start = cycletop.time_start validation_world = get_world_counter() @@ -299,52 +329,123 @@ function adjust_cycle_frame!(sv::InferenceState, cycle_valid_worlds::WorldRange, return nothing end +function get_debuginfo(src) + isa(src, CodeInfo) && return src.debuginfo + isa(src, OptimizationState) && return src.src.debuginfo + return nothing +end + function is_result_constabi_eligible(result::InferenceResult) result_type = result.result return isa(result_type, Const) && is_foldable_nothrow(result.ipo_effects) && is_inlineable_constant(result_type.val) end -function transform_result_for_cache(::AbstractInterpreter, result::InferenceResult, edges::SimpleVector) +function compute_inlining_cost(interp::AbstractInterpreter, result::InferenceResult) + src = result.src + isa(src, OptimizationState) || return MAX_INLINE_COST + compute_inlining_cost(interp, result, src.optresult) +end + +function compute_inlining_cost(interp::AbstractInterpreter, result::InferenceResult, optresult#=::OptimizationResult=#) + return inline_cost_model(interp, result, optresult.inline_flag, optresult.ir) +end + +function inline_cost_model(interp::AbstractInterpreter, result::InferenceResult, + inline_flag::UInt8, ir::IRCode) + + inline_flag === SRC_FLAG_DECLARED_NOINLINE && return MAX_INLINE_COST + + mi = result.linfo + (; def, specTypes) = mi + if !isa(def, Method) + return MAX_INLINE_COST + end + + declared_inline = inline_flag === SRC_FLAG_DECLARED_INLINE + + rt = result.result + @assert !(rt isa LimitedAccuracy) + rt = widenslotwrapper(rt) + + sig = unwrap_unionall(specTypes) + if !(isa(sig, DataType) && sig.name === Tuple.name) + return MAX_INLINE_COST + end + if !declared_inline && rt === Bottom + return MAX_INLINE_COST + end + + if declared_inline && isdispatchtuple(specTypes) + # obey @inline declaration if a dispatch barrier would not help + return MIN_INLINE_COST + else + # compute the cost (size) of inlining this code + params = OptimizationParams(interp) + cost_threshold = default = params.inline_cost_threshold + if ⊑(optimizer_lattice(interp), rt, Tuple) && !isconcretetype(widenconst(rt)) + cost_threshold += params.inline_tupleret_bonus + end + # if the method is declared as `@inline`, increase the cost threshold 20x + if declared_inline + cost_threshold += 19*default + end + # a few functions get special treatment + if def.module === _topmod(def.module) + name = def.name + if name === :iterate || name === :unsafe_convert || name === :cconvert + cost_threshold += 4*default + end + end + return inline_cost_model(ir, params, cost_threshold) + end +end + +function transform_result_for_local_cache(interp::AbstractInterpreter, result::InferenceResult) + if is_result_constabi_eligible(result) + return nothing + end src = result.src if isa(src, OptimizationState) - src = ir_to_codeinf!(src) + # Compute and store any information required to determine the inlineability of the callee. + opt = src + opt.src.inlining_cost = compute_inlining_cost(interp, result) + end + return src +end + +function transform_result_for_cache(interp::AbstractInterpreter, result::InferenceResult, edges::SimpleVector) + inlining_cost = nothing + src = result.src + if isa(src, OptimizationState) + opt = src + inlining_cost = compute_inlining_cost(interp, result, opt.optresult) + discard_optimized_result(interp, opt, inlining_cost) && return nothing + src = ir_to_codeinf!(opt) end if isa(src, CodeInfo) src.edges = edges + if inlining_cost !== nothing + src.inlining_cost = inlining_cost + elseif may_optimize(interp) + src.inlining_cost = compute_inlining_cost(interp, result) + end end return src end +function discard_optimized_result(interp::AbstractInterpreter, opt#=::OptimizationState=#, inlining_cost#=::InlineCostType=#) + may_discard_trees(interp) || return false + return inlining_cost == MAX_INLINE_COST +end + function maybe_compress_codeinfo(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInfo) def = mi.def isa(def, Method) || return ci # don't compress toplevel code can_discard_trees = may_discard_trees(interp) cache_the_tree = !can_discard_trees || is_inlineable(ci) - if cache_the_tree - if may_compress(interp) - return ccall(:jl_compress_ir, String, (Any, Any), def, ci) - else - return ci - end - else - return nothing - end -end - -function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) - @assert isdefined(ci, :inferred) - # check if the existing linfo metadata is also sufficient to describe the current inference result - # to decide if it is worth caching this right now - mi = result.linfo - cache = WorldView(code_cache(interp), result.valid_worlds) - if haskey(cache, mi) - ci = cache[mi] - # n.b.: accurate edge representation might cause the CodeInstance for this to be constructed later - @assert isdefined(ci, :inferred) - return false - end - code_cache(interp)[mi] = ci - return true + cache_the_tree || return nothing + may_compress(interp) && return ccall(:jl_compress_ir, String, (Any, Any), def, ci) + return ci end function cycle_fix_limited(@nospecialize(typ), sv::InferenceState, cycleid::Int) @@ -469,7 +570,8 @@ const empty_edges = Core.svec() # inference completed on `me` # update the MethodInstance -function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid::Int) +function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid::Int, + opt_cache::IdDict{MethodInstance, CodeInstance}) # prepare to run optimization passes on fulltree @assert isempty(me.ip) # inspect whether our inference had a limited result accuracy, @@ -503,18 +605,17 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: istoplevel = !(me.linfo.def isa Method) istoplevel || compute_edges!(me) # don't add backedges to toplevel method instance - if limited_ret - # a parent may be cached still, but not this intermediate work: - # we can throw everything else away now + if limited_ret || limited_src + # A parent may be cached still, but not this intermediate work: + # we can throw everything else away now. Caching anything can confuse later + # heuristics to consider it worth trying to pursue compiling this further and + # finding infinite work as a result. Avoiding caching helps to ensure there is only + # a finite amount of work that can be discovered later (although potentially still a + # large multiplier on it). result.src = nothing result.tombstone = true me.cache_mode = CACHE_MODE_NULL set_inlineable!(me.src, false) - elseif limited_src - # a type result will be cached still, but not this intermediate work: - # we can throw everything else away now - result.src = nothing - set_inlineable!(me.src, false) else # annotate fulltree with type information, # either because we are the outermost code, or we might use this later @@ -526,7 +627,7 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: # disable optimization if we've already obtained very accurate result !result_is_constabi(interp, result) if doopt - result.src = OptimizationState(me, interp) + result.src = OptimizationState(me, interp, opt_cache) else result.src = me.src # for reflection etc. end @@ -561,29 +662,51 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid:: rettype_const = nothing const_flags = 0x0 end + di = nothing edges = empty_edges # `edges` will be updated within `finish!` ci = result.ci + min_world, max_world = first(result.valid_worlds), last(result.valid_worlds) ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any), ci, widenconst(result_type), widenconst(result.exc_result), rettype_const, const_flags, - first(result.valid_worlds), last(result.valid_worlds), + min_world, max_world, encode_effects(result.ipo_effects), result.analysis_results, di, edges) if is_cached(me) # CACHE_MODE_GLOBAL - cached_result = cache_result!(me.interp, result, ci) - if !cached_result + already_cached = is_already_cached(me.interp, result, ci) + if already_cached me.cache_mode = CACHE_MODE_VOLATILE + else + opt_cache[result.linfo] = ci end end end nothing end -# record the backedges -function store_backedges(caller::CodeInstance, edges::SimpleVector) - isa(caller.def.def, Method) || return # don't add backedges to toplevel method instance - i = 1 - while true - i > length(edges) && return nothing +function is_already_cached(interp::AbstractInterpreter, result::InferenceResult, ci::CodeInstance) + # check if the existing linfo metadata is also sufficient to describe the current inference result + # to decide if it is worth caching this right now + mi = result.linfo + cache = WorldView(code_cache(interp), result.valid_worlds) + if haskey(cache, mi) + # n.b.: accurate edge representation might cause the CodeInstance for this to be constructed later + @assert isdefined(cache[mi], :inferred) + return true + end + return false +end + +# Iterate a series of back-edges that need registering, based on the provided forward edge list. +# Back-edges are returned as (invokesig, item), where the item is a Binding, MethodInstance, or +# MethodTable. +struct ForwardToBackedgeIterator + forward_edges::SimpleVector +end + +function Base.iterate(it::ForwardToBackedgeIterator, i::Int = 1) + edges = it.forward_edges + i > length(edges) && return nothing + while i ≤ length(edges) item = edges[i] if item isa Int i += 2 @@ -593,32 +716,55 @@ function store_backedges(caller::CodeInstance, edges::SimpleVector) i += 1 continue elseif isa(item, Core.Binding) - i += 1 - maybe_add_binding_backedge!(item, caller) - continue + return ((nothing, item), i + 1) end if isa(item, CodeInstance) - item = item.def - end - if isa(item, MethodInstance) # regular dispatch - ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), item, nothing, caller) - i += 1 + item = get_ci_mi(item) + return ((nothing, item), i + 1) + elseif isa(item, MethodInstance) # regular dispatch + return ((nothing, item), i + 1) else + invokesig = item callee = edges[i+1] - if isa(callee, MethodTable) # abstract dispatch (legacy style edges) - ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any, Any), callee, item, caller) - i += 2 - continue - elseif isa(callee, Method) - # ignore `Method`-edges (from e.g. failed `abstract_call_method`) - i += 2 - continue - # `invoke` edge - elseif isa(callee, CodeInstance) - callee = get_ci_mi(callee) + isa(callee, Method) && (i += 2; continue) # ignore `Method`-edges (from e.g. failed `abstract_call_method`) + if isa(callee, MethodTable) + # abstract dispatch (legacy style edges) + return ((invokesig, callee), i + 2) + else + # `invoke` edge + callee = isa(callee, CodeInstance) ? get_ci_mi(callee) : callee::MethodInstance + return ((invokesig, callee), i + 2) + end + end + end + return nothing +end + +# record the backedges +function store_backedges(caller::CodeInstance, edges::SimpleVector) + isa(get_ci_mi(caller).def, Method) || return # don't add backedges to toplevel method instance + + backedges = ForwardToBackedgeIterator(edges) + for (i, (invokesig, item)) in enumerate(backedges) + # check for any duplicate edges we've already registered + duplicate_found = false + for (i′, (invokesig′, item′)) in enumerate(backedges) + i == i′ && break + if item′ === item && invokesig′ == invokesig + duplicate_found = true + break + end + end + + if !duplicate_found + if item isa Core.Binding + maybe_add_binding_backedge!(item, caller) + elseif item isa MethodTable + ccall(:jl_method_table_add_backedge, Cvoid, (Any, Any), invokesig, caller) + else + item::MethodInstance + ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), item, invokesig, caller) end - ccall(:jl_method_instance_add_backedge, Cvoid, (Any, Any, Any), callee, item, caller) - i += 2 end end nothing @@ -1101,8 +1247,7 @@ function typeinf_frame(interp::AbstractInterpreter, mi::MethodInstance, run_opti else opt = OptimizationState(frame, interp) optimize(interp, opt, frame.result) - src = ir_to_codeinf!(opt) - src.edges = Core.svec(opt.inlining.edges...) + src = ir_to_codeinf!(opt, frame, Core.svec(opt.inlining.edges...)) end result.src = frame.src = src end @@ -1113,10 +1258,10 @@ end """ SOURCE_MODE_NOT_REQUIRED -Indicates to inference that the source is not required and the only fields -of the resulting `CodeInstance` that the caller is interested in are types -and effects. Inference is still free to create a CodeInstance with source, -but is not required to do so. +Indicates to inference that the source is not required and the only fields of +the resulting `CodeInstance` that the caller is interested in are return or +exception types and IPO effects. Inference is still free to create source for +it or add it to the JIT even, but is not required or expected to do so. """ const SOURCE_MODE_NOT_REQUIRED = 0x0 @@ -1124,28 +1269,51 @@ const SOURCE_MODE_NOT_REQUIRED = 0x0 SOURCE_MODE_ABI Indicates to inference that it should return a CodeInstance that can -either be `->invoke`'d (because it has already been compiled or because -it has constabi) or one that can be made so by compiling its `->inferred` -field. - -N.B.: The `->inferred` field is volatile and the compiler may delete it. +be `->invoke`'d (because it has already been compiled). """ const SOURCE_MODE_ABI = 0x1 """ - ci_has_abi(code::CodeInstance) + SOURCE_MODE_GET_SOURCE + +Indicates to inference that it should return a CodeInstance after it has +prepared interp to be able to provide source code for it. +""" +const SOURCE_MODE_GET_SOURCE = 0xf + +""" + ci_has_abi(interp::AbstractInterpreter, code::CodeInstance) -Determine whether this CodeInstance is something that could be invoked if we gave it -to the runtime system (either because it already has an ->invoke ptr, or -because it has source that could be compiled). Note that this information may -be stale by the time the user see it, so the user will need to perform their -own checks if they actually need the abi from it. +Determine whether this CodeInstance is something that could be invoked if +interp gave it to the runtime system (either because it already has an ->invoke +ptr, or because interp has source that could be compiled). """ -function ci_has_abi(code::CodeInstance) +function ci_has_abi(interp::AbstractInterpreter, code::CodeInstance) (@atomic :acquire code.invoke) !== C_NULL && return true + return ci_has_source(interp, code) +end + +""" + ci_has_source(interp::AbstractInterpreter, code::CodeInstance) + +Determine whether this CodeInstance is something that could be compiled from +source that interp has. +""" +function ci_has_source(interp::AbstractInterpreter, code::CodeInstance) + codegen = codegen_cache(interp) + codegen === nothing && return false + use_const_api(code) && return true + haskey(codegen, code) && return true inf = @atomic :monotonic code.inferred - if code.owner === nothing ? (isa(inf, CodeInfo) || isa(inf, String)) : inf !== nothing - # interp.codegen[code] = maybe_uncompress(code, inf) # TODO: the correct way to ensure this information doesn't become stale would be to push it into the stable codegen cache + if isa(inf, String) + inf = _uncompressed_ir(code, inf) + end + if code.owner === nothing + if isa(inf, CodeInfo) + codegen[code] = inf + return true + end + elseif inf !== nothing return true end return false @@ -1155,9 +1323,10 @@ function ci_has_invoke(code::CodeInstance) return (@atomic :monotonic code.invoke) !== C_NULL end -function ci_meets_requirement(code::CodeInstance, source_mode::UInt8) +function ci_meets_requirement(interp::AbstractInterpreter, code::CodeInstance, source_mode::UInt8) source_mode == SOURCE_MODE_NOT_REQUIRED && return true - source_mode == SOURCE_MODE_ABI && return ci_has_abi(code) + source_mode == SOURCE_MODE_ABI && return ci_has_abi(interp, code) + source_mode == SOURCE_MODE_GET_SOURCE && return ci_has_source(interp, code) return false end @@ -1167,7 +1336,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod let code = get(code_cache(interp), mi, nothing) if code isa CodeInstance # see if this code already exists in the cache - if ci_meets_requirement(code, source_mode) + if ci_meets_requirement(interp, code, source_mode) ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) return code end @@ -1179,7 +1348,7 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod let code = get(code_cache(interp), mi, nothing) if code isa CodeInstance # see if this code already exists in the cache - if ci_meets_requirement(code, source_mode) + if ci_meets_requirement(interp, code, source_mode) engine_reject(interp, ci) ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) return code @@ -1210,18 +1379,11 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) ci = result.ci # reload from result in case it changed + codegen = codegen_cache(interp) @assert frame.cache_mode != CACHE_MODE_NULL - @assert is_result_constabi_eligible(result) || begin - codegen = codegen_cache(interp) - codegen === nothing || haskey(codegen, ci) - end + @assert is_result_constabi_eligible(result) || codegen === nothing || haskey(codegen, ci) @assert is_result_constabi_eligible(result) == use_const_api(ci) @assert isdefined(ci, :inferred) "interpreter did not fulfill our expectations" - if !is_cached(frame) && source_mode == SOURCE_MODE_ABI - # XXX: jl_type_infer somewhat ambiguously assumes this must be cached - # XXX: this should be using the CI from the cache, if possible instead: haskey(cache, mi) && (ci = cache[mi]) - code_cache(interp)[mi] = ci - end return ci end @@ -1235,48 +1397,113 @@ end typeinf_type(interp::AbstractInterpreter, match::MethodMatch) = typeinf_type(interp, specialize_method(match)) function typeinf_type(interp::AbstractInterpreter, mi::MethodInstance) - # n.b.: this could be replaced with @something(typeinf_ext(interp, mi, SOURCE_MODE_NOT_REQUIRED), return nothing).rettype - start_time = ccall(:jl_typeinf_timing_begin, UInt64, ()) - let code = get(code_cache(interp), mi, nothing) - if code isa CodeInstance - # see if this rettype already exists in the cache - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) - return code.rettype - end - end - ci = engine_reserve(interp, mi) - let code = get(code_cache(interp), mi, nothing) - if code isa CodeInstance - engine_reject(interp, ci) - # see if this rettype already exists in the cache - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) - return code.rettype - end - end - result = InferenceResult(mi, typeinf_lattice(interp)) - result.ci = ci - frame = InferenceState(result, #=cache_mode=#:global, interp) - if frame === nothing - engine_reject(interp, ci) + ci = typeinf_ext(interp, mi, SOURCE_MODE_NOT_REQUIRED) + ci isa CodeInstance || return nothing + return ci.rettype +end + +# Resolve a call, as described by `argtype` to a single matching +# Method and return a compilable MethodInstance for the call, if +# it will be runtime-dispatched to exactly that MethodInstance +function compileable_specialization_for_call(interp::AbstractInterpreter, @nospecialize(argtype)) + mt = ccall(:jl_method_table_for, Any, (Any,), argtype) + if mt === nothing + # this would require scanning all method tables, so give up instead return nothing end - typeinf(interp, frame) - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) - is_inferred(result) || return nothing - return widenconst(ignorelimited(result.result)) + + matches = findall(argtype, method_table(interp); limit = 1) + matches === nothing && return nothing + length(matches.matches) == 0 && return nothing + match = only(matches.matches) + + compileable_atype = get_compileable_sig(match.method, match.spec_types, match.sparams) + compileable_atype === nothing && return nothing + if match.spec_types !== compileable_atype + sp_ = ccall(:jl_type_intersection_with_env, Any, (Any, Any), compileable_atype, match.method.sig)::SimpleVector + sparams = sp_[2]::SimpleVector + mi = specialize_method(match.method, compileable_atype, sparams) + else + mi = specialize_method(match.method, compileable_atype, match.sparams) + end + + return mi end +const QueueItems = Union{CodeInstance,MethodInstance,SimpleVector} + +struct CompilationQueue + tocompile::Vector{QueueItems} + inspected::IdSet{QueueItems} + interp::Union{AbstractInterpreter,Nothing} + + CompilationQueue(; + interp::Union{AbstractInterpreter,Nothing} + ) = new(QueueItems[], IdSet{QueueItems}(), interp) + + CompilationQueue(queue::CompilationQueue; + interp::Union{AbstractInterpreter,Nothing} + ) = new(empty!(queue.tocompile), empty!(queue.inspected), interp) +end + +Base.push!(queue::CompilationQueue, item) = push!(queue.tocompile, item) +Base.append!(queue::CompilationQueue, items) = append!(queue.tocompile, items) +Base.pop!(queue::CompilationQueue) = pop!(queue.tocompile) +Base.empty!(queue::CompilationQueue) = (empty!(queue.tocompile); empty!(queue.inspected)) +markinspected!(queue::CompilationQueue, item) = push!(queue.inspected, item) +isinspected(queue::CompilationQueue, item) = item in queue.inspected +Base.isempty(queue::CompilationQueue) = isempty(queue.tocompile) + # collect a list of all code that is needed along with CodeInstance to codegen it fully -function collectinvokes!(wq::Vector{CodeInstance}, ci::CodeInfo) +function collectinvokes!(workqueue::CompilationQueue, ci::CodeInfo, sptypes::Vector{VarState}; + invokelatest_queue::Union{CompilationQueue,Nothing} = nothing) src = ci.code for i = 1:length(src) stmt = src[i] isexpr(stmt, :(=)) && (stmt = stmt.args[2]) if isexpr(stmt, :invoke) || isexpr(stmt, :invoke_modify) edge = stmt.args[1] - edge isa CodeInstance && isdefined(edge, :inferred) && push!(wq, edge) + edge isa CodeInstance && isdefined(edge, :inferred) && push!(workqueue, edge) + end + + invokelatest_queue === nothing && continue + if isexpr(stmt, :call) + farg = stmt.args[1] + !applicable(argextype, farg, ci, sptypes) && continue # TODO: Why is this failing during bootstrap + ftyp = widenconst(argextype(farg, ci, sptypes)) + + if ftyp === typeof(Core.finalizer) && length(stmt.args) == 3 + finalizer = argextype(stmt.args[2], ci, sptypes) + obj = argextype(stmt.args[3], ci, sptypes) + atype = argtypes_to_type(Any[finalizer, obj]) + else + # No dynamic dispatch to resolve / enqueue + continue + end + elseif isexpr(stmt, :cfunction) && length(stmt.args) == 5 + (pointer_type, f, rt, at, call_type) = stmt.args + linfo = ci.parent + + linfo isa MethodInstance || continue + at isa SimpleVector || continue + + ft = argextype(f, ci, sptypes) + argtypes = Any[ft] + for i = 1:length(at) + push!(argtypes, sp_type_rewrap(at[i], linfo, #= isreturn =# false)) + end + atype = argtypes_to_type(argtypes) + else + # TODO: handle other StmtInfo like OpaqueClosure? + continue + end + let workqueue = invokelatest_queue + # make a best-effort attempt to enqueue the relevant code for the dynamic invokelatest call + mi = compileable_specialization_for_call(workqueue.interp, atype) + mi === nothing && continue + + push!(workqueue, mi) end - # TODO: handle other StmtInfo like @cfunction and OpaqueClosure? end end @@ -1285,14 +1512,13 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn ci isa CodeInstance && !ci_has_invoke(ci) || return ci codegen = codegen_cache(interp) codegen === nothing && return ci - inspected = IdSet{CodeInstance}() - tocompile = Vector{CodeInstance}() - push!(tocompile, ci) - while !isempty(tocompile) + workqueue = CompilationQueue(; interp) + push!(workqueue, ci) + while !isempty(workqueue) # ci_has_real_invoke(ci) && return ci # optimization: cease looping if ci happens to get compiled (not just jl_fptr_wait_for_compiled, but fully jl_is_compiled_codeinst) - callee = pop!(tocompile) + callee = pop!(workqueue) ci_has_invoke(callee) && continue - callee in inspected && continue + isinspected(workqueue, callee) && continue src = get(codegen, callee, nothing) if !isa(src, CodeInfo) src = @atomic :monotonic callee.inferred @@ -1300,18 +1526,32 @@ function add_codeinsts_to_jit!(interp::AbstractInterpreter, ci, source_mode::UIn src = _uncompressed_ir(callee, src) end if !isa(src, CodeInfo) - newcallee = typeinf_ext(interp, callee.def, source_mode) + newcallee = typeinf_ext(workqueue.interp, callee.def, source_mode) # always SOURCE_MODE_ABI if newcallee isa CodeInstance callee === ci && (ci = newcallee) # ci stopped meeting the requirements after typeinf_ext last checked, try again with newcallee - push!(tocompile, newcallee) - #else - # println("warning: could not get source code for ", callee.def) + push!(workqueue, newcallee) + end + if newcallee !== callee + markinspected!(workqueue, callee) end continue end end - push!(inspected, callee) - collectinvokes!(tocompile, src) + markinspected!(workqueue, callee) + mi = get_ci_mi(callee) + sptypes = sptypes_from_meth_instance(mi) + collectinvokes!(workqueue, src, sptypes) + if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) + cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, get_inference_world(workqueue.interp))::CodeInstance + if cached === callee + # make sure callee is gc-rooted and cached, as required by jl_add_codeinst_to_jit + code_cache(workqueue.interp)[mi] = callee + else + # use an existing CI from the cache, if there is available one that is compatible + callee === ci && (ci = cached) + callee = cached + end + end ccall(:jl_add_codeinst_to_jit, Cvoid, (Any, Any), callee, src) end return ci @@ -1324,93 +1564,129 @@ function typeinf_ext_toplevel(interp::AbstractInterpreter, mi::MethodInstance, s end # This is a bridge for the C code calling `jl_typeinf_func()` on a single Method match -function typeinf_ext_toplevel(mi::MethodInstance, world::UInt, source_mode::UInt8) - interp = NativeInterpreter(world) +function typeinf_ext_toplevel(mi::MethodInstance, world::UInt, source_mode::UInt8, trim_mode::UInt8) + inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) + interp = NativeInterpreter(world; inf_params) return typeinf_ext_toplevel(interp, mi, source_mode) end -# This is a bridge for the C code calling `jl_typeinf_func()` on set of Method matches -# The trim_mode can be any of: -const TRIM_NO = 0 -const TRIM_SAFE = 1 -const TRIM_UNSAFE = 2 -const TRIM_UNSAFE_WARN = 3 -function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::Int) - inspected = IdSet{CodeInstance}() - tocompile = Vector{CodeInstance}() - codeinfos = [] - # first compute the ABIs of everything - latest = true # whether this_world == world_counter() - for this_world in reverse(sort!(worlds)) - interp = NativeInterpreter( - this_world; - inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) - ) - for i = 1:length(methods) - # each item in this list is either a MethodInstance indicating something - # to compile, or an svec(rettype, sig) describing a C-callable alias to create. - item = methods[i] - if item isa MethodInstance - # if this method is generally visible to the current compilation world, - # and this is either the primary world, or not applicable in the primary world - # then we want to compile and emit this - if item.def.primary_world <= this_world <= item.def.deleted_world - ci = typeinf_ext(interp, item, SOURCE_MODE_NOT_REQUIRED) - ci isa CodeInstance && push!(tocompile, ci) - end - elseif item isa SimpleVector && latest - (rt::Type, sig::Type) = item - # make a best-effort attempt to enqueue the relevant code for the ccallable - ptr = ccall(:jl_get_specialization1, - #= MethodInstance =# Ptr{Cvoid}, (Any, Csize_t, Cint), - sig, this_world, #= mt_cache =# 0) - if ptr !== C_NULL - mi = unsafe_pointer_to_objref(ptr)::MethodInstance - ci = typeinf_ext(interp, mi, SOURCE_MODE_NOT_REQUIRED) - ci isa CodeInstance && push!(tocompile, ci) - end - # additionally enqueue the ccallable entrypoint / adapter, which implicitly - # invokes the above ci - push!(codeinfos, item) +function compile!(codeinfos::Vector{Any}, workqueue::CompilationQueue; + invokelatest_queue::Union{CompilationQueue,Nothing} = nothing, +) + interp = workqueue.interp + world = get_inference_world(interp) + while !isempty(workqueue) + item = pop!(workqueue) + # each item in this list is either a MethodInstance indicating something + # to compile, or an svec(rettype, sig) describing a C-callable alias to create. + if item isa MethodInstance + isinspected(workqueue, item) && continue + # if this method is generally visible to the current compilation world, + # and this is either the primary world, or not applicable in the primary world + # then we want to compile and emit this + if item.def.primary_world <= world + ci = typeinf_ext(interp, item, SOURCE_MODE_GET_SOURCE) + ci isa CodeInstance && push!(workqueue, ci) end - end - while !isempty(tocompile) - callee = pop!(tocompile) - callee in inspected && continue - push!(inspected, callee) - # now make sure everything has source code, if desired + markinspected!(workqueue, item) + elseif item isa SimpleVector + invokelatest_queue === nothing && continue + (rt::Type, sig::Type) = item + # make a best-effort attempt to enqueue the relevant code for the ccallable + ptr = ccall(:jl_get_specialization1, + #= MethodInstance =# Ptr{Cvoid}, (Any, Csize_t, Cint), + sig, world, #= mt_cache =# 0) + if ptr !== C_NULL + mi = unsafe_pointer_to_objref(ptr)::MethodInstance + ci = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) + ci isa CodeInstance && push!(invokelatest_queue, ci) + end + # additionally enqueue the ccallable entrypoint / adapter, which implicitly + # invokes the above ci + push!(codeinfos, item) + elseif item isa CodeInstance + callee = item + isinspected(workqueue, callee) && continue mi = get_ci_mi(callee) - def = mi.def + # now make sure everything has source code, if desired if use_const_api(callee) src = codeinfo_for_const(interp, mi, callee.rettype_const) - elseif haskey(interp.codegen, callee) - src = interp.codegen[callee] - elseif isa(def, Method) && !InferenceParams(interp).force_enable_inference && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 - src = retrieve_code_info(mi, get_inference_world(interp)) else - # TODO: typeinf_code could return something with different edges/ages/owner/abi (needing an update to callee), which we don't handle here - src = typeinf_code(interp, mi, true) + src = get(interp.codegen, callee, nothing) + if src === nothing + newcallee = typeinf_ext(interp, mi, SOURCE_MODE_GET_SOURCE) + if newcallee isa CodeInstance + @assert use_const_api(newcallee) || haskey(interp.codegen, newcallee) + push!(workqueue, newcallee) + end + if newcallee !== callee + markinspected!(workqueue, callee) + end + continue + end end + markinspected!(workqueue, callee) if src isa CodeInfo - collectinvokes!(tocompile, src) - # It is somewhat ambiguous if typeinf_ext might have callee in the caches, - # but for the purpose of native compile, we always want them put there. + sptypes = sptypes_from_meth_instance(mi) + collectinvokes!(workqueue, src, sptypes; invokelatest_queue) + # try to reuse an existing CodeInstance from before to avoid making duplicates in the cache if iszero(ccall(:jl_mi_cache_has_ci, Cint, (Any, Any), mi, callee)) - code_cache(interp)[mi] = callee + cached = ccall(:jl_get_ci_equiv, Any, (Any, UInt), callee, world)::CodeInstance + if cached === callee + code_cache(interp)[mi] = callee + else + # Use an existing CI from the cache, if there is available one that is compatible + callee = cached + end end push!(codeinfos, callee) push!(codeinfos, src) end - end - latest = false + else @assert false "unexpected item in queue" end end + return codeinfos +end + +# This is a bridge for the C code calling `jl_typeinf_func()` on set of Method matches +# The trim_mode can be any of: +const TRIM_NO = 0x0 +const TRIM_SAFE = 0x1 +const TRIM_UNSAFE = 0x2 +const TRIM_UNSAFE_WARN = 0x3 +function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::UInt8) + inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) + + # Create an "invokelatest" queue to enable eager compilation of speculative + # invokelatest calls such as from `Core.finalizer` and `ccallable` + invokelatest_queue = CompilationQueue(; + interp = NativeInterpreter(get_world_counter(); inf_params) + ) + + codeinfos = [] + workqueue = CompilationQueue(; interp = nothing) + for this_world in reverse!(sort!(worlds)) + workqueue = CompilationQueue(workqueue; + interp = NativeInterpreter(this_world; inf_params) + ) + + append!(workqueue, methods) + compile!(codeinfos, workqueue; invokelatest_queue) + end + + if invokelatest_queue !== nothing + # This queue is intentionally aliased, to handle e.g. a `finalizer` calling `Core.finalizer` + # (it will enqueue into itself and immediately drain) + compile!(codeinfos, invokelatest_queue; invokelatest_queue) + end + if trim_mode != TRIM_NO && trim_mode != TRIM_UNSAFE verify_typeinf_trim(codeinfos, trim_mode == TRIM_UNSAFE_WARN) end return codeinfos end -verify_typeinf_trim(codeinfos::Vector{Any}, onlywarn::Bool) = invokelatest(verify_typeinf_trim, stdout, codeinfos, onlywarn) +const _verify_trim_world_age = RefValue{UInt}(typemax(UInt)) +verify_typeinf_trim(codeinfos::Vector{Any}, onlywarn::Bool) = Core._call_in_world(_verify_trim_world_age[], verify_typeinf_trim, stdout, codeinfos, onlywarn) function return_type(@nospecialize(f), t::DataType) # this method has a special tfunc world = tls_world_age() diff --git a/Compiler/src/typelattice.jl b/Compiler/src/typelattice.jl index 2bf4954343b37..f4c3b051d3e3f 100644 --- a/Compiler/src/typelattice.jl +++ b/Compiler/src/typelattice.jl @@ -8,6 +8,19 @@ # inside the global code cache. import Core: Const, InterConditional, PartialStruct +function may_form_limited_typ(@nospecialize(aty), @nospecialize(bty), @nospecialize(xty)) + if aty isa LimitedAccuracy + if bty isa LimitedAccuracy + return LimitedAccuracy(xty, union!(copy(aty.causes), bty.causes)) + else + return LimitedAccuracy(xty, copy(aty.causes)) + end + elseif bty isa LimitedAccuracy + return LimitedAccuracy(xty, copy(bty.causes)) + end + return nothing +end + """ cnd::Conditional @@ -40,6 +53,8 @@ struct Conditional isdefined::Bool=false) assert_nested_slotwrapper(thentype) assert_nested_slotwrapper(elsetype) + limited = may_form_limited_typ(thentype, elsetype, Bool) + limited !== nothing && return limited return new(slot, thentype, elsetype, isdefined) end end @@ -83,6 +98,8 @@ struct MustAlias assert_nested_slotwrapper(fldtyp) # @assert !isalreadyconst(vartyp) "vartyp is already const" # @assert !isalreadyconst(fldtyp) "fldtyp is already const" + limited = may_form_limited_typ(vartyp, fldtyp, fldtyp) + limited !== nothing && return limited return new(slot, vartyp, fldidx, fldtyp) end end @@ -104,6 +121,8 @@ struct InterMustAlias assert_nested_slotwrapper(fldtyp) # @assert !isalreadyconst(vartyp) "vartyp is already const" # @assert !isalreadyconst(fldtyp) "fldtyp is already const" + limited = may_form_limited_typ(vartyp, fldtyp, fldtyp) + limited !== nothing && return limited return new(slot, vartyp, fldidx, fldtyp) end end diff --git a/Compiler/src/utilities.jl b/Compiler/src/utilities.jl index 17a9073d709af..fe8966c32fc17 100644 --- a/Compiler/src/utilities.jl +++ b/Compiler/src/utilities.jl @@ -4,7 +4,7 @@ # generic # ########### -if !@isdefined(var"@timeit") +if !@isdefined(var"@zone") # This is designed to allow inserting timers when loading a second copy # of inference for performing performance experiments. macro timeit(args...) @@ -158,10 +158,8 @@ end function get_compileable_sig(method::Method, @nospecialize(atype), sparams::SimpleVector) isa(atype, DataType) || return nothing - mt = ccall(:jl_method_get_table, Any, (Any,), method) - mt === nothing && return nothing - return ccall(:jl_normalize_to_compilable_sig, Any, (Any, Any, Any, Any, Cint), - mt, atype, sparams, method, #=int return_if_compileable=#1) + return ccall(:jl_normalize_to_compilable_sig, Any, (Any, Any, Any, Cint), + atype, sparams, method, #=int return_if_compileable=#1) end @@ -329,7 +327,7 @@ end inlining_enabled() = (JLOptions().can_inline == 1) -function coverage_enabled(m::Module) +function instrumentation_enabled(m::Module, only_if_affects_optimizer::Bool) generating_output() && return false # don't alter caches cov = JLOptions().code_coverage if cov == 1 # user @@ -340,6 +338,17 @@ function coverage_enabled(m::Module) elseif cov == 2 # all return true end + if !only_if_affects_optimizer + log = JLOptions().malloc_log + if log == 1 # user + m = moduleroot(m) + m === Core && return false + isdefined(Main, :Base) && m === Main.Base && return false + return true + elseif log == 2 # all + return true + end + end return false end diff --git a/Compiler/src/validation.jl b/Compiler/src/validation.jl index 4f9362e97b30d..067d9460b52ec 100644 --- a/Compiler/src/validation.jl +++ b/Compiler/src/validation.jl @@ -35,8 +35,6 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}( :aliasscope => 0:0, :popaliasscope => 0:0, :new_opaque_closure => 5:typemax(Int), - :import => 1:typemax(Int), - :using => 1:typemax(Int), :export => 1:typemax(Int), :public => 1:typemax(Int), :latestworld => 0:0, @@ -72,11 +70,13 @@ function maybe_validate_code(mi::MethodInstance, src::CodeInfo, kind::String) if !isempty(errors) for e in errors if mi.def isa Method - println(stderr, "WARNING: Encountered invalid ", kind, " code for method ", - mi.def, ": ", e) + println(Core.stderr, + "WARNING: Encountered invalid ", kind, + " code for method ", mi.def, ": ", e) else - println(stderr, "WARNING: Encountered invalid ", kind, " code for top level expression in ", - mi.def, ": ", e) + println(Core.stderr, + "WARNING: Encountered invalid ", kind, + " code for top level expression in ", mi.def, ": ", e) end end error("") @@ -225,7 +225,7 @@ function validate_code!(errors::Vector{InvalidCodeError}, mi::Core.MethodInstanc mnargs = 0 else m = mi.def::Method - mnargs = m.nargs + mnargs = Int(m.nargs) n_sig_params = length((unwrap_unionall(m.sig)::DataType).parameters) if m.is_for_opaque_closure m.sig === Tuple || push!(errors, InvalidCodeError(INVALID_SIGNATURE_OPAQUE_CLOSURE, (m.sig, m.isva))) @@ -234,6 +234,7 @@ function validate_code!(errors::Vector{InvalidCodeError}, mi::Core.MethodInstanc end end if isa(c, CodeInfo) + mnargs = Int(c.nargs) mnargs > length(c.slotnames) && push!(errors, InvalidCodeError(SLOTNAMES_NARGS_MISMATCH)) validate_code!(errors, c, is_top_level) end diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index 735fb83d6f6bb..eb775bfa290ce 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -import ..Compiler: verify_typeinf_trim +import ..Compiler: verify_typeinf_trim, NativeInterpreter, argtypes_to_type, compileable_specialization_for_call using ..Compiler: # operators @@ -14,10 +14,10 @@ using ..Compiler: argextype, empty!, error, get, get_ci_mi, get_world_counter, getindex, getproperty, hasintersect, haskey, in, isdispatchelem, isempty, isexpr, iterate, length, map!, max, pop!, popfirst!, push!, pushfirst!, reinterpret, reverse!, reverse, setindex!, - setproperty!, similar, singleton_type, sptypes_from_meth_instance, - unsafe_pointer_to_objref, widenconst, + setproperty!, similar, singleton_type, sptypes_from_meth_instance, sp_type_rewrap, + unsafe_pointer_to_objref, widenconst, isconcretetype, # misc - @nospecialize, C_NULL + @nospecialize, @assert, C_NULL using ..IRShow: LineInfoNode, print, show, println, append_scopes!, IOContext, IO, normalize_method_name using ..Base: Base, sourceinfo_slotnames using ..Base.StackTraces: StackFrame @@ -110,7 +110,7 @@ end function verify_print_error(io::IOContext{IO}, desc::CallMissing, parents::ParentMap) (; codeinst, codeinfo, sptypes, stmtidx, desc) = desc frames = verify_create_stackframes(codeinst, stmtidx, parents) - print(io, desc, " from ") + print(io, desc, " from statement ") verify_print_stmt(io, codeinfo, sptypes, stmtidx) Base.show_backtrace(io, frames) print(io, "\n\n") @@ -166,7 +166,7 @@ function may_dispatch(@nospecialize ftyp) end end -function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspected::IdSet{CodeInstance}, caches::IdDict{MethodInstance,CodeInstance}, parents::ParentMap, errors::ErrorList) +function verify_codeinstance!(interp::NativeInterpreter, codeinst::CodeInstance, codeinfo::CodeInfo, inspected::IdSet{CodeInstance}, caches::IdDict{MethodInstance,CodeInstance}, parents::ParentMap, errors::ErrorList) mi = get_ci_mi(codeinst) sptypes = sptypes_from_meth_instance(mi) src = codeinfo.code @@ -181,6 +181,11 @@ function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspec if edge isa CodeInstance haskey(parents, edge) || (parents[edge] = (codeinst, i)) edge in inspected && continue + edge_mi = get_ci_mi(edge) + if edge_mi === edge.def + ci = get(caches, edge_mi, nothing) + ci isa CodeInstance && continue # assume that only this_world matters for trim + end end # TODO: check for calls to Base.atexit? elseif isexpr(stmt, :call) @@ -194,7 +199,9 @@ function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspec if !may_dispatch(ftyp) continue end - if Core._apply_iterate isa ftyp + if !isconcretetype(ftyp) + error = "unresolved call to (unknown) builtin" + elseif Core._apply_iterate isa ftyp if length(stmt.args) >= 3 # args[1] is _apply_iterate object # args[2] is invoke object @@ -214,22 +221,62 @@ function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspec end elseif Core.finalizer isa ftyp if length(stmt.args) == 3 - # TODO: check that calling `args[1](args[2])` is defined before warning + finalizer = argextype(stmt.args[2], codeinfo, sptypes) + obj = argextype(stmt.args[3], codeinfo, sptypes) + atype = argtypes_to_type(Any[finalizer, obj]) + + mi = compileable_specialization_for_call(interp, atype) + if mi !== nothing + ci = get(caches, mi, nothing) + ci isa CodeInstance && continue + end + error = "unresolved finalizer registered" - warn = true end - else - error = "unresolved call to builtin" - end + elseif Core._apply isa ftyp + error = "trim verification not yet implemented for builtin `Core._apply`" + elseif Core._call_in_world_total isa ftyp + error = "trim verification not yet implemented for builtin `Core._call_in_world_total`" + elseif Core.invoke isa ftyp + error = "trim verification not yet implemented for builtin `Core.invoke`" + elseif Core.invoke_in_world isa ftyp + error = "trim verification not yet implemented for builtin `Core.invoke_in_world`" + elseif Core.invokelatest isa ftyp + error = "trim verification not yet implemented for builtin `Core.invokelatest`" + elseif Core.modifyfield! isa ftyp + error = "trim verification not yet implemented for builtin `Core.modifyfield!`" + elseif Core.modifyglobal! isa ftyp + error = "trim verification not yet implemented for builtin `Core.modifyglobal!`" + elseif Core.memoryrefmodify! isa ftyp + error = "trim verification not yet implemented for builtin `Core.memoryrefmodify!`" + else @assert false "unexpected builtin" end end extyp = argextype(SSAValue(i), codeinfo, sptypes) if extyp === Union{} warn = true # downgrade must-throw calls to be only a warning end elseif isexpr(stmt, :cfunction) + length(stmt.args) != 5 && continue # required by IR legality + (pointer_type, f, rt, at, call_type) = stmt.args + + at isa SimpleVector || continue # required by IR legality + ft = argextype(f, codeinfo, sptypes) + argtypes = Any[ft] + for i = 1:length(at) + push!(argtypes, sp_type_rewrap(at[i], get_ci_mi(codeinst), #= isreturn =# false)) + end + atype = argtypes_to_type(argtypes) + + mi = compileable_specialization_for_call(interp, atype) + if mi !== nothing + # n.b.: Codegen may choose unpredictably to emit this `@cfunction` as a dynamic invoke or a full + # dynamic call, but in either case it guarantees that the required adapter(s) are emitted. All + # that we are required to verify here is that the callee CodeInstance is covered. + ci = get(caches, mi, nothing) + ci isa CodeInstance && continue + end + error = "unresolved cfunction" - #TODO: parse the cfunction expression to check the target is defined - warn = true elseif isexpr(stmt, :foreigncall) foreigncall = stmt.args[1] if foreigncall isa QuoteNode @@ -252,6 +299,7 @@ end function get_verify_typeinf_trim(codeinfos::Vector{Any}) this_world = get_world_counter() + interp = NativeInterpreter(this_world) inspected = IdSet{CodeInstance}() caches = IdDict{MethodInstance,CodeInstance}() errors = ErrorList() @@ -272,7 +320,7 @@ function get_verify_typeinf_trim(codeinfos::Vector{Any}) item = codeinfos[i] if item isa CodeInstance src = codeinfos[i + 1]::CodeInfo - verify_codeinstance!(item, src, inspected, caches, parents, errors) + verify_codeinstance!(interp, item, src, inspected, caches, parents, errors) elseif item isa SimpleVector rt = item[1]::Type sig = item[2]::Type @@ -287,7 +335,7 @@ function get_verify_typeinf_trim(codeinfos::Vector{Any}) # TODO: should we find a way to indicate to the user that this gets called via ccallable? # parent[ci] = something asrt = ci.rettype - ci in inspected + true else false end @@ -326,6 +374,14 @@ function verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) verify_print_error(io, desc, parents) end + ## TODO: compute and display the minimum and/or full call graph instead of merely the first parent stacktrace? + #for i = 1:length(codeinfos) + # item = codeinfos[i] + # if item isa CodeInstance + # println(item, "::", item.rettype) + # end + #end + let severity = 0 if counts[1] > 0 || counts[2] > 0 print("Trim verify finished with ") diff --git a/Compiler/test/AbstractInterpreter.jl b/Compiler/test/AbstractInterpreter.jl index 5cb7c6fddf15f..12da527225a7e 100644 --- a/Compiler/test/AbstractInterpreter.jl +++ b/Compiler/test/AbstractInterpreter.jl @@ -2,6 +2,7 @@ using Test +include("setup_Compiler.jl") include("irutils.jl") include("newinterp.jl") @@ -547,4 +548,17 @@ let interp = InvokeInterp() mi = @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance} ci = Compiler.typeinf_ext_toplevel(interp, mi, source_mode) @test invoke(f, ci, args...) == 2 + + f = error + args = "test" + mi = @ccall jl_method_lookup(Any[f, args...]::Ptr{Any}, (1+length(args))::Csize_t, Base.tls_world_age()::Csize_t)::Ref{Core.MethodInstance} + ci = Compiler.typeinf_ext_toplevel(interp, mi, source_mode) + result = nothing + try + invoke(f, ci, args...) + catch e + result = sprint(Base.show_backtrace, catch_backtrace()) + end + @test isa(result, String) + @test contains(result, "[1] error(::Char, ::Char, ::Char, ::Char)") end diff --git a/Compiler/test/EscapeAnalysis.jl b/Compiler/test/EscapeAnalysis.jl index 60364769c95a8..07855d3362881 100644 --- a/Compiler/test/EscapeAnalysis.jl +++ b/Compiler/test/EscapeAnalysis.jl @@ -1,5 +1,6 @@ module test_EA +include("setup_Compiler.jl") include("irutils.jl") const EscapeAnalysis = Compiler.EscapeAnalysis diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index fd8bbae70a346..eb5eae29cdc54 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -4,6 +4,7 @@ using Random using InteractiveUtils +using InteractiveUtils: code_llvm, code_native using Libdl using Test @@ -21,7 +22,7 @@ end # The tests below assume a certain format and safepoint_on_entry=true breaks that. function get_llvm(@nospecialize(f), @nospecialize(t), raw=true, dump_module=false, optimize=true) params = Base.CodegenParams(safepoint_on_entry=false, gcstack_arg = false, debug_info_level=Cint(2)) - d = InteractiveUtils._dump_function(f, t, false, false, raw, dump_module, :att, optimize, :none, false, params) + d = InteractiveUtils._dump_function(InteractiveUtils.ArgInfo(f, t), false, false, raw, dump_module, :att, optimize, :none, false, params) sprint(print, d) end @@ -409,7 +410,7 @@ function g_dict_hash_alloc() end # Warm up f_dict_hash_alloc(); g_dict_hash_alloc(); -@test abs((@allocated f_dict_hash_alloc()) / (@allocated g_dict_hash_alloc()) - 1) < 0.1 # less that 10% difference +@test abs((@allocated f_dict_hash_alloc()) / (@allocated g_dict_hash_alloc()) - 1) < 0.3 # returning an argument shouldn't alloc a new box @noinline f33829(x) = (global called33829 = true; x) @@ -950,6 +951,16 @@ for (T, StructName) in ((Int128, :Issue55558), (UInt128, :UIssue55558)) end end +# Issue #42326 +primitive type PadAfter64_42326 448 end +mutable struct CheckPadAfter64_42326 + a::UInt64 + pad::PadAfter64_42326 + b::UInt64 +end +@test fieldoffset(CheckPadAfter64_42326, 3) == 80 +@test sizeof(CheckPadAfter64_42326) == 96 + @noinline Base.@nospecializeinfer f55768(@nospecialize z::UnionAll) = z === Vector @test f55768(Vector) @test f55768(Vector{T} where T) @@ -1032,3 +1043,35 @@ end const x57872 = "Hello" f57872() = (Core.isdefinedglobal(@__MODULE__, Base.compilerbarrier(:const, :x57872)), x57872) # Extra globalref here to force world age bounds @test f57872() == (true, "Hello") + +@noinline f_mutateany(@nospecialize x) = x[] = 1 +g_mutateany() = (y = Ref(0); f_mutateany(y); y[]) +@test g_mutateany() === 1 + +# 58470 tbaa for unionselbyte of heap allocated mutables +mutable struct Wrapper58470 + x::Union{Nothing,Int} +end + +function findsomething58470(dict, inds) + default = Wrapper58470(nothing) + for i in inds + x = get(dict, i, default).x + if !isnothing(x) + return x + end + end + return nothing +end + +let io = IOBuffer() + code_llvm(io, findsomething58470, Tuple{Dict{Int64, Wrapper58470}, Vector{Int}}, dump_module=true, raw=true, optimize=false) + str = String(take!(io)) + @test !occursin("jtbaa_unionselbyte", str) +end + +let io = IOBuffer() + code_llvm(io, (x, y) -> (@atomic x[1] = y; nothing), (AtomicMemory{Pair{Any,Any}}, Pair{Any,Any},), raw=true, optimize=false) + str = String(take!(io)) + @test occursin("julia.write_barrier", str) +end diff --git a/Compiler/test/compact.jl b/Compiler/test/compact.jl index b01e209d5ce9b..5b19dc68811fc 100644 --- a/Compiler/test/compact.jl +++ b/Compiler/test/compact.jl @@ -2,6 +2,7 @@ using Test +include("setup_Compiler.jl") include("irutils.jl") using .Compiler: IncrementalCompact, insert_node_here!, finish, diff --git a/Compiler/test/contextual.jl b/Compiler/test/contextual.jl index a9c63ab34c0c0..941ce172d41e2 100644 --- a/Compiler/test/contextual.jl +++ b/Compiler/test/contextual.jl @@ -1,19 +1,23 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +module contextual + # N.B.: This file is also run from interpreter.jl, so needs to be standalone-executable using Test -include("setup_Compiler.jl") # Cassette # ======== +# TODO Use CassetteBase.jl instead of this mini-cassette? + module MiniCassette # A minimal demonstration of the cassette mechanism. Doesn't support all the # fancy features, but sufficient to exercise this code path in the compiler. + using Core: SimpleVector using Core.IR - using ..Compiler - using ..Compiler: retrieve_code_info, quoted, anymap + using Base: Compiler as CC + using .CC: retrieve_code_info, quoted, anymap using Base.Meta: isexpr export Ctx, overdub @@ -21,7 +25,7 @@ module MiniCassette struct Ctx; end # A no-op cassette-like transform - function transform_expr(expr, map_slot_number, map_ssa_value, sparams::Core.SimpleVector) + function transform_expr(expr, map_slot_number, map_ssa_value, sparams::SimpleVector) @nospecialize expr transform(@nospecialize expr) = transform_expr(expr, map_slot_number, map_ssa_value, sparams) if isexpr(expr, :call) @@ -45,11 +49,11 @@ module MiniCassette end end - function transform!(mi::MethodInstance, ci::CodeInfo, nargs::Int, sparams::Core.SimpleVector) + function transform!(mi::MethodInstance, ci::CodeInfo, nargs::Int, sparams::SimpleVector) code = ci.code - di = Compiler.DebugInfoStream(mi, ci.debuginfo, length(code)) - ci.slotnames = Symbol[Symbol("#self#"), :ctx, :f, :args, ci.slotnames[nargs+1:end]...] - ci.slotflags = UInt8[(0x00 for i = 1:4)..., ci.slotflags[nargs+1:end]...] + di = CC.DebugInfoStream(mi, ci.debuginfo, length(code)) + ci.slotnames = Symbol[Symbol("#self#"), :ctx, :f, :args, ci.slotnames[nargs+2:end]...] + ci.slotflags = UInt8[(0x00 for i = 1:4)..., ci.slotflags[nargs+2:end]...] # Insert one SSAValue for every argument statement prepend!(code, Any[Expr(:call, getfield, SlotNumber(4), i) for i = 1:nargs]) prepend!(di.codelocs, fill(Int32(0), 3nargs)) @@ -76,21 +80,26 @@ module MiniCassette function overdub_generator(world::UInt, source, self, ctx, f, args) @nospecialize + argnames = Core.svec(:overdub, :ctx, :f, :args) + spnames = Core.svec() + if !Base.issingletontype(f) # (c, f, args..) -> f(args...) - ex = :(return f(args...)) - return Core.GeneratedFunctionStub(identity, Core.svec(:overdub, :ctx, :f, :args), Core.svec())(world, source, ex) + return generate_lambda_ex(world, source, argnames, spnames, :(return f(args...))) end tt = Tuple{f, args...} match = Base._which(tt; world) mi = Base.specialize_method(match) # Unsupported in this mini-cassette - @assert !mi.def.isva + !mi.def.isva || + return generate_lambda_ex(world, source, argnames, spnames, :(error("Unsupported vararg method"))) src = retrieve_code_info(mi, world) - @assert isa(src, CodeInfo) + isa(src, CodeInfo) || + return generate_lambda_ex(world, source, argnames, spnames, :(error("Unexpected code transformation"))) src = copy(src) - @assert src.edges === Core.svec() + src.edges === Core.svec() || + return generate_lambda_ex(world, source, argnames, spnames, :(error("Unexpected code transformation"))) src.edges = Any[mi] transform!(mi, src, length(args), match.sparams) # TODO: this is mandatory: code_info.min_world = max(code_info.min_world, min_world[]) @@ -98,9 +107,21 @@ module MiniCassette # Match the generator, since that's what our transform! does src.nargs = 4 src.isva = true + errors = CC.validate_code(mi, src) + if !isempty(errors) + foreach(Core.println, errors) + return generate_lambda_ex(world, source, argnames, spnames, :(error("Found errors in generated code"))) + end return src end + function generate_lambda_ex(world::UInt, source::Method, + argnames::SimpleVector, spnames::SimpleVector, + body::Expr) + stub = Core.GeneratedFunctionStub(identity, argnames, spnames) + return stub(world, source, body) + end + @inline overdub(::Ctx, f::Union{Core.Builtin, Core.IntrinsicFunction}, args...) = f(args...) @eval function overdub(ctx::Ctx, f, args...) @@ -124,3 +145,8 @@ f() = 2 foo(i) = i+bar(Val(1)) @test @inferred(overdub(Ctx(), foo, 1)) == 43 + +morethan4args(a, b, c, d, e) = (((a + b) + c) + d) + e +@test overdub(Ctx(), morethan4args, 1, 2, 3, 4, 5) == 15 + +end # module contextual diff --git a/Compiler/test/effects.jl b/Compiler/test/effects.jl index 49a9628ba97c5..18cfc0e8d5388 100644 --- a/Compiler/test/effects.jl +++ b/Compiler/test/effects.jl @@ -1,6 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test + +include("setup_Compiler.jl") include("irutils.jl") # Test that the Core._apply_iterate bail path taints effects @@ -1467,3 +1469,13 @@ end @test Base.infer_effects(Core.invoke_in_world, Tuple{Vararg{Any}}) == Compiler.Effects() @test Base.infer_effects(invokelatest, Tuple{Vararg{Any}}) == Compiler.Effects() @test Base.infer_effects(invoke, Tuple{Vararg{Any}}) == Compiler.Effects() + +# Core._svec_ref effects modeling (required for external abstract interpreter that doesn't run optimization) +let effects = Base.infer_effects((Core.SimpleVector,Int); optimize=false) do svec, i + Core._svec_ref(svec, i) + end + @test !Compiler.is_consistent(effects) + @test Compiler.is_effect_free(effects) + @test !Compiler.is_nothrow(effects) + @test Compiler.is_terminates(effects) +end diff --git a/Compiler/test/inference.jl b/Compiler/test/inference.jl index c6649afdb1a51..58cd3db49c371 100644 --- a/Compiler/test/inference.jl +++ b/Compiler/test/inference.jl @@ -4,6 +4,7 @@ module inference using Test +include("setup_Compiler.jl") include("irutils.jl") # tests for Compiler correctness and precision @@ -1209,6 +1210,7 @@ let isdefined_tfunc(@nospecialize xs...) = @test isdefined_tfunc(Union{UnionIsdefinedA,UnionIsdefinedB}, Const(:x)) === Const(true) @test isdefined_tfunc(Union{UnionIsdefinedA,UnionIsdefinedB}, Const(:y)) === Const(false) @test isdefined_tfunc(Union{UnionIsdefinedA,Nothing}, Const(:x)) === Bool + @test isdefined_tfunc(Nothing, Any) === Const(false) end # https://github.com/aviatesk/JET.jl/issues/379 @@ -1595,9 +1597,12 @@ let memoryref_tfunc(@nospecialize xs...) = Compiler.memoryref_tfunc(Compiler.fal @test memoryref_tfunc(MemoryRef{Int}, Int, Symbol) == Union{} @test memoryref_tfunc(MemoryRef{Int}, Int, Bool) == MemoryRef{Int} @test memoryref_tfunc(MemoryRef{Int}, Int, Vararg{Bool}) == MemoryRef{Int} - @test memoryref_tfunc(Memory{Int}, Int) == Union{} - @test memoryref_tfunc(Any, Any, Any) == Any # also probably could be GenericMemoryRef - @test memoryref_tfunc(Any, Any) == Any # also probably could be GenericMemoryRef + @test memoryref_tfunc(Memory{Int}, Int) == MemoryRef{Int} + @test memoryref_tfunc(Memory{Int}, Int, Symbol) == Union{} + @test memoryref_tfunc(Memory{Int}, Int, Bool) == MemoryRef{Int} + @test memoryref_tfunc(Memory{Int}, Int, Vararg{Bool}) == MemoryRef{Int} + @test memoryref_tfunc(Any, Any, Any) == GenericMemoryRef + @test memoryref_tfunc(Any, Any) == GenericMemoryRef @test memoryref_tfunc(Any) == GenericMemoryRef @test memoryrefget_tfunc(MemoryRef{Int}, Symbol, Bool) === Int @test memoryrefget_tfunc(MemoryRef{Int}, Any, Any) === Int @@ -2263,12 +2268,18 @@ struct AliasableFields{S,T} f1::S f2::T end +struct NullableAliasableFields{S,T} + f1::S + f2::T + NullableAliasableFields(f1::S, f2::T) where {S,T} = new{S,T}(f1, f2) + NullableAliasableFields(f1::S) where {S} = new{S,Union{}}(f1) +end mutable struct AliasableConstField{S,T} const f1::S f2::T end -import .Compiler: +using .Compiler: InferenceLattice, MustAliasesLattice, InterMustAliasesLattice, BaseInferenceLattice, SimpleInferenceLattice, IPOResultLattice, typeinf_lattice, ipo_lattice, optimizer_lattice @@ -2281,7 +2292,7 @@ Compiler.optimizer_lattice(::MustAliasInterpreter) = SimpleInferenceLattice.inst # lattice # ------- -import .Compiler: MustAlias, Const, PartialStruct, ⊑, tmerge +using .Compiler: MustAlias, Const, PartialStruct, ⊑, tmerge let 𝕃ᵢ = InferenceLattice(MustAliasesLattice(BaseInferenceLattice.instance)) ⊑(@nospecialize(a), @nospecialize(b)) = Compiler.:⊑(𝕃ᵢ, a, b) tmerge(@nospecialize(a), @nospecialize(b)) = Compiler.tmerge(𝕃ᵢ, a, b) @@ -2305,12 +2316,6 @@ let 𝕃ᵢ = InferenceLattice(MustAliasesLattice(BaseInferenceLattice.instance) @test ifelse_tfunc(MustAlias(2, AliasableField{Any}, 1, Int), Int, Int) === Union{} end -@testset "issue #56913: `BoundsError` in type inference" begin - R = UnitRange{Int} - @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, Vararg{R}}) - @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, R, Vararg{R}}) -end - maybeget_mustalias_tmerge(x::AliasableField) = x.f maybeget_mustalias_tmerge(x) = x @test Base.return_types((Union{Nothing,AliasableField{Any}},); interp=MustAliasInterpreter()) do x @@ -2519,6 +2524,15 @@ jet509_hasitems(list) = length(list) >= 1 error("list is empty") end |> only == Vector{Int} +# don't form nested slot wrappers +@test Base.infer_return_type((NullableAliasableFields{NullableAliasableFields},); interp=MustAliasInterpreter()) do x + y = getfield(x, :f1) + if isdefined(y, :f2) && isa(getfield(y, :f2), Int) + return getfield(y, :f2) + end + return 0 +end == Int + # === constraint # -------------- @@ -2576,6 +2590,19 @@ end |> only === Compiler.InterMustAlias return 0 end == Integer +# `isdefined` accuracy for `MustAlias` +@test Base.infer_return_type((Any,); interp=MustAliasInterpreter()) do x + xx = Ref{Any}(x) + xxx = Some{Any}(xx) + Val(isdefined(xxx.value, :x)) +end == Val{true} + +@testset "issue #56913: `BoundsError` in type inference" begin + R = UnitRange{Int} + @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, Vararg{R}}) + @test Type{AbstractVector} == Base.infer_return_type(Base.promote_typeof, Tuple{R, R, Vector{Any}, R, Vararg{R}}) +end + function f25579(g) h = g[] t = (h === nothing) @@ -3522,8 +3549,14 @@ f31974(n::Int) = f31974(1:n) # call cycles. @test code_typed(f31974, Tuple{Int}) !== nothing -f_overly_abstract_complex() = Complex(Ref{Number}(1)[]) -@test Base.return_types(f_overly_abstract_complex, Tuple{}) == [Complex] +# Issue #33472 +struct WrapperWithUnionall33472{T<:Real} + x::T +end + +f_overly_abstract33472() = WrapperWithUnionall33472(Base.inferencebarrier(1)::Number) +# Check that this doesn't infer as `WrapperWithUnionall33472{T<:Number}`. +@test Base.return_types(f_overly_abstract33472, Tuple{}) == [WrapperWithUnionall33472] # Issue 26724 const IntRange = AbstractUnitRange{<:Integer} @@ -6347,14 +6380,28 @@ end === Int swapglobal!(@__MODULE__, :swapglobal!_xxx, x) end === Union{} +@newinterp AssumeBindingsStaticInterp +Compiler.InferenceParams(::AssumeBindingsStaticInterp) = Compiler.InferenceParams(; assume_bindings_static=true) + eval(Expr(:const, :swapglobal!_must_throw)) -@newinterp SwapGlobalInterp -Compiler.InferenceParams(::SwapGlobalInterp) = Compiler.InferenceParams(; assume_bindings_static=true) function func_swapglobal!_must_throw(x) swapglobal!(@__MODULE__, :swapglobal!_must_throw, x) end -@test Base.infer_return_type(func_swapglobal!_must_throw, (Int,); interp=SwapGlobalInterp()) === Union{} -@test !Compiler.is_effect_free(Base.infer_effects(func_swapglobal!_must_throw, (Int,); interp=SwapGlobalInterp()) ) +@test Base.infer_return_type(func_swapglobal!_must_throw, (Int,); interp=AssumeBindingsStaticInterp()) === Union{} +@test !Compiler.is_effect_free(Base.infer_effects(func_swapglobal!_must_throw, (Int,); interp=AssumeBindingsStaticInterp()) ) + +global global_decl_defined +global_decl_defined = 42 +@test Base.infer_effects(; interp=AssumeBindingsStaticInterp()) do + global global_decl_defined + return global_decl_defined +end |> Compiler.is_nothrow +global global_decl_defined2::Int +global_decl_defined2 = 42 +@test Base.infer_effects(; interp=AssumeBindingsStaticInterp()) do + global global_decl_defined2 + return global_decl_defined2 +end |> Compiler.is_nothrow @eval get_exception() = $(Expr(:the_exception)) @test Base.infer_return_type() do @@ -6408,4 +6455,64 @@ let src = code_typed1((Base.RefValue{String}, String)) do x, val @test isa(retval, Core.SSAValue) end +global invalid_setglobal!_exct_modeling::Int +@test Base.infer_exception_type((Float64,)) do x + setglobal!(@__MODULE__, :invalid_setglobal!_exct_modeling, x) +end == ErrorException + +# Issue #58257 - Hang in inference during BindingPartition resolution +module A58257 + module B58257 + using ..A58257 + # World age here is N + end + using .B58257 + # World age here is N+1 + @eval f() = $(GlobalRef(B58257, :get!)) +end + +## The sequence of events is critical here. +A58257.get! # Creates binding partition in A, N+1:∞ +A58257.B58257.get! # Creates binding partition in A.B, N+1:∞ +Base.invoke_in_world(UInt(38678), getglobal, A58257, :get!) # Expands binding partition in A through only |> first ir.argtypes[1] = Tuple{} @@ -2226,7 +2229,7 @@ let ir = Base.code_ircode((Issue52644,); optimize_until="Inlining") do t @test irfunc(Issue52644(Tuple{<:Integer})) === :UnionAll end issue52644_single(x::DataType) = :DataType -let ir = Base.code_ircode((Issue52644,); optimize_until="Inlining") do t +let ir = Base.code_ircode((Issue52644,); optimize_until="CC: INLINING") do t issue52644_single(t.tuple) end |> only |> first ir.argtypes[1] = Tuple{} @@ -2310,3 +2313,5 @@ g_noinline_invoke(x) = f_noinline_invoke(x) let src = code_typed1(g_noinline_invoke, (Union{Symbol,Nothing},)) @test !any(@nospecialize(x)->isa(x,GlobalRef), src.code) end + +end # module inline_tests diff --git a/Compiler/test/invalidation.jl b/Compiler/test/invalidation.jl index db9d3ac06048f..b51411db1da05 100644 --- a/Compiler/test/invalidation.jl +++ b/Compiler/test/invalidation.jl @@ -3,6 +3,7 @@ # setup # ----- +include("setup_Compiler.jl") include("irutils.jl") using Test @@ -161,6 +162,42 @@ begin @test "42" == String(take!(GLOBAL_BUFFER)) end +begin + deduped_callee(x::Int) = @noinline rand(Int) + deduped_caller1(x::Int) = @noinline deduped_callee(x) + deduped_caller2(x::Int) = @noinline deduped_callee(x) + + # run inference on both `deduped_callerx` and `deduped_callee` + let (src, rt) = code_typed((Int,); interp=InvalidationTester()) do x + @inline deduped_caller1(x) + @inline deduped_caller2(x) + end |> only + @test rt === Int + @test any(isinvoke(:deduped_callee), src.code) + end + + # Verify that adding the backedge again does not actually add a new backedge + let mi = Base.method_instance(deduped_caller1, (Int,)), + ci = mi.cache + + callee_mi = Base.method_instance(deduped_callee, (Int,)) + + # Inference should have added the callers to the callee's backedges + @test ci in callee_mi.backedges + + # In practice, inference will never end up calling `store_backedges` + # twice on the same CodeInstance like this - we only need to check + # that de-duplication works for a single invocation + N = length(callee_mi.backedges) + Core.Compiler.store_backedges(ci, Core.svec(callee_mi, callee_mi)) + N′ = length(callee_mi.backedges) + + # A single `store_backedges` invocation should de-duplicate any of the + # edges it is adding. + @test N′ - N == 1 + end +end + # we can avoid adding backedge even if the callee's return type is not the top # when the return value is not used within the caller begin take!(GLOBAL_BUFFER) @@ -283,3 +320,8 @@ begin take!(GLOBAL_BUFFER) @test isnothing(pr48932_caller_inlined(42)) @test "42" == String(take!(GLOBAL_BUFFER)) end + +# Issue #57696 +# This test checks for invalidation of recursive backedges. However, unfortunately, the original failure +# manifestation was an unreliable segfault or an assertion failure, so we don't have a more compact test. +@test success(`$(Base.julia_cmd()) -e 'Base.typejoin(x, ::Type) = 0; exit()'`) diff --git a/Compiler/test/irpasses.jl b/Compiler/test/irpasses.jl index 7912f1c9c7f9e..758efaab9ab6b 100644 --- a/Compiler/test/irpasses.jl +++ b/Compiler/test/irpasses.jl @@ -4,6 +4,7 @@ using Test using Base.Meta using Core.IR +include("setup_Compiler.jl") include("irutils.jl") # domsort @@ -920,7 +921,7 @@ let # Test that CFG simplify doesn't try to merge every block in a loop into end # `cfg_simplify!` shouldn't error in a presence of `try/catch` block -let ir = Base.code_ircode(; optimize_until="slot2ssa") do +let ir = Base.code_ircode(; optimize_until="CC: SLOT2REG") do v = try catch end @@ -1180,7 +1181,7 @@ end @test Compiler.is_effect_free(Base.infer_effects(getfield, (Complex{Int}, Symbol))) -# We consider a potential deprecatio warning an effect, so for completely unkown getglobal, +# We consider a potential deprecatio warning an effect, so for completely unknown getglobal, # we taint the effect_free bit. @test !Compiler.is_effect_free(Base.infer_effects(getglobal, (Module, Symbol))) @@ -1458,7 +1459,7 @@ function f_with_early_try_catch_exit() result end -let ir = first(only(Base.code_ircode(f_with_early_try_catch_exit, (); optimize_until="compact"))) +let ir = first(only(Base.code_ircode(f_with_early_try_catch_exit, (); optimize_until="CC: SLOT2REG"))) for i = 1:length(ir.stmts) expr = ir.stmts[i][:stmt] if isa(expr, PhiCNode) @@ -2110,3 +2111,13 @@ let src = code_typed1(()) do end @test count(iscall((src, isdefined)), src.code) == 0 end +# We should successfully fold the default values of a ScopedValue +const svalconstprop = ScopedValue(1) +foosvalconstprop() = svalconstprop[] + +let src = code_typed1(foosvalconstprop, ()) + function is_constfield_load(expr) + iscall((src, getfield))(expr) && expr.args[3] in (:(:has_default), :(:default)) + end + @test count(is_constfield_load, src.code) == 0 +end diff --git a/Compiler/test/irutils.jl b/Compiler/test/irutils.jl index c1616ad4a8fd0..e50491420c338 100644 --- a/Compiler/test/irutils.jl +++ b/Compiler/test/irutils.jl @@ -14,7 +14,7 @@ macro code_typed1(ex0...) end get_code(args...; kwargs...) = code_typed1(args...; kwargs...).code macro get_code(ex0...) - return gen_call_with_extracted_types_and_kwargs(__module__, :get_code, ex0) + return gen_call_with_extracted_types_and_kwargs(__module__, :get_code, ex0; is_source_reflection = false) end # check if `x` is a statement with a given `head` @@ -58,7 +58,7 @@ function fully_eliminated(code::Vector{Any}; retval=(@__FILE__), kwargs...) return retval′ == retval end macro fully_eliminated(ex0...) - return gen_call_with_extracted_types_and_kwargs(__module__, :fully_eliminated, ex0) + return gen_call_with_extracted_types_and_kwargs(__module__, :fully_eliminated, ex0; is_source_reflection = false) end let m = Meta.@lower 1 + 1 diff --git a/Compiler/test/newinterp.jl b/Compiler/test/newinterp.jl index 5ebcf332895fa..1e0c50192fb5b 100644 --- a/Compiler/test/newinterp.jl +++ b/Compiler/test/newinterp.jl @@ -2,6 +2,8 @@ # TODO set up a version who defines new interpreter with persistent cache? +include("setup_Compiler.jl") + """ @newinterp NewInterpreter [ephemeral_cache::Bool=false] diff --git a/Compiler/test/runtests.jl b/Compiler/test/runtests.jl index 6a38fce678ba0..8a35d8e71102f 100644 --- a/Compiler/test/runtests.jl +++ b/Compiler/test/runtests.jl @@ -1,8 +1,10 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Test, Compiler -using InteractiveUtils: @activate -@activate Compiler +Base.runtests(["Compiler"]; propagate_project=true) + +#= +# To run serially +using Test, Compiler @testset "Compiler.jl" begin for file in readlines(joinpath(@__DIR__, "testgroups")) file == "special_loading" && continue # Only applicable to Base.Compiler @@ -10,3 +12,4 @@ using InteractiveUtils: @activate @eval @testset $testfile include($testfile) end end +=# diff --git a/Compiler/test/setup_Compiler.jl b/Compiler/test/setup_Compiler.jl index a28a3f918aaf9..83cc42e2d0936 100644 --- a/Compiler/test/setup_Compiler.jl +++ b/Compiler/test/setup_Compiler.jl @@ -1,5 +1,12 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +using InteractiveUtils: @activate + +if Base.identify_package("Compiler") !== nothing && !isdefined(Main, :__custom_compiler_active) + Base.eval(Main, :(__custom_compiler_active=true)) + @activate Compiler +end + if !@isdefined(Compiler) if Base.REFLECTION_COMPILER[] === nothing using Base.Compiler: Compiler diff --git a/Compiler/test/special_loading.jl b/Compiler/test/special_loading.jl index ba012446dc61f..ba8cbc635eae8 100644 --- a/Compiler/test/special_loading.jl +++ b/Compiler/test/special_loading.jl @@ -1,9 +1,12 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -mktempdir() do dir - withenv("JULIA_DEPOT_PATH" => dir * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do - cd(joinpath(@__DIR__, "CompilerLoadingTest")) do - @test success(pipeline(`$(Base.julia_cmd()[1]) --startup-file=no --project=. compiler_loading_test.jl`; stdout, stderr)) +# Only run when testing Base compiler +if Base.identify_package("Compiler") === nothing + mktempdir() do dir + withenv("JULIA_DEPOT_PATH" => dir * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do + cd(joinpath(@__DIR__, "CompilerLoadingTest")) do + @test success(pipeline(`$(Base.julia_cmd()[1]) --startup-file=no --project=. compiler_loading_test.jl`; stdout, stderr)) + end end end end diff --git a/Compiler/test/ssair.jl b/Compiler/test/ssair.jl index 3026202466599..7aca2b8977a4e 100644 --- a/Compiler/test/ssair.jl +++ b/Compiler/test/ssair.jl @@ -1,5 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +include("setup_Compiler.jl") include("irutils.jl") using Test @@ -418,7 +419,7 @@ end @test first(only(Base.code_ircode(+, (Float64, Float64)))) isa Compiler.IRCode @test first(only(Base.code_ircode(+, (Float64, Float64); optimize_until = 3))) isa Compiler.IRCode - @test first(only(Base.code_ircode(+, (Float64, Float64); optimize_until = "SROA"))) isa + @test first(only(Base.code_ircode(+, (Float64, Float64); optimize_until = "CC: SROA"))) isa Compiler.IRCode function demo(f) @@ -428,7 +429,7 @@ end end @test first(only(Base.code_ircode(demo))) isa Compiler.IRCode @test first(only(Base.code_ircode(demo; optimize_until = 3))) isa Compiler.IRCode - @test first(only(Base.code_ircode(demo; optimize_until = "SROA"))) isa Compiler.IRCode + @test first(only(Base.code_ircode(demo; optimize_until = "CC: SROA"))) isa Compiler.IRCode end # slots after SSA conversion @@ -441,12 +442,12 @@ end let # #self#, a, b, c, d unopt = code_typed1(f_with_slots, (Int,Int); optimize=false) @test length(unopt.slotnames) == length(unopt.slotflags) == length(unopt.slottypes) == 5 - ir_withslots = first(only(Base.code_ircode(f_with_slots, (Int,Int); optimize_until="convert"))) + ir_withslots = first(only(Base.code_ircode(f_with_slots, (Int,Int); optimize_until="CC: CONVERT"))) @test length(ir_withslots.argtypes) == 5 # #self#, a, b opt = code_typed1(f_with_slots, (Int,Int); optimize=true) @test length(opt.slotnames) == length(opt.slotflags) == length(opt.slottypes) == 3 - ir_ssa = first(only(Base.code_ircode(f_with_slots, (Int,Int); optimize_until="slot2reg"))) + ir_ssa = first(only(Base.code_ircode(f_with_slots, (Int,Int); optimize_until="CC: SLOT2REG"))) @test length(ir_ssa.argtypes) == 3 end @@ -458,7 +459,7 @@ let @test stmt.cond === v elseif isa(stmt, ReturnNode) || isa(stmt, UpsilonNode) @test stmt.val === v - elseif isa(stmt, SSAValue) || isa(stmt, NewSSAValue) + elseif isa(stmt, SSAValue) || isa(stmt, NewSSAValue) || isa(stmt, Argument) @test stmt === v elseif isa(stmt, PiNode) @test stmt.val === v && stmt.typ === typeof(stmt) @@ -507,6 +508,7 @@ let GotoNode(5), SSAValue(7), NewSSAValue(9), + Argument(1), ReturnNode(SSAValue(11)), ] @@ -597,9 +599,10 @@ import Core: SSAValue import .Compiler: NewInstruction, insert_node! # insert_node! for pending node -let ir = Base.code_ircode((Int,Int); optimize_until="inlining") do a, b +let ir = Base.code_ircode((Int,Int); optimize_until="CC: INLINING") do a, b a^b end |> only |> first + ir = Compiler.compact!(ir) nstmts = length(ir.stmts) invoke_idx = findfirst(@nospecialize(stmt)->Meta.isexpr(stmt, :invoke), ir.stmts.stmt) @test invoke !== nothing @@ -658,9 +661,10 @@ let code = Any[ end # insert_node! with new instruction with flag computed -let ir = Base.code_ircode((Int,Int); optimize_until="inlining") do a, b +let ir = Base.code_ircode((Int,Int); optimize_until="CC: INLINING") do a, b a^b end |> only |> first + ir = Compiler.compact!(ir) invoke_idx = findfirst(@nospecialize(stmt)->Meta.isexpr(stmt, :invoke), ir.stmts.stmt) @test invoke_idx !== nothing invoke_expr = ir.stmts.stmt[invoke_idx] @@ -722,7 +726,7 @@ end @test any(j -> isa(unopt.code[j], Core.Const) && unopt.ssavaluetypes[j] == Union{}, 1:length(unopt.code)) # Any GotoIfNot destinations after IRCode conversion should not be statically unreachable - ircode = first(only(Base.code_ircode(f_with_maybe_nonbool_cond, (Int, Bool); optimize_until="convert"))) + ircode = first(only(Base.code_ircode(f_with_maybe_nonbool_cond, (Int, Bool); optimize_until="CC: CONVERT"))) for i = 1:length(ircode.stmts) expr = ircode.stmts[i][:stmt] if isa(expr, GotoIfNot) @@ -818,3 +822,26 @@ let cl = Int32[32, 1, 1, 1000, 240, 230] cl2 = ccall(:jl_uncompress_codelocs, Any, (Any, Int), str, 2) @test cl == cl2 end + +@test_throws ErrorException Base.code_ircode(+, (Float64, Float64); optimize_until = "nonexisting pass name") +@test_throws ErrorException Base.code_ircode(+, (Float64, Float64); optimize_until = typemax(Int)) + +#57153 check that the CFG has a #0 block predecessor and that we don't fail to compile code that observes that +function _worker_task57153() + while true + r = let + try + if @noinline rand(Bool) + return nothing + end + q, m + finally + missing + end + end + r[1]::Bool + end +end +let ir = Base.code_ircode(_worker_task57153, (), optimize_until="CC: COMPACT_2")[1].first + @test findfirst(x->x==0, ir.cfg.blocks[1].preds) !== nothing +end diff --git a/Compiler/test/tarjan.jl b/Compiler/test/tarjan.jl index aa04bd94a6f6a..8fe940463b558 100644 --- a/Compiler/test/tarjan.jl +++ b/Compiler/test/tarjan.jl @@ -2,6 +2,7 @@ using Test +include("setup_Compiler.jl") include("irutils.jl") using .Compiler: CFGReachability, DomTree, CFG, BasicBlock, StmtRange, dominates, diff --git a/Compiler/test/validation.jl b/Compiler/test/validation.jl index 5328516f63d36..f01ff85e4321c 100644 --- a/Compiler/test/validation.jl +++ b/Compiler/test/validation.jl @@ -20,13 +20,15 @@ function f22938(a, b, x...) return i * a end -msig = Tuple{typeof(f22938),Int,Int,Int,Int} -world = Base.get_world_counter() -match = only(Base._methods_by_ftype(msig, -1, world)) -mi = Compiler.specialize_method(match) -c0 = Compiler.retrieve_code_info(mi, world) - -@test isempty(Compiler.validate_code(mi, c0)) +const c0 = let + msig = Tuple{typeof(f22938),Int,Int,Int,Int} + world = Base.get_world_counter() + match = only(Base._methods_by_ftype(msig, -1, world)) + mi = Compiler.specialize_method(match) + c0 = Compiler.retrieve_code_info(mi, world) + @test isempty(Compiler.validate_code(mi, c0)) + c0 +end @testset "INVALID_EXPR_HEAD" begin c = copy(c0) @@ -114,15 +116,6 @@ end @test errors[1].kind === Compiler.SSAFLAGS_MISMATCH end -@testset "SIGNATURE_NARGS_MISMATCH" begin - old_sig = mi.def.sig - mi.def.sig = Tuple{1,2} - errors = Compiler.validate_code(mi, nothing) - mi.def.sig = old_sig - @test length(errors) == 1 - @test errors[1].kind === Compiler.SIGNATURE_NARGS_MISMATCH -end - @testset "NON_TOP_LEVEL_METHOD" begin c = copy(c0) c.code[1] = Expr(:method, :dummy) @@ -130,12 +123,3 @@ end @test length(errors) == 1 @test errors[1].kind === Compiler.NON_TOP_LEVEL_METHOD end - -@testset "SLOTNAMES_NARGS_MISMATCH" begin - mi.def.nargs += 20 - errors = Compiler.validate_code(mi, c0) - mi.def.nargs -= 20 - @test length(errors) == 2 - @test count(e.kind === Compiler.SLOTNAMES_NARGS_MISMATCH for e in errors) == 1 - @test count(e.kind === Compiler.SIGNATURE_NARGS_MISMATCH for e in errors) == 1 -end diff --git a/Compiler/test/verifytrim.jl b/Compiler/test/verifytrim.jl index 34ca0f89f26fb..a84afd6933266 100644 --- a/Compiler/test/verifytrim.jl +++ b/Compiler/test/verifytrim.jl @@ -18,31 +18,63 @@ let infos = Any[] @test isempty(parents) end -# use TRIM_UNSAFE to bypass verifier inside typeinf_ext_toplevel -let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Base.SecretBuffer}})], [Base.get_world_counter()], TRIM_UNSAFE) - @test_broken length(infos) > 4 +finalizer(@nospecialize(f), @nospecialize(o)) = Core.finalizer(f, o) + +let infos = typeinf_ext_toplevel(Any[Core.svec(Nothing, Tuple{typeof(finalizer), typeof(identity), Any})], [Base.get_world_counter()], TRIM_UNSAFE) errors, parents = get_verify_typeinf_trim(infos) - @test_broken isempty(errors) # missing cfunction - #resize!(infos, 4) - #errors, = get_verify_typeinf_trim(infos) + @test !isempty(errors) # unresolvable finalizer - desc = only(errors) - @test desc.first - desc = desc.second + # the only error should be a CallMissing error for the Core.finalizer builtin + (warn, desc) = only(errors) + @test !warn @test desc isa CallMissing @test occursin("finalizer", desc.desc) repr = sprint(verify_print_error, desc, parents) @test occursin( - r"""^unresolved finalizer registered from \(Core.finalizer\)\(Base.final_shred!, %new\(\)::Base.SecretBuffer\)::Nothing + r"""^unresolved finalizer registered from statement \(Core.finalizer\)\(f::Any, o::Any\)::Nothing + Stacktrace: + \[1\] finalizer\(f::Any, o::Any\)""", repr) +end + +# test that basic `cfunction` generation is allowed, when the dispatch target can be resolved +make_cfunction() = @cfunction(+, Float64, (Int64,Int64)) +let infos = typeinf_ext_toplevel(Any[Core.svec(Ptr{Cvoid}, Tuple{typeof(make_cfunction)})], [Base.get_world_counter()], TRIM_UNSAFE) + errors, parents = get_verify_typeinf_trim(infos) + @test isempty(errors) +end + +# use TRIM_UNSAFE to bypass verifier inside typeinf_ext_toplevel +make_cfunction_bad(@nospecialize(f::Any)) = @cfunction($f, Float64, (Int64,Int64))::Base.CFunction +let infos = typeinf_ext_toplevel(Any[Core.svec(Base.CFunction, Tuple{typeof(make_cfunction_bad), Any})], [Base.get_world_counter()], TRIM_UNSAFE) + errors, parents = get_verify_typeinf_trim(infos) + @test !isempty(errors) # missing cfunction + + (is_warning, desc) = only(errors) + @test !is_warning + @test desc isa CallMissing + @test occursin("cfunction", desc.desc) + repr = sprint(verify_print_error, desc, parents) + @test occursin(r"""^unresolved cfunction from statement \$\(Expr\(:cfunction, Base.CFunction, :\(f::Any\), Float64, :\(svec\(Int64, Int64\)::Core.SimpleVector\), :\(:ccall\)\)\)::Base.CFunction Stacktrace: - \[1\] finalizer\(f::typeof\(Base.final_shred!\), o::Base.SecretBuffer\) - @ Base gcutils.jl:(\d+) \[inlined\] - \[2\] Base.SecretBuffer\(; sizehint::Int\d\d\) - @ Base secretbuffer.jl:(\d+) \[inlined\] - \[3\] Base.SecretBuffer\(\) - @ Base secretbuffer.jl:(\d+) - - $""", repr) + \[1\] make_cfunction_bad\(f::Any\)""", repr) + resize!(infos, 1) + @test infos[1] isa Core.SimpleVector && infos[1][1] isa Type && infos[1][2] isa Type + errors, parents = get_verify_typeinf_trim(infos) + desc = only(errors) + @test !desc.first + desc = desc.second + @test desc isa CCallableMissing + @test desc.rt == Base.CFunction + @test desc.sig == Tuple{typeof(make_cfunction_bad), Any} + @test occursin("unresolved ccallable", desc.desc) + repr = sprint(verify_print_error, desc, parents) + @test repr == "unresolved ccallable for Tuple{$(typeof(make_cfunction_bad)), Any} => Base.CFunction\n\n" +end + +let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Base.SecretBuffer}})], [Base.get_world_counter()], TRIM_UNSAFE) + @test length(infos) > 4 + errors, parents = get_verify_typeinf_trim(infos) + @test isempty(errors) resize!(infos, 1) @test infos[1] isa Core.SimpleVector && infos[1][1] isa Type && infos[1][2] isa Type diff --git a/HISTORY.md b/HISTORY.md index e8f3ed01a1baa..dd666eb3312ed 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,11 +1,14 @@ Julia v1.12 Release Notes -======================== +========================= New language features --------------------- -* New option `--trim` creates smaller binaries by removing code that was not proven to be reachable from - entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). +* New experimental option `--trim` that creates smaller binaries by removing code not proven to be reachable from + entry points. Entry points can be marked using `Base.Experimental.entrypoint` ([#55047]). Not all + code is expected to work with this option, and since it is experimental you may encounter problems. +* Redefinition of constants is now well defined and follows world age semantics ([#57253]). Additional redefinitions + (e.g. of types) are now allowed. See [the new manual chapter on world age](https://docs.julialang.org/en/v1.13-dev/manual/worldage/). * A new keyword argument `usings::Bool` has been added to `names`, returning all names visible via `using` ([#54609]). * The `@atomic` macro family now supports reference assignment syntax, e.g. `@atomic :monotonic v[3] += 4`, @@ -33,9 +36,10 @@ Language changes * Julia now defaults to 1 "interactive" thread, in addition to the 1 default "worker" thread. i.e. `-t1,1`. This means in default configuration the main task and repl (when in interactive mode), which both run on thread 1, now run within the `interactive` threadpool. The libuv IO loop also runs on thread 1, - helping efficient utilization of the worker threadpool used by `Threads.@spawn`. Pass `0` to disable the - interactive thread i.e. `-t1,0` or `JULIA_NUM_THREADS=1,0`, or `-tauto,0` etc. The zero is explicitly - required to disable it, `-t2` will set the equivalent of `-t2,1` ([#57087]). + helping efficient utilization of the worker threadpool used by `Threads.@spawn`. Asking for specifically 1 thread + (`-t1`/`JULIA_NUM_THREADS=1`) or passing `0` will disable the interactive thread i.e. `-t1,0` or `JULIA_NUM_THREADS=1,0` + , or `-tauto,0` etc. Asking for more than 1 thread will enable the interactive thread so + `-t2` will set the equivalent of `-t2,1` ([#57087]). * When a method is replaced with an exactly equivalent one, the old method is not deleted. Instead, the new method takes priority and becomes more specific than the old method. Thus if the new method is deleted later, the old method will resume operating. This can be useful in mocking frameworks (as in SparseArrays, @@ -55,7 +59,7 @@ Language changes * Calling `using` on a package name inside of that package of that name (especially relevant for a submodule) now explicitly uses that package without examining the Manifest and environment, which is identical to the behavior of `..Name`. This appears to better match - how users expect this to behave in the wild. ([#57727]) + how users expect this to behave in the wild ([#57727]). Compiler/Runtime improvements ----------------------------- @@ -130,6 +134,8 @@ New library features * `Timer` now has readable `timeout` and `interval` properties, and a more descriptive `show` method ([#57081]). * `sort` now supports `NTuple`s ([#54494]). * `map!(f, A)` now stores the results in `A`, like `map!(f, A, A)` or `A .= f.(A)` ([#40632]). +* `setprecision` with a function argument (typically a `do` block) is now thread safe. Other forms + should be avoided, and types should switch to an implementation using `ScopedValue` ([#51362]). Standard library changes ------------------------ @@ -279,9 +285,10 @@ Tooling Improvements [#57081]: https://github.com/JuliaLang/julia/issues/57081 [#57087]: https://github.com/JuliaLang/julia/issues/57087 [#57109]: https://github.com/JuliaLang/julia/issues/57109 +[#57253]: https://github.com/JuliaLang/julia/issues/57253 Julia v1.11 Release Notes -======================== +========================= New language features --------------------- @@ -293,20 +300,6 @@ New language features * The new macro `Base.Cartesian.@ncallkw` is analogous to `Base.Cartesian.@ncall`, but allows to add keyword arguments to the function call ([#51501]). * Support for Unicode 15.1 ([#51799]). -* Three new types around the idea of text with "annotations" (`Pair{Symbol, Any}` - entries, e.g. `:lang => "en"` or `:face => :magenta`). These annotations - are preserved across operations (e.g. string concatenation with `*`) when - possible. - * `AnnotatedString` is a new `AbstractString` type. It wraps an underlying - string and allows for annotations to be attached to regions of the string. - This type is used extensively in the new `StyledStrings` standard library to - hold styling information. - * `AnnotatedChar` is a new `AbstractChar` type. It wraps another char and - holds a list of annotations that apply to it. - * `AnnotatedIOBuffer` is a new `IO` type that mimics an `IOBuffer`, but has - specialised `read`/`write` methods for annotated content. This can be - thought of both as a "string builder" of sorts and also as glue between - annotated and unannotated content. * `Manifest.toml` files can now be renamed in the format `Manifest-v{major}.{minor}.toml` to be preferentially picked up by the given julia version. i.e. in the same folder, a `Manifest-v1.11.toml` would be used by v1.11 and `Manifest.toml` by every other julia @@ -408,7 +401,20 @@ Standard library changes #### StyledStrings -* A new standard library for handling styling in a more comprehensive and structured way ([#49586]). +* A new experimental standard library for handling styling in a more comprehensive and structured way ([#49586]). +* Three new types around the idea of text with "annotations" (`Pair{Symbol, Any}` + entries, e.g. `:lang => "en"` or `:face => :magenta`). These annotations + are preserved across operations (e.g. string concatenation with `*`) when + possible. + * `AnnotatedString` is a new `AbstractString` type. It wraps an underlying + string and allows for annotations to be attached to regions of the string. + This type is used extensively to hold styling information. + * `AnnotatedChar` is a new `AbstractChar` type. It wraps another char and + holds a list of annotations that apply to it. + * `AnnotatedIOBuffer` is a new `IO` type that mimics an `IOBuffer`, but has + specialised `read`/`write` methods for annotated content. This can be + thought of both as a "string builder" of sorts and also as glue between + annotated and unannotated content. * The new `Faces` struct serves as a container for text styling information (think typeface, as well as color and decoration), and comes with a framework to provide a convenient, extensible (via `addface!`), and customisable (with a diff --git a/Make.inc b/Make.inc index da7dfcdd35a05..049d0bebe05de 100644 --- a/Make.inc +++ b/Make.inc @@ -59,6 +59,7 @@ USE_SYSTEM_LIBGIT2:=0 USE_SYSTEM_PATCHELF:=0 USE_SYSTEM_LIBWHICH:=0 USE_SYSTEM_ZLIB:=0 +USE_SYSTEM_ZSTD:=0 USE_SYSTEM_P7ZIP:=0 USE_SYSTEM_LLD:=0 @@ -373,11 +374,15 @@ $(1)_rel_eval = $(call rel_path,$(2),$($(1))) $(1)_rel = $$(call hit_cache,$(1)_rel_eval) endef $(foreach D,libdir private_libdir datarootdir libexecdir private_libexecdir docdir sysconfdir includedir,$(eval $(call cache_rel_path,$(D),$(bindir)))) -$(foreach D,build_libdir build_private_libdir,$(eval $(call cache_rel_path,$(D),$(build_bindir)))) +$(foreach D,build_libdir build_private_libdir ,$(eval $(call cache_rel_path,$(D),$(build_bindir)))) # Save a special one: reverse_private_libdir_rel: usually just `../`, but good to be general: reverse_private_libdir_rel_eval = $(call rel_path,$(private_libdir),$(libdir)) reverse_private_libdir_rel = $(call hit_cache,reverse_private_libdir_rel_eval) +reverse_private_libexecdir_rel_eval = $(call rel_path,$(private_libexecdir),$(private_libdir)) +reverse_private_libexecdir_rel = $(call hit_cache,reverse_private_libexecdir_rel_eval) +reverse_build_private_libexecdir_rel_eval = $(call rel_path,$(build_private_libexecdir),$(build_libdir)) +reverse_build_private_libexecdir_rel = $(call hit_cache,reverse_build_private_libexecdir_rel_eval) INSTALL_F := $(JULIAHOME)/contrib/install.sh 644 INSTALL_M := $(JULIAHOME)/contrib/install.sh 755 @@ -426,9 +431,10 @@ export PKG_CONFIG_PATH = $(JULIAHOME)/usr/lib/pkgconfig export PKG_CONFIG_LIBDIR = $(JULIAHOME)/usr/lib/pkgconfig # Figure out OS and architecture -BUILD_OS := $(shell uname) +RAW_BUILD_OS = $(shell uname) +BUILD_OS := $(RAW_BUILD_OS) -ifneq (,$(findstring CYGWIN,$(BUILD_OS))) +ifneq (,$(findstring CYGWIN,$(RAW_BUILD_OS))) XC_HOST ?= $(shell uname -m)-w64-mingw32 endif @@ -516,6 +522,10 @@ endif # Compiler specific stuff +ifneq ($(USECLANG)$(USEGCC),) +$(error "These internal variables are not overridable. Set CC instead.") +endif + ifeq (default,$(origin CC)) CC := $(CROSS_COMPILE)$(CC) # attempt to add cross-compiler prefix, if the user # is not overriding the default, to form target-triple-cc (which @@ -552,7 +562,7 @@ JCPPFLAGS_COMMON := -fasynchronous-unwind-tables JCPPFLAGS_CLANG := $(JCPPFLAGS_COMMON) -mllvm -enable-tail-merge=0 JCPPFLAGS_GCC := $(JCPPFLAGS_COMMON) -fno-tree-tail-merge -JCXXFLAGS_COMMON := -pipe $(fPIC) -fno-rtti -std=c++17 -Wformat -Wformat-security +JCXXFLAGS_COMMON := -pipe $(fPIC) -fno-rtti -std=c++17 -Wformat -Wformat-security -fno-strict-aliasing JCXXFLAGS_CLANG := $(JCXXFLAGS_COMMON) -pedantic JCXXFLAGS_GCC := $(JCXXFLAGS_COMMON) -fno-gnu-unique @@ -788,6 +798,8 @@ ifneq ($(OS), Darwin) endif endif +bootstrap_julia_flags := + ifeq ($(SANITIZE),1) SANITIZE_OPTS := SANITIZE_LDFLAGS := @@ -805,6 +817,9 @@ endif ifeq ($(SANITIZE_THREAD),1) SANITIZE_OPTS += -fsanitize=thread SANITIZE_LDFLAGS += -fsanitize=thread +ifneq ($(CROSS_BOOTSTRAP_JULIA),) +bootstrap_julia_flags += --target-sanitize=thread +endif endif ifeq ($(SANITIZE_OPTS),) $(error SANITIZE=1, but no sanitizer selected, set either SANITIZE_MEMORY, SANITIZE_THREAD, or SANITIZE_ADDRESS) @@ -822,6 +837,9 @@ $(error "please install either GNU tar or bsdtar") endif endif +# Do not try to extract owner uids, even if we're root (e.g. in sandboxes) +TAR += --no-same-owner + ifeq ($(WITH_GC_VERIFY), 1) JCXXFLAGS += -DGC_VERIFY JCFLAGS += -DGC_VERIFY @@ -1417,7 +1435,7 @@ CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.34|GLIBCXX_3\.5\.|GLIBCXX_4\. # Note: we explicitly _do not_ define `CSL` here, since it requires some more # advanced techniques to decide whether it should be installed from a BB source # or not. See `deps/csl.mk` for more detail. -BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP OPENSSL LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP LLD LIBTRACYCLIENT BOLT +BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP OPENSSL LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB ZSTD P7ZIP LLD LIBTRACYCLIENT BOLT ifeq (${USE_THIRD_PARTY_GC},mmtk) BB_PROJECTS += MMTK_JULIA @@ -1567,7 +1585,9 @@ ifeq ($(OS), WINNT) HAVE_SSP := 1 OSLIBS += -Wl,--export-all-symbols -Wl,--version-script=$(BUILDROOT)/src/julia.expmap \ $(NO_WHOLE_ARCHIVE) -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp -luserenv -lsecur32 -latomic -lole32 -JLDFLAGS += -Wl,--stack,8388608 --disable-auto-import --disable-runtime-pseudo-reloc +# N.B.: Unlike in the sysimage, we cannot -Wl,--disable-auto-import -Wl,--disable-runtime-pseudo-reloc here, because libstdc++/LLVM are not fully correct under +# enforced visibility at this point. +JLDFLAGS += -Wl,--stack,8388608 ifeq ($(ARCH),i686) JLDFLAGS += -Wl,--large-address-aware endif @@ -1656,8 +1676,8 @@ endif # Note: we're passing *FLAGS here computed based on your system compiler to # clang. If that causes you problems, you might want to build and/or run # specific clang-sa-* files with clang explicitly selected: -# make CC=~+/../usr/tools/clang CXX=~+/../usr/tools/clang USECLANG=1 analyzegc -# make USECLANG=1 clang-sa-* +# make CC=~+/../usr/tools/clang CXX=~+/../usr/tools/clang analyzegc +# make clang-sa-* CLANGSA_FLAGS := CLANGSA_CXXFLAGS := ifeq ($(OS), Darwin) # on new XCode, the files are hidden @@ -1746,9 +1766,15 @@ JULIA_BUILD_MODE := debug endif endif +ifneq ($(CROSS_BOOTSTRAP_JULIA),) +JULIA_EXECUTABLE_debug := $(CROSS_BOOTSTRAP_JULIA) +JULIA_EXECUTABLE_release := $(CROSS_BOOTSTRAP_JULIA) +JULIA_EXECUTABLE := $(CROSS_BOOTSTRAP_JULIA) +else JULIA_EXECUTABLE_debug := $(build_bindir)/julia-debug$(EXE) JULIA_EXECUTABLE_release := $(build_bindir)/julia$(EXE) JULIA_EXECUTABLE := $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE)) +endif JULIA_SYSIMG_debug := $(build_private_libdir)/sys-debug.$(SHLIB_EXT) JULIA_SYSIMG_release := $(build_private_libdir)/sys.$(SHLIB_EXT) diff --git a/Makefile b/Makefile index 7df749bd74541..3e4fc1356b08d 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ default: $(JULIA_BUILD_MODE) # contains either "debug" or "release" all: debug release # sort is used to remove potential duplicates -DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/stdlib $(build_man1dir)) +DIRS := $(sort $(build_bindir) $(build_depsbindir) $(build_libdir) $(build_private_libdir) $(build_private_libexecdir) $(build_libexecdir) $(build_includedir) $(build_includedir)/julia $(build_sysconfdir)/julia $(build_datarootdir)/julia $(build_datarootdir)/julia/stdlib $(build_man1dir)) ifneq ($(BUILDROOT),$(JULIAHOME)) BUILDDIRS := $(BUILDROOT) $(addprefix $(BUILDROOT)/,base src src/flisp src/support src/clangsa cli doc deps stdlib test test/clangsa test/embedding test/gcext test/llvmpasses) BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) $(BUILDROOT)/sysimage.mk $(BUILDROOT)/pkgimage.mk @@ -89,7 +89,7 @@ julia-deps: | $(DIRS) $(build_datarootdir)/julia/base $(build_datarootdir)/julia julia-stdlib: | $(DIRS) julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/stdlib -julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac.jl $(build_datarootdir)/julia/juliac-buildscript.jl +julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/julia.1 $(build_datarootdir)/julia/julia-config.jl $(build_datarootdir)/julia/juliac/juliac.jl $(build_datarootdir)/julia/juliac/juliac-buildscript.jl $(build_datarootdir)/julia/juliac/juliac-trim-base.jl $(build_datarootdir)/julia/juliac/juliac-trim-stdlib.jl $(build_datarootdir)/julia/juliac/Artifacts.toml @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/base julia-libccalltest: julia-deps @@ -110,9 +110,13 @@ julia-src-release julia-src-debug : julia-src-% : julia-deps julia_flisp.boot.in julia-cli-release julia-cli-debug: julia-cli-% : julia-deps @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/cli $* -julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% julia-src-% | $(build_private_libdir) +julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir) @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$* +# Useful for cross-bootstrapping +julia-sysbase-release julia-sysbase-debug : julia-sysbase-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir) + @$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysbase-$* + julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest \ julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest julia-base-cache @@ -136,6 +140,15 @@ else $(warn "Skipping whitespace check because git is unavailable") endif +fix-whitespace: +ifneq ($(NO_GIT), 1) + @# Append the directory containing the julia we just built to the end of `PATH`, + @# to give us the best chance of being able to run this check. + @PATH="$(PATH):$(dir $(JULIA_EXECUTABLE))" julia $(call cygpath_w,$(JULIAHOME)/contrib/check-whitespace.jl) --fix +else + $(warn "Skipping whitespace fix because git is unavailable") +endif + release-candidate: release testall @$(JULIA_EXECUTABLE) $(JULIAHOME)/contrib/add_license_to_files.jl #add license headers @#Check documentation @@ -184,6 +197,7 @@ $(build_sysconfdir)/julia/startup.jl: $(JULIAHOME)/etc/startup.jl | $(build_sysc @cp $< $@ $(build_datarootdir)/julia/%: $(JULIAHOME)/contrib/% | $(build_datarootdir)/julia + mkdir -p $(dir $@) $(INSTALL_M) $< $(dir $@) $(build_depsbindir)/stringreplace: $(JULIAHOME)/contrib/stringreplace.c | $(build_depsbindir) @@ -203,6 +217,12 @@ endif # private libraries, that are installed in $(prefix)/lib/julia JL_PRIVATE_LIBS-0 := libccalltest libccalllazyfoo libccalllazybar libllvmcalltest +JL_PRIVATE_LIBS-1 := # libraries from USE_SYSTEM=1 +JL_PRIVATE_EXES := 7z +ifeq ($(OS),WINNT) +JL_PRIVATE_EXES += 7z.dll +endif +JL_PRIVATE_TOOLS := ifeq ($(JULIA_BUILD_MODE),release) JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen else ifeq ($(JULIA_BUILD_MODE),debug) @@ -232,9 +252,12 @@ JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += zlib else JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libz endif +JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libzstd +JL_PRIVATE_EXES += zstd$(EXE) zstdmt$(EXE) ifeq ($(USE_LLVM_SHLIB),1) JL_PRIVATE_LIBS-$(USE_SYSTEM_LLVM) += libLLVM $(LLVM_SHARED_LIB_NAME) endif +JL_PRIVATE_TOOLS += lld$(EXE) dsymutil$(EXE) JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBUNWIND) += libunwind ifeq ($(USE_SYSTEM_LIBM),0) @@ -313,37 +336,33 @@ install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index $(INSTALL_M) $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE)) $(DESTDIR)$(bindir)/ ifeq ($(OS),WINNT) - -$(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/ + $(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/ ifeq ($(JULIA_BUILD_MODE),release) - -$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/ - -$(INSTALL_M) $(build_libdir)/libjulia-internal.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libjulia-internal.dll.a $(DESTDIR)$(libdir)/ else ifeq ($(JULIA_BUILD_MODE),debug) - -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ - -$(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/ endif - -$(INSTALL_M) $(wildcard $(build_private_libdir)/*.a) $(DESTDIR)$(private_libdir)/ - -rm -f $(DESTDIR)$(private_libdir)/sys-o.a + $(INSTALL_M) $(filter-out %-bc.a %-o.a,$(wildcard $(build_private_libdir)/lib*.a)) $(DESTDIR)$(private_libdir)/ - # We have a single exception; we want 7z.dll to live in private_libexecdir, - # not bindir, so that 7z.exe can find it. - -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(private_libexecdir)/ - -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ - -$(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ + $(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/ else # Copy over .dSYM directories directly for Darwin ifneq ($(DARWIN_FRAMEWORK),1) ifeq ($(OS),Darwin) ifeq ($(JULIA_BUILD_MODE),release) - -cp -a $(build_libdir)/libjulia.*.dSYM $(DESTDIR)$(libdir) - -cp -a $(build_libdir)/libjulia-internal.*.dSYM $(DESTDIR)$(private_libdir) - -cp -a $(build_libdir)/libjulia-codegen.*.dSYM $(DESTDIR)$(private_libdir) - -cp -a $(build_private_libdir)/sys.dylib.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_libdir)/libjulia.*.dSYM $(DESTDIR)$(libdir) + cp -a $(build_libdir)/libjulia-internal.*.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_libdir)/libjulia-codegen.*.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_private_libdir)/sys.dylib.dSYM $(DESTDIR)$(private_libdir) else ifeq ($(JULIA_BUILD_MODE),debug) - -cp -a $(build_libdir)/libjulia-debug.*.dSYM $(DESTDIR)$(libdir) - -cp -a $(build_libdir)/libjulia-internal-debug.*.dSYM $(DESTDIR)$(private_libdir) - -cp -a $(build_libdir)/libjulia-codegen-debug.*.dSYM $(DESTDIR)$(private_libdir) - -cp -a $(build_private_libdir)/sys-debug.dylib.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_libdir)/libjulia-debug.*.dSYM $(DESTDIR)$(libdir) + cp -a $(build_libdir)/libjulia-internal-debug.*.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_libdir)/libjulia-codegen-debug.*.dSYM $(DESTDIR)$(private_libdir) + cp -a $(build_private_libdir)/sys-debug.dylib.dSYM $(DESTDIR)$(private_libdir) endif endif @@ -351,7 +370,7 @@ endif for suffix in $(JL_TARGETS) ; do \ for lib in $(build_libdir)/lib$${suffix}.*$(SHLIB_EXT)*; do \ if [ "$${lib##*.}" != "dSYM" ]; then \ - $(INSTALL_M) $$lib $(DESTDIR)$(libdir) ; \ + $(INSTALL_M) $$lib $(DESTDIR)$(libdir) || exit 1; \ fi \ done \ done @@ -371,26 +390,24 @@ endif for suffix in $(JL_PRIVATE_LIBS-0) ; do \ for lib in $(build_libdir)/$${suffix}.*$(SHLIB_EXT)*; do \ if [ "$${lib##*.}" != "dSYM" ]; then \ - $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \ + $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) || exit 1; \ fi \ done \ done for suffix in $(JL_PRIVATE_LIBS-1) ; do \ for lib in $(build_private_libdir)/$${suffix}.$(SHLIB_EXT)*; do \ if [ "$${lib##*.}" != "dSYM" ]; then \ - $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \ + $(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) || exit 1; \ fi \ done \ done endif - # Install `7z` into private_libexecdir - $(INSTALL_M) $(build_bindir)/7z$(EXE) $(DESTDIR)$(private_libexecdir)/ - - # Install `lld` into private_libexecdir - $(INSTALL_M) $(build_depsbindir)/lld$(EXE) $(DESTDIR)$(private_libexecdir)/ - - # Install `dsymutil` into private_libexecdir/ - $(INSTALL_M) $(build_depsbindir)/dsymutil$(EXE) $(DESTDIR)$(private_libexecdir)/ + for exe in $(JL_PRIVATE_EXES) ; do \ + $(INSTALL_M) $(build_private_libexecdir)/$$exe $(DESTDIR)$(private_libexecdir) || exit 1; \ + done + for exe in $(JL_PRIVATE_TOOLS) ; do \ + $(INSTALL_M) $(build_depsbindir)/$$exe $(DESTDIR)$(private_libexecdir) || exit 1; \ + done # Copy public headers cp -R -L $(build_includedir)/julia/* $(DESTDIR)$(includedir)/julia @@ -442,13 +459,13 @@ ifneq ($(private_libdir_rel),$(build_private_libdir_rel)) ifeq ($(OS), Darwin) ifneq ($(DARWIN_FRAMEWORK),1) for j in $(JL_TARGETS) ; do \ - install_name_tool -rpath @executable_path/$(build_private_libdir_rel) @executable_path/$(private_libdir_rel) $(DESTDIR)$(bindir)/$$j; \ - install_name_tool -add_rpath @executable_path/$(build_libdir_rel) @executable_path/$(libdir_rel) $(DESTDIR)$(bindir)/$$j; \ + install_name_tool -rpath @executable_path/$(build_private_libdir_rel) @executable_path/$(private_libdir_rel) $(DESTDIR)$(bindir)/$$j || exit 1; \ + install_name_tool -rpath @executable_path/$(build_libdir_rel) @executable_path/$(libdir_rel) $(DESTDIR)$(bindir)/$$j || exit 1; \ done endif else ifneq (,$(findstring $(OS),Linux FreeBSD)) for j in $(JL_TARGETS) ; do \ - $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(private_libdir_rel):$$ORIGIN/$(libdir_rel)' $(DESTDIR)$(bindir)/$$j; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(private_libdir_rel):$$ORIGIN/$(libdir_rel)' $(DESTDIR)$(bindir)/$$j || exit 1; \ done endif @@ -471,11 +488,11 @@ endif ifeq ($(OS), Darwin) ifneq ($(DARWIN_FRAMEWORK),1) ifeq ($(JULIA_BUILD_MODE),release) - install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-internal.$(SHLIB_EXT) - install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-codegen.$(SHLIB_EXT) + install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-internal.$(SHLIB_EXT) + install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-codegen.$(SHLIB_EXT) else ifeq ($(JULIA_BUILD_MODE),debug) - install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT) - install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-codegen-debug.$(SHLIB_EXT) + install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT) + install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-codegen-debug.$(SHLIB_EXT) endif endif else ifneq (,$(findstring $(OS),Linux FreeBSD)) @@ -486,11 +503,43 @@ else ifeq ($(JULIA_BUILD_MODE),debug) $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN:$$ORIGIN/$(reverse_private_libdir_rel)' $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT) $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN:$$ORIGIN/$(reverse_private_libdir_rel)' $(DESTDIR)$(private_libdir)/libjulia-codegen-debug.$(SHLIB_EXT) endif +endif + +ifeq ($(OS), Darwin) +ifneq ($(DARWIN_FRAMEWORK),1) + for j in $(JL_PRIVATE_TOOLS) ; do \ + [ -L $(DESTDIR)$(private_libexecdir)/$$j ] && continue; \ + install_name_tool -rpath @loader_path/$(build_libdir_rel) @executable_path/$(reverse_private_libexecdir_rel) $(DESTDIR)$(private_libexecdir)/$$j || exit 1; \ + done +endif +else ifneq (,$(findstring $(OS),Linux FreeBSD)) + for j in $(JL_PRIVATE_TOOLS) ; do \ + [ -L $(DESTDIR)$(private_libexecdir)/$$j ] && continue; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_private_libexecdir_rel)' $(DESTDIR)$(private_libexecdir)/$$j || exit 1; \ + done +endif + +ifneq ($(reverse_private_libexecdir_rel),$(reverse_build_private_libexecdir_rel)) +ifeq ($(OS), Darwin) +ifneq ($(DARWIN_FRAMEWORK),1) + for j in $(JL_PRIVATE_EXES) ; do \ + [ $$j = 7z ] && continue; \ + [ -L $(DESTDIR)$(private_libexecdir)/$$j ] && continue; \ + install_name_tool -rpath @executable_path/$(reverse_build_private_libexecdir_rel) @executable_path/$(reverse_private_libexecdir_rel) $(DESTDIR)$(private_libexecdir)/$$j || exit 1; \ + done +endif +else ifneq (,$(findstring $(OS),Linux FreeBSD)) + for j in $(JL_PRIVATE_EXES) ; do \ + [ $$j = 7z ] && continue; \ + [ -L $(DESTDIR)$(private_libexecdir)/$$j ] && continue; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_private_libexecdir_rel)' $(DESTDIR)$(private_libexecdir)/$$j || exit 1; \ + done +endif endif # Fix rpaths for dependencies. This should be fixed in BinaryBuilder later. ifeq ($(OS), Linux) - -$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $(DESTDIR)$(private_shlibdir)/libLLVM.$(SHLIB_EXT) + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $(DESTDIR)$(private_shlibdir)/libLLVM.$(SHLIB_EXT) endif ifneq ($(LOADER_BUILD_DEP_LIBS),$(LOADER_INSTALL_DEP_LIBS)) # Next, overwrite relative path to libjulia-internal in our loader if $$(LOADER_BUILD_DEP_LIBS) != $$(LOADER_INSTALL_DEP_LIBS) @@ -511,7 +560,7 @@ ifeq ($(OS),FreeBSD) # don't set libgfortran's RPATH, it won't be able to find its friends on systems # that don't have the exact GCC port installed used for the build. for lib in $(DESTDIR)$(private_libdir)/libgfortran*$(SHLIB_EXT)*; do \ - $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $$lib; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $$lib || exit 1; \ done endif @@ -555,7 +604,7 @@ endif ifeq ($(OS), WINNT) cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe endif - cd $(BUILDROOT) && $(TAR) zcvf $(JULIA_BINARYDIST_FILENAME).tar.gz julia-$(JULIA_COMMIT) + cd $(BUILDROOT) && $(TAR) -zcvf $(JULIA_BINARYDIST_FILENAME).tar.gz julia-$(JULIA_COMMIT) exe: @@ -650,7 +699,7 @@ distcleanall: cleanall @-$(MAKE) -C $(BUILDROOT)/doc cleanall .FORCE: -.PHONY: .FORCE default debug release check-whitespace release-candidate \ +.PHONY: .FORCE default debug release check-whitespace fix-whitespace release-candidate \ julia-debug julia-release julia-stdlib julia-deps julia-deps-libs \ julia-cli-release julia-cli-debug julia-src-release julia-src-debug \ julia-symlink julia-base julia-sysimg julia-sysimg-ji julia-sysimg-release julia-sysimg-debug \ diff --git a/NEWS.md b/NEWS.md index 2aec4f53588a6..ea4d8337beeb6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,8 +4,21 @@ Julia v1.13 Release Notes New language features --------------------- + - New `Base.@acquire` macro for a non-closure version of `Base.acquire(f, s::Base.Semaphore)`, like `@lock`. ([#56845]) + - New `nth` function to access the `n`-th element of a generic iterable. ([#56580]) + - New `@__FUNCTION__` macro to refer to the innermost enclosing function. ([#58940]) + - The character U+1F8B2 🢲 (RIGHTWARDS ARROW WITH LOWER HOOK), newly added by Unicode 16, + is now a valid operator with arrow precedence, accessible as `\hookunderrightarrow` at the REPL. + ([JuliaLang/JuliaSyntax.jl#525], [#57143]) + Language changes ---------------- +* `mod(x::AbstractFloat, -Inf)` now returns `x` (as long as `x` is finite), this aligns with C standard and +is considered a bug fix ([#47102]) + + - The `hash` algorithm and its values have changed. Most `hash` specializations will remain correct and require no action. Types that reimplement the core hashing logic independently, such as some third-party string packages do, may require a migration to the new algorithm. ([#57509]) + +* Indexless `getindex` and `setindex!` (i.e. `A[]`) on `ReinterpretArray` now correctly throw a `BoundsError` when there is more than one element. ([#58814]) Compiler/Runtime improvements ----------------------------- @@ -13,19 +26,38 @@ Compiler/Runtime improvements Command-line option changes --------------------------- +* The option `--sysimage-native-code=no` has been deprecated. +* The `JULIA_CPU_TARGET` environment variable now supports a `sysimage` keyword to match (or extend) the CPU target used to build the current system image ([#58970]). + Multi-threading changes ----------------------- +* A new `AbstractSpinLock` is defined with `SpinLock <: AbstractSpinLock` ([#55944]). +* A new `PaddedSpinLock <: AbstractSpinLock` is defined. It has extra padding to avoid false sharing ([#55944]). +* New types are defined to handle the pattern of code that must run once per process, called + a `OncePerProcess{T}` type, which allows defining a function that should be run exactly once + the first time it is called, and then always return the same result value of type `T` + every subsequent time afterwards. There are also `OncePerThread{T}` and `OncePerTask{T}` types for + similar usage with threads or tasks. ([#TBD]) + Build system changes -------------------- New library functions --------------------- +* `ispositive(::Real)` and `isnegative(::Real)` are provided for performance and convenience ([#53677]). +* Exporting function `fieldindex` to get the index of a struct's field ([#58119]). +* `Base.donotdelete` is now public. It prevents deadcode elemination of its arguments ([#55774]). +* `Sys.sysimage_target()` returns the CPU target string used to build the current system image ([#58970]). + New library features -------------------- -`sort(keys(::Dict))` and `sort(values(::Dict))` now automatically collect, they previously threw ([#56978]). +* `fieldoffset` now also accepts the field name as a symbol as `fieldtype` already did ([#58100]). +* `sort(keys(::Dict))` and `sort(values(::Dict))` now automatically collect, they previously threw ([#56978]). +* `Base.AbstractOneTo` is added as a supertype of one-based axes, with `Base.OneTo` as its subtype ([#56902]). +* `takestring!(::IOBuffer)` removes the content from the buffer, returning the content as a `String`. Standard library changes ------------------------ @@ -36,14 +68,26 @@ Standard library changes #### Profile +#### Random + +* `randperm!` and `randcycle!` now support non-`Array` `AbstractArray` inputs, assuming they are mutable and their indices are one-based ([#58596]). + #### REPL +* The display of `AbstractChar`s in the main REPL mode now includes LaTeX input information like what is shown in help mode ([#58181]). +* Display of repeated frames and cycles in stack traces has been improved by bracketing them in the trace and treating them consistently ([#55841]). + #### Test * Test failures when using the `@test` macro now show evaluated arguments for all function calls ([#57825], [#57839]). +* Transparent test sets (`@testset let`) now show context when tests error ([#58727]). +* `@test_throws` now supports a three-argument form `@test_throws ExceptionType pattern expr` to test both exception type and message pattern in one call ([#59117]). #### InteractiveUtils +* Introspection utilities such as `@code_typed`, `@which` and `@edit` now accept type annotations as substitutes for values, recognizing forms such as `f(1, ::Float64, 3)` or even `sum(::Vector{T}; init = ::T) where {T<:Real}`. Type-annotated variables as in `f(val::Int; kw::Float64)` are not evaluated if the type annotation provides the necessary information, making this syntax compatible with signatures found in stacktraces ([#57909], [#58222]). +* Code introspection macros such as `@code_lowered` and `@code_typed` now have a much better support for broadcasting expressions, including broadcasting assignments of the form `x .+= f(y)` ([#58349]). + External dependencies --------------------- diff --git a/README.md b/README.md index 021322336d286..cbd1b11e982cb 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ and installing Julia, below. ## Resources - **Homepage:** -- **Binaries:** +- **Install:** - **Source code:** - **Documentation:** - **Packages:** @@ -63,17 +63,22 @@ helpful to start contributing to the Julia codebase. ## Binary Installation -If you would rather not compile the latest Julia from source, -platform-specific tarballs with pre-compiled binaries are also -[available for download](https://julialang.org/downloads/). The -downloads page also provides details on the -[different tiers of support](https://julialang.org/downloads/#supported_platforms) -for OS and platform combinations. - -If everything works correctly, you will see a Julia banner and an -interactive prompt into which you can enter expressions for -evaluation. You can read about [getting -started](https://docs.julialang.org/en/v1/manual/getting-started/) in the manual. +The recommended way of installing Julia is to use `juliaup` which will install +the latest stable `julia` for you and help keep it up to date. It can also let +you install and run different Julia versions simultaneously. Instructions for +this can be found [here](https://julialang.org/install/). If you want to manually +download specific Julia binaries, you can find those on the [downloads +page](https://julialang.org/downloads/). The downloads page also provides +details on the [different tiers of +support](https://julialang.org/downloads/#supported_platforms) for OS and +platform combinations. + +If everything works correctly, you will get a `julia` program and when you run +it in a terminal or command prompt, you will see a Julia banner and an +interactive prompt into which you can enter expressions for evaluation. You can +read about [getting +started](https://docs.julialang.org/en/v1/manual/getting-started/) in the +manual. **Note**: Although some OS package managers provide Julia, such installations are neither maintained nor endorsed by the Julia @@ -92,7 +97,7 @@ and then use the command prompt to change into the resulting julia directory. By Julia. However, most users should use the [most recent stable version](https://github.com/JuliaLang/julia/releases) of Julia. You can get this version by running: - git checkout v1.11.2 + git checkout v1.11.5 To build the `julia` executable, run `make` from within the julia directory. diff --git a/THIRDPARTY.md b/THIRDPARTY.md index f3f59ca4ff3f7..06973b0163e5e 100644 --- a/THIRDPARTY.md +++ b/THIRDPARTY.md @@ -67,6 +67,7 @@ Julia bundles the following external programs and libraries: - [7-Zip](https://www.7-zip.org/license.txt) - [ZLIB](https://zlib.net/zlib_license.html) +- [ZSTD](https://github.com/facebook/zstd/blob/v1.5.7/LICENSE) On some platforms, distributions of Julia contain SSL certificate authority certificates, released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License). diff --git a/base/Base.jl b/base/Base.jl index 3bb8988d36804..9d510b5c5d47c 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -23,7 +23,7 @@ include(strcat(BUILDROOT, "version_git.jl")) # include($BUILDROOT/base/version_g # a slightly more verbose fashion than usual, because we're running so early. let os = ccall(:jl_get_UNAME, Any, ()) if os === :Darwin || os === :Apple - if Base.DARWIN_FRAMEWORK + if DARWIN_FRAMEWORK push!(DL_LOAD_PATH, "@loader_path/Frameworks") end push!(DL_LOAD_PATH, "@loader_path") @@ -127,6 +127,12 @@ include("missing.jl") # version include("version.jl") +#= +isdebugbuild is defined here as this is imported in libdl.jl (included in libc.jl) +The method is added in util.jl +=# +function isdebugbuild end + # system & environment include("sysinfo.jl") include("libc.jl") @@ -261,7 +267,9 @@ include("uuid.jl") include("pkgid.jl") include("toml_parser.jl") include("linking.jl") +module StaticData include("staticdata.jl") +end include("loading.jl") # BinaryPlatforms, used by Artifacts. Needs `Sort`. @@ -306,8 +314,8 @@ a_method_to_overwrite_in_test() = inferencebarrier(1) (this::IncludeInto)(mapexpr::Function, fname::AbstractString) = include(mapexpr, this.m, fname) # Compatibility with when Compiler was in Core -@eval Core const Compiler = Main.Base.Compiler -@eval Compiler const fl_parse = Core.Main.Base.fl_parse +@eval Core const Compiler = $Base.Compiler +@eval Compiler const fl_parse = $Base.fl_parse # External libraries vendored into Base Core.println("JuliaSyntax/src/JuliaSyntax.jl") @@ -323,13 +331,13 @@ if is_primary_base_module # Profiling helper # triggers printing the report and (optionally) saving a heap snapshot after a SIGINFO/SIGUSR1 profile request # Needs to be in Base because Profile is no longer loaded on boot -function profile_printing_listener(cond::Base.AsyncCondition) +function profile_printing_listener(cond::AsyncCondition) profile = nothing try while _trywait(cond) profile = @something(profile, require_stdlib(PkgId(UUID("9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"), "Profile")))::Module invokelatest(profile.peek_report[]) - if Base.get_bool_env("JULIA_PROFILE_PEEK_HEAP_SNAPSHOT", false) === true + if get_bool_env("JULIA_PROFILE_PEEK_HEAP_SNAPSHOT", false) === true println(stderr, "Saving heap snapshot...") fname = invokelatest(profile.take_heap_snapshot) println(stderr, "Heap snapshot saved to `$(fname)`") @@ -344,8 +352,8 @@ function profile_printing_listener(cond::Base.AsyncCondition) end function start_profile_listener() - cond = Base.AsyncCondition() - Base.uv_unref(cond.handle) + cond = AsyncCondition() + uv_unref(cond.handle) t = errormonitor(Threads.@spawn(profile_printing_listener(cond))) atexit() do # destroy this callback when exiting @@ -405,7 +413,6 @@ end const _compiler_require_dependencies = Any[] @Core.latestworld for i = 1:length(_included_files) - isassigned(_included_files, i) || continue (mod, file) = _included_files[i] if mod === Compiler || parentmodule(mod) === Compiler || endswith(file, "/Compiler.jl") _include_dependency!(_compiler_require_dependencies, true, mod, file, true, false) @@ -421,7 +428,3 @@ end @assert length(_compiler_require_dependencies) >= 15 end - -# Ensure this file is also tracked -@assert !isassigned(_included_files, 1) -_included_files[1] = (parentmodule(Base), abspath(@__FILE__)) diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index 7c63e169e9462..ef448a02a15e9 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -1,9 +1,11 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -baremodule Base +module Base -using Core -using Core.Intrinsics, Core.IR +Core._import(Base, Core, :_eval_import, :_eval_import, true) +Core._import(Base, Core, :_eval_using, :_eval_using, true) + +using .Core.Intrinsics, .Core.IR # to start, we're going to use a very simple definition of `include` # that doesn't require any function (except what we can get from the `Core` top-module) @@ -158,11 +160,40 @@ if false println(io::IO, x...) = Core.println(io, x...) end +## Load essential files and libraries +include("essentials.jl") + +# Because lowering inserts direct references, it is mandatory for this binding +# to exist before we start inferring code. +function string end +import Core: String + +# For OS specific stuff +# We need to strcat things here, before strings are really defined +function strcat(x::String, y::String) + out = ccall(:jl_alloc_string, Ref{String}, (Int,), Core.sizeof(x) + Core.sizeof(y)) + gc_x = @_gc_preserve_begin(x) + gc_y = @_gc_preserve_begin(y) + gc_out = @_gc_preserve_begin(out) + out_ptr = unsafe_convert(Ptr{UInt8}, out) + unsafe_copyto!(out_ptr, unsafe_convert(Ptr{UInt8}, x), Core.sizeof(x)) + unsafe_copyto!(out_ptr + Core.sizeof(x), unsafe_convert(Ptr{UInt8}, y), Core.sizeof(y)) + @_gc_preserve_end(gc_x) + @_gc_preserve_end(gc_y) + @_gc_preserve_end(gc_out) + return out +end + + """ time_ns()::UInt64 -Get the time in nanoseconds relative to some arbitrary time in the past. The primary use is for measuring the elapsed time -between two moments in time. +Get the time in nanoseconds relative to some machine-specific arbitrary time in the past. +The primary use is for measuring elapsed times during program execution. The return value is guaranteed to +be monotonic (mod 2⁶⁴) while the system is running, and is unaffected by clock drift or changes to local calendar time, +but it may change arbitrarily across system reboots or suspensions. + +(Although the returned time is always in nanoseconds, the timing resolution is platform-dependent.) """ time_ns() = ccall(:jl_hrtime, UInt64, ()) @@ -172,8 +203,6 @@ const _DOCS_ALIASING_WARNING = """ Behavior can be unexpected when any mutated argument shares memory with any other argument. """ -## Load essential files and libraries -include("essentials.jl") include("ctypes.jl") include("gcutils.jl") include("generator.jl") @@ -189,7 +218,7 @@ function Core.kwcall(kwargs::NamedTuple, ::typeof(invoke), f, T, args...) return invoke(Core.kwcall, T, kwargs, f, args...) end # invoke does not have its own call cache, but kwcall for invoke does -setfield!(typeof(invoke).name.mt, :max_args, 3, :monotonic) # invoke, f, T, args... +setfield!(typeof(invoke).name, :max_args, Int32(3), :monotonic) # invoke, f, T, args... # define applicable(f, T, args...; kwargs...), without kwargs wrapping # to forward to applicable @@ -223,7 +252,7 @@ function Core.kwcall(kwargs::NamedTuple, ::typeof(invokelatest), f, args...) @inline return Core.invokelatest(Core.kwcall, kwargs, f, args...) end -setfield!(typeof(invokelatest).name.mt, :max_args, 2, :monotonic) # invokelatest, f, args... +setfield!(typeof(invokelatest).name, :max_args, Int32(2), :monotonic) # invokelatest, f, args... """ invoke_in_world(world, f, args...; kwargs...) @@ -236,7 +265,7 @@ support libraries, etc. In these cases it can be useful to prevent unwanted method invalidation and recompilation latency, and to prevent the user from breaking supporting infrastructure by mistake. -The current world age can be queried using [`Base.get_world_counter()`](@ref) +The global world age can be queried using [`Base.get_world_counter()`](@ref) and stored for later use within the lifetime of the current Julia session, or when serializing and reloading the system image. @@ -257,7 +286,7 @@ function Core.kwcall(kwargs::NamedTuple, ::typeof(invoke_in_world), world::UInt, @inline return Core.invoke_in_world(world, Core.kwcall, kwargs, f, args...) end -setfield!(typeof(invoke_in_world).name.mt, :max_args, 3, :monotonic) # invoke_in_world, world, f, args... +setfield!(typeof(invoke_in_world).name, :max_args, Int32(3), :monotonic) # invoke_in_world, world, f, args... # core operations & types include("promotion.jl") @@ -284,7 +313,6 @@ include("rounding.jl") include("float.jl") # Lazy strings -import Core: String include("strings/lazy.jl") function cld end @@ -319,24 +347,9 @@ include("ordering.jl") using .Order include("coreir.jl") +include("module.jl") include("invalidation.jl") -# Because lowering inserts direct references, it is mandatory for this binding -# to exist before we start inferring code. -function string end - -# For OS specific stuff -# We need to strcat things here, before strings are really defined -function strcat(x::String, y::String) - out = ccall(:jl_alloc_string, Ref{String}, (Csize_t,), Core.sizeof(x) + Core.sizeof(y)) - GC.@preserve x y out begin - out_ptr = unsafe_convert(Ptr{UInt8}, out) - unsafe_copyto!(out_ptr, unsafe_convert(Ptr{UInt8}, x), Core.sizeof(x)) - unsafe_copyto!(out_ptr + Core.sizeof(x), unsafe_convert(Ptr{UInt8}, y), Core.sizeof(y)) - end - return out -end - BUILDROOT::String = "" DATAROOT::String = "" const DL_LOAD_PATH = String[] @@ -370,10 +383,15 @@ const _return_type = Compiler.return_type # Enable compiler Compiler.bootstrap!() -include("flparse.jl") +include("flfrontend.jl") Core._setparser!(fl_parse) +Core._setlowerer!(fl_lower) # Further definition of Base will happen in Base.jl if loaded. -end # baremodule Base +# Ensure this file is also tracked +@assert !isassigned(_included_files, 1) +_included_files[1] = (@__MODULE__, ccall(:jl_prepend_cwd, Any, (Any,), "Base_compiler.jl")) + +end # module Base using .Base diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 46907af4aacf6..adc104dadb90b 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -321,11 +321,13 @@ eachindex(itrs...) = keys(itrs...) eachindex(A::AbstractVector) = (@inline(); axes1(A)) -@noinline function throw_eachindex_mismatch_indices(::IndexLinear, inds...) - throw(DimensionMismatch("all inputs to eachindex must have the same indices, got $(join(inds, ", ", " and "))")) -end -@noinline function throw_eachindex_mismatch_indices(::IndexCartesian, inds...) - throw(DimensionMismatch("all inputs to eachindex must have the same axes, got $(join(inds, ", ", " and "))")) +# we unroll the join for easier inference +_join_comma_and(indsA, indsB) = LazyString(indsA, " and ", indsB) +_join_comma_and(indsA, indsB, indsC...) = LazyString(indsA, ", ", _join_comma_and(indsB, indsC...)) +@noinline function throw_eachindex_mismatch_indices(indices_str, indsA, indsBs...) + throw(DimensionMismatch( + LazyString("all inputs to eachindex must have the same ", indices_str, ", got ", + _join_comma_and(indsA, indsBs...)))) end """ @@ -385,20 +387,17 @@ function eachindex(A::AbstractArray, B::AbstractArray...) @inline eachindex(IndexStyle(A,B...), A, B...) end +eachindex(::IndexLinear, A::Union{Array, Memory}) = unchecked_oneto(length(A)) eachindex(::IndexLinear, A::AbstractArray) = (@inline; oneto(length(A))) eachindex(::IndexLinear, A::AbstractVector) = (@inline; axes1(A)) function eachindex(::IndexLinear, A::AbstractArray, B::AbstractArray...) @inline indsA = eachindex(IndexLinear(), A) - _all_match_first(X->eachindex(IndexLinear(), X), indsA, B...) || - throw_eachindex_mismatch_indices(IndexLinear(), eachindex(A), eachindex.(B)...) + indsBs = map(X -> eachindex(IndexLinear(), X), B) + all(==(indsA), indsBs) || + throw_eachindex_mismatch_indices("indices", indsA, indsBs...) indsA end -function _all_match_first(f::F, inds, A, B...) where F<:Function - @inline - (inds == f(A)) & _all_match_first(f, inds, B...) -end -_all_match_first(f::F, inds) where F<:Function = true # keys with an IndexStyle keys(s::IndexStyle, A::AbstractArray, B::AbstractArray...) = eachindex(s, A, B...) @@ -798,7 +797,7 @@ julia> similar(1:10, 1, 4) Conversely, `similar(trues(10,10), 2)` returns an uninitialized `BitVector` with two elements since `BitArray`s are both mutable and can support 1-dimensional arrays: -```julia-repl +```jldoctest; filter = r"[01]" julia> similar(trues(10,10), 2) 2-element BitVector: 0 @@ -818,15 +817,18 @@ julia> similar(falses(10), Float64, 2, 4) See also: [`undef`](@ref), [`isassigned`](@ref). """ similar(a::AbstractArray{T}) where {T} = similar(a, T) -similar(a::AbstractArray, ::Type{T}) where {T} = similar(a, T, to_shape(axes(a))) -similar(a::AbstractArray{T}, dims::Tuple) where {T} = similar(a, T, to_shape(dims)) -similar(a::AbstractArray{T}, dims::DimOrInd...) where {T} = similar(a, T, to_shape(dims)) -similar(a::AbstractArray, ::Type{T}, dims::DimOrInd...) where {T} = similar(a, T, to_shape(dims)) +similar(a::AbstractArray, ::Type{T}) where {T} = similar(a, T, axes(a)) +similar(a::AbstractArray{T}, dims::Tuple) where {T} = similar(a, T, dims) +similar(a::AbstractArray{T}, dims::DimOrInd...) where {T} = similar(a, T, dims) +similar(a::AbstractArray, ::Type{T}, dims::DimOrInd...) where {T} = similar(a, T, dims) # Similar supports specifying dims as either Integers or AbstractUnitRanges or any mixed combination # thereof. Ideally, we'd just convert Integers to OneTos and then call a canonical method with the axes, # but we don't want to require all AbstractArray subtypes to dispatch on Base.OneTo. So instead we # define this method to convert supported axes to Ints, with the expectation that an offset array # package will define a method with dims::Tuple{Union{Integer, UnitRange}, Vararg{Union{Integer, UnitRange}}} +similar(a::AbstractArray, ::Type{T}, dims::Tuple{Union{Integer, AbstractOneTo}, Vararg{Union{Integer, AbstractOneTo}}}) where {T} = similar(a, T, to_shape(dims)) +# legacy method for packages that specialize similar(A::AbstractArray, ::Type{T}, dims::Tuple{Union{Integer, OneTo, CustomAxis}, Vararg{Union{Integer, OneTo, CustomAxis}}} +# leaving this method in ensures that Base owns the more specific method similar(a::AbstractArray, ::Type{T}, dims::Tuple{Union{Integer, OneTo}, Vararg{Union{Integer, OneTo}}}) where {T} = similar(a, T, to_shape(dims)) # similar creates an Array by default similar(a::AbstractArray, ::Type{T}, dims::Dims{N}) where {T,N} = Array{T,N}(undef, dims) @@ -837,7 +839,9 @@ to_shape(dims::DimsOrInds) = map(to_shape, dims)::DimsOrInds # each dimension to_shape(i::Int) = i to_shape(i::Integer) = Int(i) -to_shape(r::OneTo) = Int(last(r)) +to_shape(r::AbstractOneTo) = _to_shape(last(r)) +_to_shape(x::Integer) = to_shape(x) +_to_shape(x) = Int(x) to_shape(r::AbstractUnitRange) = r """ @@ -863,6 +867,8 @@ would create a 1-dimensional logical array whose indices match those of the columns of `A`. """ similar(::Type{T}, dims::DimOrInd...) where {T<:AbstractArray} = similar(T, dims) +similar(::Type{T}, shape::Tuple{Union{Integer, AbstractOneTo}, Vararg{Union{Integer, AbstractOneTo}}}) where {T<:AbstractArray} = similar(T, to_shape(shape)) +# legacy method for packages that specialize similar(::Type{T}, dims::Tuple{Union{Integer, OneTo, CustomAxis}, Vararg{Union{Integer, OneTo, CustomAxis}}) similar(::Type{T}, shape::Tuple{Union{Integer, OneTo}, Vararg{Union{Integer, OneTo}}}) where {T<:AbstractArray} = similar(T, to_shape(shape)) similar(::Type{T}, dims::Dims) where {T<:AbstractArray} = T(undef, dims) @@ -1229,11 +1235,19 @@ oneunit(x::AbstractMatrix{T}) where {T} = _one(oneunit(T), x) # While the definitions for IndexLinear are all simple enough to inline on their # own, IndexCartesian's CartesianIndices is more complicated and requires explicit # inlining. -function iterate(A::AbstractArray, state=(eachindex(A),)) +iterate_starting_state(A) = iterate_starting_state(A, IndexStyle(A)) +iterate_starting_state(A, ::IndexLinear) = firstindex(A) +iterate_starting_state(A, ::IndexStyle) = (eachindex(A),) +@inline iterate(A::AbstractArray, state = iterate_starting_state(A)) = _iterate(A, state) +@inline function _iterate(A::AbstractArray, state::Tuple) y = iterate(state...) y === nothing && return nothing A[y[1]], (state[1], tail(y)...) end +@inline function _iterate(A::AbstractArray, state::Integer) + checkbounds(Bool, A, state) || return nothing + A[state], state + one(state) +end isempty(a::AbstractArray) = (length(a) == 0) @@ -2553,16 +2567,23 @@ function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as end # discover number of rows or columns + # d1 dimension is increased by 1 to appropriately handle 0-length arrays for i ∈ 1:dims[d1] outdims[d1] += cat_size(as[i], d1) end + # adjustment to handle 0-length arrays + first_dim_zero = outdims[d1] == 0 + if first_dim_zero + outdims[d1] = dims[d1] + end + currentdims = zeros(Int, N) blockcount = 0 elementcount = 0 for i ∈ eachindex(as) elementcount += cat_length(as[i]) - currentdims[d1] += cat_size(as[i], d1) + currentdims[d1] += first_dim_zero ? 1 : cat_size(as[i], d1) if currentdims[d1] == outdims[d1] currentdims[d1] = 0 for d ∈ (d2, 3:N...) @@ -2590,6 +2611,10 @@ function _typed_hvncat_dims(::Type{T}, dims::NTuple{N, Int}, row_first::Bool, as throw(DimensionMismatch("argument $i has too many elements along axis $d1")) end end + # restore 0-length adjustment + if first_dim_zero + outdims[d1] = 0 + end outlen = prod(outdims) elementcount == outlen || @@ -3412,12 +3437,19 @@ function ith_all(i, as) end function map_n!(f::F, dest::AbstractArray, As) where F - idxs1 = LinearIndices(As[1]) - @boundscheck LinearIndices(dest) == idxs1 && all(x -> LinearIndices(x) == idxs1, As) - for i = idxs1 - @inbounds I = ith_all(i, As) - val = f(I...) - @inbounds dest[i] = val + idxs = LinearIndices(dest) + if all(x -> LinearIndices(x) == idxs, As) + for i in idxs + @inbounds as = ith_all(i, As) + val = f(as...) + @inbounds dest[i] = val + end + else + for (i, Is...) in zip(eachindex(dest), map(eachindex, As)...) + as = ntuple(j->getindex(As[j], Is[j]), length(As)) + val = f(as...) + dest[i] = val + end end return dest end @@ -3547,81 +3579,6 @@ pushfirst!(A, a, b, c...) = pushfirst!(pushfirst!(A, c...), a, b) # sizehint! does not nothing by default sizehint!(a::AbstractVector, _) = a -## hashing AbstractArray ## - -const hash_abstractarray_seed = UInt === UInt64 ? 0x7e2d6fb6448beb77 : 0xd4514ce5 -function hash(A::AbstractArray, h::UInt) - h += hash_abstractarray_seed - # Axes are themselves AbstractArrays, so hashing them directly would stack overflow - # Instead hash the tuple of firsts and lasts along each dimension - h = hash(map(first, axes(A)), h) - h = hash(map(last, axes(A)), h) - - # For short arrays, it's not worth doing anything complicated - if length(A) < 8192 - for x in A - h = hash(x, h) - end - return h - end - - # Goal: Hash approximately log(N) entries with a higher density of hashed elements - # weighted towards the end and special consideration for repeated values. Colliding - # hashes will often subsequently be compared by equality -- and equality between arrays - # works elementwise forwards and is short-circuiting. This means that a collision - # between arrays that differ by elements at the beginning is cheaper than one where the - # difference is towards the end. Furthermore, choosing `log(N)` arbitrary entries from a - # sparse array will likely only choose the same element repeatedly (zero in this case). - - # To achieve this, we work backwards, starting by hashing the last element of the - # array. After hashing each element, we skip `fibskip` elements, where `fibskip` - # is pulled from the Fibonacci sequence -- Fibonacci was chosen as a simple - # ~O(log(N)) algorithm that ensures we don't hit a common divisor of a dimension - # and only end up hashing one slice of the array (as might happen with powers of - # two). Finally, we find the next distinct value from the one we just hashed. - - # This is a little tricky since skipping an integer number of values inherently works - # with linear indices, but `findprev` uses `keys`. Hoist out the conversion "maps": - ks = keys(A) - key_to_linear = LinearIndices(ks) # Index into this map to compute the linear index - linear_to_key = vec(ks) # And vice-versa - - # Start at the last index - keyidx = last(ks) - linidx = key_to_linear[keyidx] - fibskip = prevfibskip = oneunit(linidx) - first_linear = first(LinearIndices(linear_to_key)) - n = 0 - while true - n += 1 - # Hash the element - elt = A[keyidx] - h = hash(keyidx=>elt, h) - - # Skip backwards a Fibonacci number of indices -- this is a linear index operation - linidx = key_to_linear[keyidx] - linidx < fibskip + first_linear && break - linidx -= fibskip - keyidx = linear_to_key[linidx] - - # Only increase the Fibonacci skip once every N iterations. This was chosen - # to be big enough that all elements of small arrays get hashed while - # obscenely large arrays are still tractable. With a choice of N=4096, an - # entirely-distinct 8000-element array will have ~75% of its elements hashed, - # with every other element hashed in the first half of the array. At the same - # time, hashing a `typemax(Int64)`-length Float64 range takes about a second. - if rem(n, 4096) == 0 - fibskip, prevfibskip = fibskip + prevfibskip, fibskip - end - - # Find a key index with a value distinct from `elt` -- might be `keyidx` itself - keyidx = findprev(!isequal(elt), A, keyidx) - keyidx === nothing && break - end - - return h -end - # The semantics of `collect` are weird. Better to write our own function rest(a::AbstractArray{T}, state...) where {T} v = Vector{T}(undef, 0) @@ -3630,7 +3587,6 @@ function rest(a::AbstractArray{T}, state...) where {T} return foldl(push!, Iterators.rest(a, state...), init=v) end - ## keepat! ## # NOTE: since these use `@inbounds`, they are actually only intended for Vector and BitVector diff --git a/base/abstractdict.jl b/base/abstractdict.jl index 786720bb6359f..31c00c5b4f8d5 100644 --- a/base/abstractdict.jl +++ b/base/abstractdict.jl @@ -92,7 +92,7 @@ But `keys(a)`, `values(a)` and `pairs(a)` all iterate `a` and return the elements in the same order. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+'\\S'.*\$"m julia> D = Dict('a'=>2, 'b'=>3) Dict{Char, Int64} with 2 entries: 'a' => 2 @@ -118,7 +118,7 @@ But `keys(a)`, `values(a)` and `pairs(a)` all iterate `a` and return the elements in the same order. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+(\\s+=>\\s+\\d)?\$"m julia> D = Dict('a'=>2, 'b'=>3) Dict{Char, Int64} with 2 entries: 'a' => 2 @@ -332,7 +332,7 @@ value for that key will be the value it has in the last collection listed. See also [`mergewith`](@ref) for custom handling of values with the same key. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+\\s+=>\\s+\\S+\$"m julia> a = Dict("foo" => 0.0, "bar" => 42.0) Dict{String, Float64} with 2 entries: "bar" => 42.0 @@ -377,7 +377,7 @@ Method `merge(combine::Union{Function,Type}, args...)` as an alias of `mergewith` requires Julia 1.5 or later. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+\\s+=>\\s+\\S+\$"m julia> a = Dict("foo" => 0.0, "bar" => 42.0) Dict{String, Float64} with 2 entries: "bar" => 42.0 diff --git a/base/accumulate.jl b/base/accumulate.jl index 2748a4da481fa..c155ecfb4f75f 100644 --- a/base/accumulate.jl +++ b/base/accumulate.jl @@ -5,12 +5,14 @@ # it does double the number of operations compared to accumulate, # though for cheap operations like + this does not have much impact (20%) function _accumulate_pairwise!(op::Op, c::AbstractVector{T}, v::AbstractVector, s, i1, n)::T where {T,Op} - @inbounds if n < 128 - s_ = v[i1] - c[i1] = op(s, s_) + if n < 128 + @inbounds s_ = v[i1] + ci1 = op(s, s_) + @inbounds c[i1] = ci1 for i = i1+1:i1+n-1 - s_ = op(s_, v[i]) - c[i] = op(s, s_) + s_ = op(s_, @inbounds(v[i])) + ci = op(s, s_) + @inbounds c[i] = ci end else n2 = n >> 1 @@ -26,7 +28,8 @@ function accumulate_pairwise!(op::Op, result::AbstractVector, v::AbstractVector) n = length(li) n == 0 && return result i1 = first(li) - @inbounds result[i1] = v1 = reduce_first(op,v[i1]) + v1 = reduce_first(op, @inbounds(v[i1])) + @inbounds result[i1] = v1 n == 1 && return result _accumulate_pairwise!(op, result, v, v1, i1+1, n-1) return result @@ -378,16 +381,16 @@ function _accumulate!(op, B, A, dims::Integer, init::Union{Nothing, Some}) # We can accumulate to a temporary variable, which allows # register usage and will be slightly faster ind1 = inds_t[1] - @inbounds for I in CartesianIndices(tail(inds_t)) + for I in CartesianIndices(tail(inds_t)) if init === nothing - tmp = reduce_first(op, A[first(ind1), I]) + tmp = reduce_first(op, @inbounds(A[first(ind1), I])) else - tmp = op(something(init), A[first(ind1), I]) + tmp = op(something(init), @inbounds(A[first(ind1), I])) end - B[first(ind1), I] = tmp + @inbounds B[first(ind1), I] = tmp for i_1 = first(ind1)+1:last(ind1) - tmp = op(tmp, A[i_1, I]) - B[i_1, I] = tmp + tmp = op(tmp, @inbounds(A[i_1, I])) + @inbounds B[i_1, I] = tmp end end else @@ -401,12 +404,15 @@ end @noinline function _accumulaten!(op, B, A, R1, ind, R2, init::Nothing) # Copy the initial element in each 1d vector along dimension `dim` ii = first(ind) - @inbounds for J in R2, I in R1 - B[I, ii, J] = reduce_first(op, A[I, ii, J]) + for J in R2, I in R1 + tmp = reduce_first(op, @inbounds(A[I, ii, J])) + @inbounds B[I, ii, J] = tmp end # Accumulate - @inbounds for J in R2, i in first(ind)+1:last(ind), I in R1 - B[I, i, J] = op(B[I, i-1, J], A[I, i, J]) + for J in R2, i in first(ind)+1:last(ind), I in R1 + @inbounds Bv, Av = B[I, i-1, J], A[I, i, J] + tmp = op(Bv, Av) + @inbounds B[I, i, J] = tmp end B end @@ -414,12 +420,15 @@ end @noinline function _accumulaten!(op, B, A, R1, ind, R2, init::Some) # Copy the initial element in each 1d vector along dimension `dim` ii = first(ind) - @inbounds for J in R2, I in R1 - B[I, ii, J] = op(something(init), A[I, ii, J]) + for J in R2, I in R1 + tmp = op(something(init), @inbounds(A[I, ii, J])) + @inbounds B[I, ii, J] = tmp end # Accumulate - @inbounds for J in R2, i in first(ind)+1:last(ind), I in R1 - B[I, i, J] = op(B[I, i-1, J], A[I, i, J]) + for J in R2, i in first(ind)+1:last(ind), I in R1 + @inbounds Bv, Av = B[I, i-1, J], A[I, i, J] + tmp = op(Bv, Av) + @inbounds B[I, i, J] = tmp end B end @@ -433,10 +442,10 @@ function _accumulate1!(op, B, v1, A::AbstractVector, dim::Integer) cur_val = v1 B[i1] = cur_val next = iterate(inds, state) - @inbounds while next !== nothing + while next !== nothing (i, state) = next - cur_val = op(cur_val, A[i]) - B[i] = cur_val + cur_val = op(cur_val, @inbounds(A[i])) + @inbounds B[i] = cur_val next = iterate(inds, state) end return B diff --git a/base/array.jl b/base/array.jl index c56a3757b0103..d62436ca30497 100644 --- a/base/array.jl +++ b/base/array.jl @@ -316,10 +316,16 @@ end # It is also mitigated by using a constant string. _throw_argerror(s) = (@noinline; throw(ArgumentError(s))) -copyto!(dest::Array, src::Array) = copyto!(dest, 1, src, 1, length(src)) +_copyto2arg!(dest, src) = copyto!(dest, firstindex(dest), src, firstindex(src), length(src)) + +copyto!(dest::Array, src::Array) = _copyto2arg!(dest, src) +copyto!(dest::Array, src::Memory) = _copyto2arg!(dest, src) +copyto!(dest::Memory, src::Array) = _copyto2arg!(dest, src) # also to avoid ambiguities in packages -copyto!(dest::Array{T}, src::Array{T}) where {T} = copyto!(dest, 1, src, 1, length(src)) +copyto!(dest::Array{T}, src::Array{T}) where {T} = _copyto2arg!(dest, src) +copyto!(dest::Array{T}, src::Memory{T}) where {T} = _copyto2arg!(dest, src) +copyto!(dest::Memory{T}, src::Array{T}) where {T} = _copyto2arg!(dest, src) # N.B: The generic definition in multidimensional.jl covers, this, this is just here # for bootstrapping purposes. @@ -330,7 +336,7 @@ function fill!(dest::Array{T}, x) where T end function _fill!(dest::Array{T}, x::T) where T for i in eachindex(dest) - @inbounds dest[i] = x + dest[i] = x end return dest end @@ -742,9 +748,16 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown) return a end -_collect_indices(::Tuple{}, A) = copyto!(Array{eltype(A),0}(undef), A) -_collect_indices(indsA::Tuple{Vararg{OneTo}}, A) = - copyto!(Array{eltype(A)}(undef, length.(indsA)), A) +function _collect_indices(::Tuple{}, A) + dest = Array{eltype(A),0}(undef) + isempty(A) && return dest + return copyto_unaliased!(IndexStyle(dest), dest, IndexStyle(A), A) +end +function _collect_indices(indsA::Tuple{Vararg{OneTo}}, A) + dest = Array{eltype(A)}(undef, length.(indsA)) + isempty(A) && return dest + return copyto_unaliased!(IndexStyle(dest), dest, IndexStyle(A), A) +end function _collect_indices(indsA, A) B = Array{eltype(A)}(undef, length.(indsA)) copyto!(B, CartesianIndices(axes(B)), A, CartesianIndices(indsA)) @@ -896,10 +909,6 @@ function grow_to!(dest, itr, st) return dest end -## Iteration ## - -iterate(A::Array, i=1) = (@inline; (i - 1)%UInt < length(A)%UInt ? (@inbounds A[i], i + 1) : nothing) - ## Indexing: getindex ## """ @@ -987,7 +996,7 @@ function setindex!(A::Array{T}, x, i::Int) where {T} end function _setindex!(A::Array{T}, x::T, i::Int) where {T} @_noub_if_noinbounds_meta - @boundscheck (i - 1)%UInt < length(A)%UInt || throw_boundserror(A, (i,)) + @boundscheck checkbounds(Bool, A, i) || throw_boundserror(A, (i,)) memoryrefset!(memoryrefnew(A.ref, i, false), x, :not_atomic, false) return A end @@ -1066,6 +1075,41 @@ end array_new_memory(mem::Memory, newlen::Int) = typeof(mem)(undef, newlen) # when implemented, this should attempt to first expand mem +function _growbeg_internal!(a::Vector, delta::Int, len::Int) + @_terminates_locally_meta + ref = a.ref + mem = ref.mem + offset = memoryrefoffset(ref) + newlen = len + delta + memlen = length(mem) + if offset + len - 1 > memlen || offset < 1 + throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) + end + # since we will allocate the array in the middle of the memory we need at least 2*delta extra space + # the +1 is because I didn't want to have an off by 1 error. + newmemlen = max(overallocation(len), len + 2 * delta + 1) + newoffset = div(newmemlen - newlen, 2) + 1 + # If there is extra data after the end of the array we can use that space so long as there is enough + # space at the end that there won't be quadratic behavior with a mix of growth from both ends. + # Specifically, we want to ensure that we will only do this operation once before + # increasing the size of the array, and that we leave enough space at both the beginning and the end. + if newoffset + newlen < memlen + newoffset = div(memlen - newlen, 2) + 1 + newmem = mem + unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) + for j in offset:newoffset+delta-1 + @inbounds _unsetindex!(mem, j) + end + else + newmem = array_new_memory(mem, newmemlen) + unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) + end + if ref !== a.ref + throw(ConcurrencyViolationError("Vector can not be resized concurrently")) + end + setfield!(a, :ref, @inbounds memoryref(newmem, newoffset)) +end + function _growbeg!(a::Vector, delta::Integer) @_noub_meta delta = Int(delta) @@ -1081,40 +1125,46 @@ function _growbeg!(a::Vector, delta::Integer) if delta <= offset - 1 setfield!(a, :ref, @inbounds memoryref(ref, 1 - delta)) else - @noinline (function() - @_terminates_locally_meta - memlen = length(mem) - if offset + len - 1 > memlen || offset < 1 - throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) - end - # since we will allocate the array in the middle of the memory we need at least 2*delta extra space - # the +1 is because I didn't want to have an off by 1 error. - newmemlen = max(overallocation(len), len + 2 * delta + 1) - newoffset = div(newmemlen - newlen, 2) + 1 - # If there is extra data after the end of the array we can use that space so long as there is enough - # space at the end that there won't be quadratic behavior with a mix of growth from both ends. - # Specifically, we want to ensure that we will only do this operation once before - # increasing the size of the array, and that we leave enough space at both the beginning and the end. - if newoffset + newlen < memlen - newoffset = div(memlen - newlen, 2) + 1 - newmem = mem - unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) - for j in offset:newoffset+delta-1 - @inbounds _unsetindex!(mem, j) - end - else - newmem = array_new_memory(mem, newmemlen) - unsafe_copyto!(newmem, newoffset + delta, mem, offset, len) - end - if ref !== a.ref - @noinline throw(ConcurrencyViolationError("Vector can not be resized concurrently")) - end - setfield!(a, :ref, @inbounds memoryref(newmem, newoffset)) - end)() + @noinline _growbeg_internal!(a, delta, len) end return end +function _growend_internal!(a::Vector, delta::Int, len::Int) + ref = a.ref + mem = ref.mem + memlen = length(mem) + newlen = len + delta + offset = memoryrefoffset(ref) + newmemlen = offset + newlen - 1 + if offset + len - 1 > memlen || offset < 1 + throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) + end + + if offset - 1 > div(5 * newlen, 4) + # If the offset is far enough that we can copy without resizing + # while maintaining proportional spacing on both ends of the array + # note that this branch prevents infinite growth when doing combinations + # of push! and popfirst! (i.e. when using a Vector as a queue) + newmem = mem + newoffset = div(newlen, 8) + 1 + else + # grow either by our computed overallocation factor + # or exactly the requested size, whichever is larger + # TODO we should possibly increase the offset if the current offset is nonzero. + newmemlen2 = max(overallocation(memlen), newmemlen) + newmem = array_new_memory(mem, newmemlen2) + newoffset = offset + end + newref = @inbounds memoryref(newmem, newoffset) + unsafe_copyto!(newref, ref, len) + if ref !== a.ref + @noinline throw(ConcurrencyViolationError("Vector can not be resized concurrently")) + end + setfield!(a, :ref, newref) +return +end + function _growend!(a::Vector, delta::Integer) @_noub_meta delta = Int(delta) @@ -1128,33 +1178,7 @@ function _growend!(a::Vector, delta::Integer) setfield!(a, :size, (newlen,)) newmemlen = offset + newlen - 1 if memlen < newmemlen - @noinline (function() - if offset + len - 1 > memlen || offset < 1 - throw(ConcurrencyViolationError("Vector has invalid state. Don't modify internal fields incorrectly, or resize without correct locks")) - end - - if offset - 1 > div(5 * newlen, 4) - # If the offset is far enough that we can copy without resizing - # while maintaining proportional spacing on both ends of the array - # note that this branch prevents infinite growth when doing combinations - # of push! and popfirst! (i.e. when using a Vector as a queue) - newmem = mem - newoffset = div(newlen, 8) + 1 - else - # grow either by our computed overallocation factor - # or exactly the requested size, whichever is larger - # TODO we should possibly increase the offset if the current offset is nonzero. - newmemlen2 = max(overallocation(memlen), newmemlen) - newmem = array_new_memory(mem, newmemlen2) - newoffset = offset - end - newref = @inbounds memoryref(newmem, newoffset) - unsafe_copyto!(newref, ref, len) - if ref !== a.ref - @noinline throw(ConcurrencyViolationError("Vector can not be resized concurrently")) - end - setfield!(a, :ref, newref) - end)() + @noinline _growend_internal!(a, delta, len) end return end diff --git a/base/arraymath.jl b/base/arraymath.jl index 62dc3772e4938..53a7d132a2c0c 100644 --- a/base/arraymath.jl +++ b/base/arraymath.jl @@ -72,12 +72,12 @@ _reverse!(A::AbstractArray{<:Any,N}, ::Colon) where {N} = _reverse!(A, ntuple(id _reverse!(A, dim::Integer) = _reverse!(A, (Int(dim),)) _reverse!(A, dims::NTuple{M,Integer}) where {M} = _reverse!(A, Int.(dims)) function _reverse!(A::AbstractArray{<:Any,N}, dims::NTuple{M,Int}) where {N,M} + dims === () && return A # nothing to reverse dimrev = ntuple(k -> k in dims, Val{N}()) # boolean tuple indicating reversed dims if N < M || M != sum(dimrev) throw(ArgumentError("invalid dimensions $dims in reverse!")) end - M == 0 && return A # nothing to reverse # swapping loop only needs to traverse ≈half of the array halfsz = ntuple(k -> k == dims[1] ? size(A,k) ÷ 2 : size(A,k), Val{N}()) diff --git a/base/arrayshow.jl b/base/arrayshow.jl index 31e95e6659657..f792d26d8e9b5 100644 --- a/base/arrayshow.jl +++ b/base/arrayshow.jl @@ -361,7 +361,7 @@ print_array(io::IO, X::AbstractArray) = show_nd(io, X, print_matrix, true) # typeinfo aware # implements: show(io::IO, ::MIME"text/plain", X::AbstractArray) function show(io::IO, ::MIME"text/plain", X::AbstractArray) - if isempty(X) && (get(io, :compact, false)::Bool || X isa Vector) + if isempty(X) && (get(io, :compact, false)::Bool || X isa AbstractVector) return show(io, X) end # 1) show summary before setting :compact diff --git a/base/asyncevent.jl b/base/asyncevent.jl index 8297626ee0f97..13b3a94580fec 100644 --- a/base/asyncevent.jl +++ b/base/asyncevent.jl @@ -9,7 +9,8 @@ Create a async condition that wakes up tasks waiting for it (by calling [`wait`](@ref) on the object) when notified from C by a call to `uv_async_send`. Waiting tasks are woken with an error when the object is closed (by [`close`](@ref)). -Use [`isopen`](@ref) to check whether it is still active. +Use [`isopen`](@ref) to check whether it is still active. A closed condition is inactive and will +not wake up tasks. This provides an implicit acquire & release memory ordering between the sending and waiting threads. """ @@ -74,8 +75,8 @@ Create a timer that wakes up tasks waiting for it (by calling [`wait`](@ref) on Waiting tasks are woken after an initial delay of at least `delay` seconds, and then repeating after at least `interval` seconds again elapse. If `interval` is equal to `0`, the timer is only triggered once. When the timer is closed (by [`close`](@ref)) waiting tasks are woken with an error. Use -[`isopen`](@ref) to check whether a timer is still active. Use `t.timeout` and `t.interval` to read -the setup conditions of a `Timer` `t`. +[`isopen`](@ref) to check whether a timer is still active. An inactive timer will not fire. +Use `t.timeout` and `t.interval` to read the setup conditions of a `Timer` `t`. ```julia-repl julia> t = Timer(1.0; interval=0.5) @@ -206,6 +207,14 @@ end isopen(t::Union{Timer, AsyncCondition}) = @atomic :acquire t.isopen +""" + close(t::Union{Timer, AsyncCondition}) + +Close an object `t` and thus mark it as inactive. Once a timer or condition is inactive, it will not produce +a new event. + +See also: [`isopen`](@ref) +""" function close(t::Union{Timer, AsyncCondition}) t.handle == C_NULL && !t.isopen && return # short-circuit path, :monotonic iolock_begin() diff --git a/base/atomics.jl b/base/atomics.jl index e6f3a5654cbf7..432c9120939ac 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -1,7 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Core.Intrinsics: llvmcall - import .Base: setindex!, getindex, unsafe_convert import .Base.Sys: ARCH, WORD_SIZE @@ -13,34 +11,6 @@ export atomic_and!, atomic_nand!, atomic_or!, atomic_xor!, atomic_max!, atomic_min!, atomic_fence -## -# Filter out unsupported atomic types on platforms -# - 128-bit atomics do not exist on AArch32. -# - Omitting 128-bit types on 32bit x86 and ppc64 -# - LLVM doesn't currently support atomics on floats for ppc64 -# C++20 is adding limited support for atomics on float, but as of -# now Clang does not support that yet. -if Sys.ARCH === :i686 || startswith(string(Sys.ARCH), "arm") || - Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le - const inttypes = (Int8, Int16, Int32, Int64, - UInt8, UInt16, UInt32, UInt64) -else - const inttypes = (Int8, Int16, Int32, Int64, Int128, - UInt8, UInt16, UInt32, UInt64, UInt128) -end -const floattypes = (Float16, Float32, Float64) -const arithmetictypes = (inttypes..., floattypes...) -# TODO: Support Ptr -if Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le - const atomictypes = (inttypes..., Bool) -else - const atomictypes = (arithmetictypes..., Bool) -end - -const IntTypes = Union{inttypes...} -const FloatTypes = Union{floattypes...} -const ArithmeticTypes = Union{arithmetictypes...} -const AtomicTypes = Union{atomictypes...} """ Threads.Atomic{T} @@ -48,10 +18,6 @@ const AtomicTypes = Union{atomictypes...} Holds a reference to an object of type `T`, ensuring that it is only accessed atomically, i.e. in a thread-safe manner. -Only certain "simple" types can be used atomically, namely the -primitive boolean, integer, and float-point types. These are `Bool`, -`Int8`...`Int128`, `UInt8`...`UInt128`, and `Float16`...`Float64`. - New atomic objects can be created from a non-atomic values; if none is specified, the atomic object is initialized with zero. @@ -72,10 +38,10 @@ julia> x[] Atomic operations use an `atomic_` prefix, such as [`atomic_add!`](@ref), [`atomic_xchg!`](@ref), etc. """ -mutable struct Atomic{T<:AtomicTypes} - value::T - Atomic{T}() where {T<:AtomicTypes} = new(zero(T)) - Atomic{T}(value) where {T<:AtomicTypes} = new(value) +mutable struct Atomic{T} + @atomic value::T + Atomic{T}() where {T} = new(zero(T)) + Atomic{T}(value) where {T} = new(value) end Atomic() = Atomic{Int}() @@ -332,120 +298,21 @@ julia> x[] """ function atomic_min! end -unsafe_convert(::Type{Ptr{T}}, x::Atomic{T}) where {T} = convert(Ptr{T}, pointer_from_objref(x)) -setindex!(x::Atomic{T}, v) where {T} = setindex!(x, convert(T, v)) - -const llvmtypes = IdDict{Any,String}( - Bool => "i8", # julia represents bools with 8-bits for now. # TODO: is this okay? - Int8 => "i8", UInt8 => "i8", - Int16 => "i16", UInt16 => "i16", - Int32 => "i32", UInt32 => "i32", - Int64 => "i64", UInt64 => "i64", - Int128 => "i128", UInt128 => "i128", - Float16 => "half", - Float32 => "float", - Float64 => "double", -) -inttype(::Type{T}) where {T<:Integer} = T -inttype(::Type{Float16}) = Int16 -inttype(::Type{Float32}) = Int32 -inttype(::Type{Float64}) = Int64 - - -import ..Base.gc_alignment - -# All atomic operations have acquire and/or release semantics, depending on -# whether the load or store values. Most of the time, this is what one wants -# anyway, and it's only moderately expensive on most hardware. -for typ in atomictypes - lt = llvmtypes[typ] - ilt = llvmtypes[inttype(typ)] - rt = "$lt, $lt*" - irt = "$ilt, $ilt*" - @eval getindex(x::Atomic{$typ}) = - GC.@preserve x llvmcall($""" - %ptr = bitcast i8* %0 to $lt* - %rv = load atomic $rt %ptr acquire, align $(gc_alignment(typ)) - ret $lt %rv - """, $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x)) - @eval setindex!(x::Atomic{$typ}, v::$typ) = - GC.@preserve x llvmcall($""" - %ptr = bitcast i8* %0 to $lt* - store atomic $lt %1, $lt* %ptr release, align $(gc_alignment(typ)) - ret void - """, Cvoid, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) - - # Note: atomic_cas! succeeded (i.e. it stored "new") if and only if the result is "cmp" - if typ <: Integer - @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = - GC.@preserve x llvmcall($""" - %ptr = bitcast i8* %0 to $lt* - %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 acq_rel acquire - %rv = extractvalue { $lt, i1 } %rs, 0 - ret $lt %rv - """, $typ, Tuple{Ptr{$typ},$typ,$typ}, - unsafe_convert(Ptr{$typ}, x), cmp, new) - else - @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = - GC.@preserve x llvmcall($""" - %iptr = bitcast i8* %0 to $ilt* - %icmp = bitcast $lt %1 to $ilt - %inew = bitcast $lt %2 to $ilt - %irs = cmpxchg $ilt* %iptr, $ilt %icmp, $ilt %inew acq_rel acquire - %irv = extractvalue { $ilt, i1 } %irs, 0 - %rv = bitcast $ilt %irv to $lt - ret $lt %rv - """, $typ, Tuple{Ptr{$typ},$typ,$typ}, - unsafe_convert(Ptr{$typ}, x), cmp, new) - end - - arithmetic_ops = [:add, :sub] - for rmwop in [arithmetic_ops..., :xchg, :and, :nand, :or, :xor, :max, :min] - rmw = string(rmwop) - fn = Symbol("atomic_", rmw, "!") - if (rmw == "max" || rmw == "min") && typ <: Unsigned - # LLVM distinguishes signedness in the operation, not the integer type. - rmw = "u" * rmw - end - if rmwop in arithmetic_ops && !(typ <: ArithmeticTypes) continue end - if typ <: Integer - @eval $fn(x::Atomic{$typ}, v::$typ) = - GC.@preserve x llvmcall($""" - %ptr = bitcast i8* %0 to $lt* - %rv = atomicrmw $rmw $lt* %ptr, $lt %1 acq_rel - ret $lt %rv - """, $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) - else - rmwop === :xchg || continue - @eval $fn(x::Atomic{$typ}, v::$typ) = - GC.@preserve x llvmcall($""" - %iptr = bitcast i8* %0 to $ilt* - %ival = bitcast $lt %1 to $ilt - %irv = atomicrmw $rmw $ilt* %iptr, $ilt %ival acq_rel - %rv = bitcast $ilt %irv to $lt - ret $lt %rv - """, $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) - end - end -end - -# Provide atomic floating-point operations via atomic_cas! -const opnames = Dict{Symbol, Symbol}(:+ => :add, :- => :sub) -for op in [:+, :-, :max, :min] - opname = get(opnames, op, op) - @eval function $(Symbol("atomic_", opname, "!"))(var::Atomic{T}, val::T) where T<:FloatTypes - IT = inttype(T) - old = var[] - while true - new = $op(old, val) - cmp = old - old = atomic_cas!(var, cmp, new) - reinterpret(IT, old) == reinterpret(IT, cmp) && return old - # Temporary solution before we have gc transition support in codegen. - ccall(:jl_gc_safepoint, Cvoid, ()) - end - end -end +#const nand = (~) ∘ (&) # ComposedFunction generated very poor code quality +nand(x, y) = ~(x & y) + +getindex(x::Atomic) = @atomic :acquire x.value +setindex!(x::Atomic, v) = (@atomic :release x.value = v; x) +atomic_cas!(x::Atomic, cmp, new) = (@atomicreplace :acquire_release :acquire x.value cmp => new).old +atomic_add!(x::Atomic, v) = (@atomic :acquire_release x.value + v).first +atomic_sub!(x::Atomic, v) = (@atomic :acquire_release x.value - v).first +atomic_and!(x::Atomic, v) = (@atomic :acquire_release x.value & v).first +atomic_or!(x::Atomic, v) = (@atomic :acquire_release x.value | v).first +atomic_xor!(x::Atomic, v) = (@atomic :acquire_release x.value ⊻ v).first +atomic_nand!(x::Atomic, v) = (@atomic :acquire_release x.value nand v).first +atomic_xchg!(x::Atomic, v) = (@atomicswap :acquire_release x.value = v) +atomic_min!(x::Atomic, v) = (@atomic :acquire_release x.value min v).first +atomic_max!(x::Atomic, v) = (@atomic :acquire_release x.value max v).first """ Threads.atomic_fence() @@ -462,7 +329,4 @@ fences should not be necessary in most cases. For further details, see LLVM's `fence` instruction. """ -atomic_fence() = llvmcall(""" - fence seq_cst - ret void - """, Cvoid, Tuple{}) +atomic_fence() = Core.Intrinsics.atomic_fence(:sequentially_consistent) diff --git a/base/binaryplatforms.jl b/base/binaryplatforms.jl index 51c65a6b310a6..c2f019c4d4eea 100644 --- a/base/binaryplatforms.jl +++ b/base/binaryplatforms.jl @@ -157,7 +157,7 @@ end # Hash definition to ensure that it's stable function Base.hash(p::Platform, h::UInt) - h += 0x506c6174666f726d % UInt + h ⊻= 0x506c6174666f726d % UInt h = hash(p.tags, h) h = hash(p.compare_strategies, h) return h @@ -796,6 +796,17 @@ function platform_dlext(p::AbstractPlatform = HostPlatform()) end end +# Not general purpose, just for parse_dl_name_version +function _this_os_name() + if Sys.iswindows() + return "windows" + elseif Sys.isapple() + return "macos" + else + return "other" + end +end + """ parse_dl_name_version(path::String, platform::AbstractPlatform) @@ -806,12 +817,14 @@ valid dynamic library, this method throws an error. If no soversion can be extracted from the filename, as in "libbar.so" this method returns `"libbar", nothing`. """ -function parse_dl_name_version(path::String, os::String) +function parse_dl_name_version(path::String, os::String=_this_os_name()) # Use an extraction regex that matches the given OS local dlregex + # Keep this up to date with _this_os_name if os == "windows" - # On Windows, libraries look like `libnettle-6.dll` - dlregex = r"^(.*?)(?:-((?:[\.\d]+)*))?\.dll$"sa + # On Windows, libraries look like `libnettle-6.dll`. + # Stay case-insensitive, the suffix might be `.DLL`. + dlregex = r"^(.*?)(?:-((?:[\.\d]+)*))?\.dll$"isa elseif os == "macos" # On OSX, libraries look like `libnettle.6.3.dylib` dlregex = r"^(.*?)((?:\.[\d]+)*)\.dylib$"sa @@ -837,7 +850,7 @@ function parse_dl_name_version(path::String, os::String) end # Adapter for `AbstractString` -function parse_dl_name_version(path::AbstractString, os::AbstractString) +function parse_dl_name_version(path::AbstractString, os::AbstractString=_this_os_name()) return parse_dl_name_version(string(path)::String, string(os)::String) end diff --git a/base/bitarray.jl b/base/bitarray.jl index e47cf82dd603f..5a3469fa7c7a2 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -53,7 +53,7 @@ Construct an undef [`BitArray`](@ref) with the given dimensions. Behaves identically to the [`Array`](@ref) constructor. See [`undef`](@ref). # Examples -```julia-repl +```jldoctest; filter = r"[01]" julia> BitArray(undef, 2, 2) 2×2 BitMatrix: 0 0 diff --git a/base/bool.jl b/base/bool.jl index 3658318d158e5..12144756c76c8 100644 --- a/base/bool.jl +++ b/base/bool.jl @@ -156,6 +156,7 @@ abs(x::Bool) = x abs2(x::Bool) = x iszero(x::Bool) = !x isone(x::Bool) = x +ispositive(x::Bool) = x # could use fallback once #21712 is resolved <(x::Bool, y::Bool) = y&!x <=(x::Bool, y::Bool) = y|!x diff --git a/base/boot.jl b/base/boot.jl index f9208c3874229..d055c47516f91 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -590,8 +590,9 @@ const undef = UndefInitializer() # empty vector constructor (self::Type{GenericMemory{kind,T,addrspace}})() where {T,kind,addrspace} = self(undef, 0) +# memoryref is simply convenience wrapper function around memoryrefnew memoryref(mem::GenericMemory) = memoryrefnew(mem) -memoryref(mem::GenericMemory, i::Integer) = memoryrefnew(memoryrefnew(mem), Int(i), @_boundscheck) +memoryref(mem::GenericMemory, i::Integer) = memoryrefnew(mem, Int(i), @_boundscheck) memoryref(ref::GenericMemoryRef, i::Integer) = memoryrefnew(ref, Int(i), @_boundscheck) GenericMemoryRef(mem::GenericMemory) = memoryref(mem) GenericMemoryRef(mem::GenericMemory, i::Integer) = memoryref(mem, i) @@ -695,18 +696,68 @@ function Symbol(a::Array{UInt8, 1}) end Symbol(s::Symbol) = s +# Minimal implementations of using/import for bootstrapping (supports only +# `import .M: a, b, c, ...`, little error checking) +let + fail() = throw(ArgumentError("unsupported import/using while bootstrapping")) + length(a::Array{T, 1}) where {T} = getfield(getfield(a, :size), 1) + function getindex(A::Array, i::Int) + Intrinsics.ult_int(Intrinsics.bitcast(UInt, Intrinsics.sub_int(i, 1)), Intrinsics.bitcast(UInt, length(A))) || fail() + memoryrefget(memoryrefnew(getfield(A, :ref), i, false), :not_atomic, false) + end + x == y = Intrinsics.eq_int(x, y) + x + y = Intrinsics.add_int(x, y) + x <= y = Intrinsics.sle_int(x, y) + + global function _eval_import(explicit::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...) + from isa Expr || fail() + if length(from.args) == 2 && getindex(from.args, 1) === :. + from = getglobal(to, getindex(from.args, 2)) + elseif length(from.args) == 1 && getindex(from.args, 1) === :Core + from = Core + elseif length(from.args) == 1 && getindex(from.args, 1) === :Base + from = Main.Base + else + fail() + end + from isa Module || fail() + i = 1 + while i <= nfields(paths) + a = getfield(paths, i).args + length(a) == 1 || fail() + s = getindex(a, 1) + Core._import(to, from, s, s, explicit) + i += 1 + end + end + + global function _eval_using(to::Module, path::Expr) + getindex(path.args, 1) === :. || fail() + from = getglobal(to, getindex(path.args, 2)) + i = 3 + while i <= length(path.args) + from = getfield(from, getindex(path.args, i)) + i += 1 + end + from isa Module || fail() + Core._using(to, from) + end +end + # module providing the IR object model +# excluding types already exported by Core (GlobalRef, QuoteNode, Expr, LineNumberNode) +# any type beyond these is self-quoting (see also Base.is_ast_node) module IR export CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode, NewvarNode, SSAValue, SlotNumber, Argument, PiNode, PhiNode, PhiCNode, UpsilonNode, DebugInfo, - Const, PartialStruct, InterConditional, EnterNode, memoryref + Const, PartialStruct, InterConditional, EnterNode using Core: CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode, NewvarNode, SSAValue, SlotNumber, Argument, PiNode, PhiNode, PhiCNode, UpsilonNode, DebugInfo, - Const, PartialStruct, InterConditional, EnterNode, memoryref + Const, PartialStruct, InterConditional, EnterNode end # module IR @@ -998,8 +1049,9 @@ function struct_name_shim(@nospecialize(x), name::Symbol, mod::Module, @nospecia return x === mod ? t : getfield(x, name) end -# Binding for the julia parser, called as -# +# Bindings for the julia frontend. The internal jl_parse and jl_lower will call +# Core._parse and Core._lower respectively (if they are not `nothing`.) + # Core._parse(text, filename, lineno, offset, options) # # Parse Julia code from the buffer `text`, starting at `offset` and attributing @@ -1009,11 +1061,17 @@ end # # `_parse` must return an `svec` containing an `Expr` and the new offset as an # `Int`. -# -# The internal jl_parse will call into Core._parse if not `nothing`. _parse = nothing +# Core._lower(code, module, filename="none", linenum=0, world=0xfff..., warn=false) +# +# Lower `code` (usually Expr), returning `svec(e::Any xs::Any...)` where `e` is +# the lowered code, and `xs` is possible additional information from +# JuliaLowering (TBD). +_lower = nothing + _setparser!(parser) = setglobal!(Core, :_parse, parser) +_setlowerer!(lowerer) = setglobal!(Core, :_lower, lowerer) # support for deprecated uses of builtin functions _apply(x...) = _apply_iterate(Main.Base.iterate, x...) diff --git a/base/broadcast.jl b/base/broadcast.jl index 2a16cd11c1d5c..b86baf08ddfe0 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -585,15 +585,17 @@ an `Int`. """ Base.@propagate_inbounds newindex(arg, I::CartesianIndex) = to_index(_newindex(axes(arg), I.I)) Base.@propagate_inbounds newindex(arg, I::Integer) = to_index(_newindex(axes(arg), (I,))) -Base.@propagate_inbounds _newindex(ax::Tuple, I::Tuple) = (ifelse(length(ax[1]) == 1, ax[1][1], I[1]), _newindex(tail(ax), tail(I))...) +Base.@propagate_inbounds _newindex(ax::Tuple, I::Tuple) = (ifelse(length(ax[1]) == 1, ax[1][begin], I[1]), _newindex(tail(ax), tail(I))...) Base.@propagate_inbounds _newindex(ax::Tuple{}, I::Tuple) = () -Base.@propagate_inbounds _newindex(ax::Tuple, I::Tuple{}) = (ax[1][1], _newindex(tail(ax), ())...) +Base.@propagate_inbounds _newindex(ax::Tuple, I::Tuple{}) = (ax[1][begin], _newindex(tail(ax), ())...) Base.@propagate_inbounds _newindex(ax::Tuple{}, I::Tuple{}) = () # If dot-broadcasting were already defined, this would be `ifelse.(keep, I, Idefault)`. @inline newindex(I::CartesianIndex, keep, Idefault) = to_index(_newindex(I.I, keep, Idefault)) -@inline newindex(i::Integer, keep::Tuple, idefault) = ifelse(keep[1], i, idefault[1]) -@inline newindex(i::Integer, keep::Tuple{}, idefault) = CartesianIndex(()) +@inline newindex(I::CartesianIndex{1}, keep, Idefault) = newindex(I.I[1], keep, Idefault) +@inline newindex(i::Integer, keep::Tuple, idefault) = CartesianIndex(ifelse(keep[1], Int(i), Int(idefault[1])), idefault[2]) +@inline newindex(i::Integer, keep::Tuple{Bool}, idefault) = ifelse(keep[1], i, idefault[1]) +@inline newindex(i::Integer, keep::Tuple{}, idefault) = CartesianIndex() @inline _newindex(I, keep, Idefault) = (ifelse(keep[1], I[1], Idefault[1]), _newindex(tail(I), tail(keep), tail(Idefault))...) @inline _newindex(I, keep::Tuple{}, Idefault) = () # truncate if keep is shorter than I diff --git a/base/c.jl b/base/c.jl index 78c48f267ca71..69ea3adf24404 100644 --- a/base/c.jl +++ b/base/c.jl @@ -2,7 +2,7 @@ # definitions related to C interface -import Core.Intrinsics: cglobal +import .Intrinsics: cglobal """ cglobal((symbol, library) [, type=Cvoid]) diff --git a/base/channels.jl b/base/channels.jl index ef508bd40e3ed..0bb73e9acba87 100644 --- a/base/channels.jl +++ b/base/channels.jl @@ -61,7 +61,7 @@ Channel(sz=0) = Channel{Any}(sz) """ Channel{T=Any}(func::Function, size=0; taskref=nothing, spawn=false, threadpool=nothing) -Create a new task from `func`, bind it to a new channel of type +Create a new task from `func`, [`bind`](@ref) it to a new channel of type `T` and size `size`, and schedule the task, all in a single call. The channel is automatically closed when the task terminates. diff --git a/base/char.jl b/base/char.jl index fc173bb3c3a44..90636a6d9536e 100644 --- a/base/char.jl +++ b/base/char.jl @@ -205,7 +205,7 @@ size(c::AbstractChar, d::Integer) = d < 1 ? throw(BoundsError()) : 1 ndims(c::AbstractChar) = 0 ndims(::Type{<:AbstractChar}) = 0 length(c::AbstractChar) = 1 -IteratorSize(::Type{Char}) = HasShape{0}() +IteratorSize(::Type{<:AbstractChar}) = HasShape{0}() firstindex(c::AbstractChar) = 1 lastindex(c::AbstractChar) = 1 getindex(c::AbstractChar) = c @@ -222,7 +222,7 @@ in(x::AbstractChar, y::AbstractChar) = x == y ==(x::Char, y::Char) = bitcast(UInt32, x) == bitcast(UInt32, y) isless(x::Char, y::Char) = bitcast(UInt32, x) < bitcast(UInt32, y) hash(x::Char, h::UInt) = - hash_uint64(((bitcast(UInt32, x) + UInt64(0xd4d64234)) << 32) ⊻ UInt64(h)) + hash_finalizer(((bitcast(UInt32, x) + UInt64(0xd4d64234)) << 32) ⊻ UInt64(h)) % UInt # fallbacks: isless(x::AbstractChar, y::AbstractChar) = isless(Char(x), Char(y)) diff --git a/base/checked.jl b/base/checked.jl index b374d34830280..39d487cba6e37 100644 --- a/base/checked.jl +++ b/base/checked.jl @@ -16,12 +16,13 @@ export checked_neg, checked_abs, checked_add, checked_sub, checked_mul, checked_div, checked_rem, checked_fld, checked_mod, checked_cld, checked_pow, checked_length, add_with_overflow, sub_with_overflow, mul_with_overflow -import Core.Intrinsics: +import Core: Intrinsics +import .Intrinsics: checked_sadd_int, checked_ssub_int, checked_smul_int, checked_sdiv_int, checked_srem_int, checked_uadd_int, checked_usub_int, checked_umul_int, checked_udiv_int, checked_urem_int -import ..no_op_err, ..@inline, ..@noinline, ..checked_length +import Base: no_op_err, @inline, @noinline, checked_length # define promotion behavior for checked operations checked_add(x::Integer, y::Integer) = checked_add(promote(x,y)...) diff --git a/base/client.jl b/base/client.jl index 9b7d80f51c219..8ff925d0d9ba4 100644 --- a/base/client.jl +++ b/base/client.jl @@ -32,9 +32,6 @@ stackframe_lineinfo_color() = repl_color("JULIA_STACKFRAME_LINEINFO_COLOR", :bol stackframe_function_color() = repl_color("JULIA_STACKFRAME_FUNCTION_COLOR", :bold) function repl_cmd(cmd, out) - shell = shell_split(get(ENV, "JULIA_SHELL", get(ENV, "SHELL", "/bin/sh"))) - shell_name = Base.basename(shell[1]) - # Immediately expand all arguments, so that typing e.g. ~/bin/foo works. cmd.exec .= expanduser.(cmd.exec) @@ -64,21 +61,17 @@ function repl_cmd(cmd, out) cd(dir) println(out, pwd()) else - @static if !Sys.iswindows() - if shell_name == "fish" - shell_escape_cmd = "begin; $(shell_escape_posixly(cmd)); and true; end" - else - shell_escape_cmd = "($(shell_escape_posixly(cmd))) && true" - end + if !Sys.iswindows() + shell = shell_split(get(ENV, "JULIA_SHELL", get(ENV, "SHELL", "/bin/sh"))) + shell_escape_cmd = shell_escape_posixly(cmd) cmd = `$shell -c $shell_escape_cmd` end try run(ignorestatus(cmd)) catch - # Windows doesn't shell out right now (complex issue), so Julia tries to run the program itself - # Julia throws an exception if it can't find the program, but the stack trace isn't useful + # Julia throws an exception if it can't find the cmd (which may be the shell itself), but the stack trace isn't useful lasterr = current_exceptions() - lasterr = ExceptionStack([(exception = e[1], backtrace = [] ) for e in lasterr]) + lasterr = ExceptionStack(NamedTuple[(exception = e[1], backtrace = [] ) for e in lasterr]) invokelatest(display_error, lasterr) end end @@ -106,7 +99,7 @@ function scrub_repl_backtrace(bt) return bt end scrub_repl_backtrace(stack::ExceptionStack) = - ExceptionStack(Any[(;x.exception, backtrace = scrub_repl_backtrace(x.backtrace)) for x in stack]) + ExceptionStack(NamedTuple[(;x.exception, backtrace = scrub_repl_backtrace(x.backtrace)) for x in stack]) istrivialerror(stack::ExceptionStack) = length(stack) == 1 && length(stack[1].backtrace) ≤ 1 && !isa(stack[1].exception, MethodError) @@ -223,10 +216,13 @@ function incomplete_tag(ex::Expr) return :none elseif isempty(ex.args) return :other - elseif ex.args[1] isa String - return fl_incomplete_tag(ex.args[1]) else - return incomplete_tag(ex.args[1]) + a = ex.args[1] + if a isa String + return fl_incomplete_tag(a)::Symbol + else + return incomplete_tag(a)::Symbol + end end end incomplete_tag(exc::Meta.ParseError) = incomplete_tag(exc.detail) @@ -274,10 +270,6 @@ function exec_options(opts) interactiveinput = (repl || is_interactive::Bool) && isa(stdin, TTY) is_interactive::Bool |= interactiveinput - # load terminfo in for styled printing - term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb") - global current_terminfo = load_terminfo(term_env) - # load ~/.julia/config/startup.jl file if startup try @@ -439,11 +431,12 @@ function run_fallback_repl(interactive::Bool) eval_user_input(stderr, ex, true) end else - while !eof(input) + while true if interactive print("julia> ") flush(stdout) end + eof(input) && break try line = "" ex = nothing diff --git a/base/combinatorics.jl b/base/combinatorics.jl index dac217cd4fb41..5180f830ce187 100644 --- a/base/combinatorics.jl +++ b/base/combinatorics.jl @@ -184,13 +184,12 @@ end """ permute!(v, p) -Permute vector `v` in-place, according to permutation `p`. No checking is done -to verify that `p` is a permutation. +Permute vector `v` according to permutation `p`, storing the result back into `v`. +No checking is done to verify that `p` is a permutation. To return a new permutation, use `v[p]`. This is generally faster than `permute!(v, p)`; it is even faster to write into a pre-allocated output array with `u .= @view v[p]`. -(Even though `permute!` overwrites `v` in-place, it internally requires some allocation -to keep track of which elements have been moved.) +(Even though `permute!` overwrites `v` in-place, it internally requires some allocation.) $(_DOCS_ALIASING_WARNING) diff --git a/base/complex.jl b/base/complex.jl index 37fa974cc0c7d..6651581a96240 100644 --- a/base/complex.jl +++ b/base/complex.jl @@ -1141,3 +1141,9 @@ function complex(A::AbstractArray{T}) where T end convert(AbstractArray{typeof(complex(zero(T)))}, A) end + +## Machine epsilon for complex ## + +eps(z::Complex{<:AbstractFloat}) = hypot(eps(real(z)), eps(imag(z))) + +eps(::Type{Complex{T}}) where {T<:AbstractFloat} = sqrt(2*one(T))*eps(T) diff --git a/base/deepcopy.jl b/base/deepcopy.jl index f60ce2043dd5a..58c753705a61f 100644 --- a/base/deepcopy.jl +++ b/base/deepcopy.jl @@ -23,6 +23,11 @@ where `T` is the type to be specialized for, and `dict` keeps track of objects c so far within the recursion. Within the definition, `deepcopy_internal` should be used in place of `deepcopy`, and the `dict` variable should be updated as appropriate before returning. + +!!! warning + It is better to avoid this function in favor of custom `copy` methods or use-case-specific + copying functions. `deepcopy` is slow and can easily copy too many objects, or generate an + object that violates invariants, since it does not respect abstraction boundaries. """ function deepcopy(@nospecialize x) isbitstype(typeof(x)) && return x diff --git a/base/deprecated.jl b/base/deprecated.jl index eeb7c0e60638e..0ee6b6b790837 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -87,7 +87,7 @@ else end ``` -To find out the correct verrsion to use as the first argument, you may use +To find out the correct version to use as the first argument, you may use `Base.__next_removal_version`, which is set to the next version number in which the list of changes will be cleared. @@ -211,7 +211,7 @@ macro deprecate(old, new, export_old=true) maybe_export, :($(esc(old)) = begin $meta - depwarn($"`$oldcall` is deprecated, use `$newcall` instead.", Core.Typeof($(esc(fnexpr))).name.mt.name) + depwarn($"`$oldcall` is deprecated, use `$newcall` instead.", Core.Typeof($(esc(fnexpr))).name.singletonname) $(esc(new)) end)) else @@ -222,7 +222,7 @@ macro deprecate(old, new, export_old=true) export_old ? Expr(:export, esc(old)) : nothing, :(function $(esc(old))(args...; kwargs...) $meta - depwarn($"`$old` is deprecated, use `$new` instead.", Core.Typeof($(esc(old))).name.mt.name) + depwarn($"`$old` is deprecated, use `$new` instead.", Core.Typeof($(esc(old))).name.singletonname) $(esc(new))(args...; kwargs...) end)) end @@ -557,4 +557,11 @@ true """ isbindingresolved +# Some packages call this function +function to_power_type(x::Number) + T = promote_type(typeof(x), typeof(x*x)) + convert(T, x) +end +to_power_type(x) = oftype(x*x, x) + # END 1.12 deprecations diff --git a/base/dict.jl b/base/dict.jl index 1272229c714da..6cbf4429ca9e0 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -124,9 +124,10 @@ _shorthash7(hsh::UInt) = (hsh >> (8sizeof(UInt)-7))%UInt8 | 0x80 # hashindex (key, sz) - computes optimal position and shorthash7 # idx - optimal position in the hash table # sh::UInt8 - short hash (7 highest hash bits) -function hashindex(key, sz) +function hashindex(key, sz::Integer) + sz = Int(sz)::Int hsh = hash(key)::UInt - idx = (((hsh % Int) & (sz-1)) + 1)::Int + idx = ((hsh % Int) & (sz-1)) + 1 return idx, _shorthash7(hsh) end @@ -190,7 +191,8 @@ end return h end -function sizehint!(d::Dict{T}, newsz; shrink::Bool=true) where T +function sizehint!(d::Dict{T}, newsz::Integer; shrink::Bool=true) where T + newsz = Int(newsz)::Int oldsz = length(d.slots) # limit new element count to max_values of the key type newsz = min(max(newsz, length(d)), max_values(T)::Int) @@ -532,7 +534,7 @@ end Determine whether a collection has a mapping for a given `key`. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+\\s+=>\\s+\\d\$"m julia> D = Dict('a'=>2, 'b'=>3) Dict{Char, Int64} with 2 entries: 'a' => 2 @@ -554,7 +556,7 @@ in(key, v::KeySet{<:Any, <:Dict}) = (ht_keyindex(v.dict, key) >= 0) Return the key matching argument `key` if one exists in `collection`, otherwise return `default`. # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\S+\\s+=>\\s+\\d\$"m julia> D = Dict('a'=>2, 'b'=>3) Dict{Char, Int64} with 2 entries: 'a' => 2 diff --git a/base/docs/Docs.jl b/base/docs/Docs.jl index ae3891e218824..13a5b35a115da 100644 --- a/base/docs/Docs.jl +++ b/base/docs/Docs.jl @@ -51,6 +51,10 @@ You can retrieve docs for functions, macros and other objects as follows: @doc @time @doc md"" +!!! compat "Julia 1.11" + In Julia 1.11 and newer, retrieving documentation with the `@doc` macro requires that + the `REPL` stdlib is loaded. + ## Functions & Methods Placing documentation before a method definition (e.g. `function foo() ...` or `foo() = ...`) will cause that specific method to be documented, as opposed to the whole function. Method @@ -787,6 +791,9 @@ When `pattern` is a string, case is ignored. Results are printed to `io`. ``` help?> "pattern" ``` + +!!! compat "Julia 1.11" + In Julia 1.11 and newer, `apropos` requires that the `REPL` stdlib is loaded. """ function apropos end @@ -797,6 +804,9 @@ Return all documentation that matches both `binding` and `sig`. If `getdoc` returns a non-`nothing` result on the value of the binding, then a dynamic docstring is returned instead of one based on the binding itself. + +!!! compat "Julia 1.11" + In Julia 1.11 and newer, `Docs.doc` requires that the `REPL` stdlib is loaded. """ function doc end diff --git a/base/docs/basedocs.jl b/base/docs/basedocs.jl index cf8a087b76489..a6abf3e384cfb 100644 --- a/base/docs/basedocs.jl +++ b/base/docs/basedocs.jl @@ -2470,14 +2470,19 @@ julia> Tuple(Real[1, 2, pi]) # takes a collection tuple """ - getfield(value, name::Symbol, [order::Symbol]) - getfield(value, i::Int, [order::Symbol]) - -Extract a field from a composite `value` by name or position. Optionally, an -ordering can be defined for the operation. If the field was declared `@atomic`, -the specification is strongly recommended to be compatible with the stores to -that location. Otherwise, if not declared as `@atomic`, this parameter must be -`:not_atomic` if specified. + getfield(value, name::Symbol, [boundscheck::Bool=true], [order::Symbol]) + getfield(value, i::Int, [boundscheck::Bool=true], [order::Symbol]) + +Extract a field from a composite `value` by name or position. + +Optionally, an ordering can be defined for the operation. If the field was +declared `@atomic`, the specification is strongly recommended to be compatible +with the stores to that location. Otherwise, if not declared as `@atomic`, this +parameter must be `:not_atomic` if specified. + +The bounds check may be disabled, in which case the behavior of this function is +undefined if `i` is out of bounds. + See also [`getproperty`](@ref Base.getproperty) and [`fieldnames`](@ref). # Examples @@ -2753,6 +2758,25 @@ See also [`setpropertyonce!`](@ref Base.setpropertyonce!) and [`setglobal!`](@re """ setglobalonce! +""" + _import(to::Module, from::Module, asname::Symbol, [sym::Symbol, imported::Bool]) + +With all five arguments, imports `sym` from module `from` into `to` with name +`asname`. `imported` is true for bindings created with `import` (set it to +false for `using A: ...`). + +With only the first three arguments, creates a binding for the module `from` +with name `asname` in `to`. +""" +Core._import + +""" + _using(to::Module, from::Module) + +Add `from` to the usings list of `to`. +""" +Core._using + """ typeof(x) @@ -2831,6 +2855,9 @@ a value set. If `allow_import` is `false`, the global variable must be defined inside `m` and may not be imported from another module. +!!! compat "Julia 1.12" + This function requires Julia 1.12 or later. + See also [`@isdefined`](@ref). # Examples diff --git a/base/docs/bindings.jl b/base/docs/bindings.jl index 5c65a35659f81..34aa87bd13076 100644 --- a/base/docs/bindings.jl +++ b/base/docs/bindings.jl @@ -16,8 +16,8 @@ end bindingexpr(x) = Expr(:call, Binding, splitexpr(x)...) -defined(b::Binding) = invokelatest(isdefined, b.mod, b.var) -resolve(b::Binding) = invokelatest(getfield, b.mod, b.var) +defined(b::Binding) = invokelatest(isdefinedglobal, b.mod, b.var) +resolve(b::Binding) = invokelatest(getglobal, b.mod, b.var) function splitexpr(x::Expr) isexpr(x, :macrocall) ? splitexpr(x.args[1]) : @@ -42,6 +42,6 @@ end aliasof(b::Binding) = defined(b) ? (a = aliasof(resolve(b), b); defined(a) ? a : b) : b aliasof(d::DataType, b) = Binding(d.name.module, d.name.name) -aliasof(λ::Function, b) = (m = typeof(λ).name.mt; Binding(m.module, m.name)) +aliasof(λ::Function, b) = (m = typeof(λ).name; Binding(m.module, m.singletonname)) aliasof(m::Module, b) = Binding(m, nameof(m)) aliasof(other, b) = b diff --git a/base/docs/intrinsicsdocs.jl b/base/docs/intrinsicsdocs.jl index 3e7b982e85377..db54c1d0dc437 100644 --- a/base/docs/intrinsicsdocs.jl +++ b/base/docs/intrinsicsdocs.jl @@ -22,6 +22,15 @@ The `Core.Intrinsics` module holds the `Core.IntrinsicFunction` objects. """ Core.Intrinsics +""" + Core.memorynew(::Type{T} where T <: GenericMemory, n::Int) + +Construct an uninitialized [`GenericMemory`](@ref) of length `n`. + +See also [`Memory`](@ref Core.Memory), [`Memory{T}(undef, n)`](@ref Core.Memory(::UndefInitializer, ::Int)). +""" +Core.memorynew + """ Core.memoryrefnew(::GenericMemory) Core.memoryrefnew(::GenericMemoryRef, index::Int, [boundscheck::Bool]) @@ -129,6 +138,35 @@ See also [`setpropertyonce!`](@ref Base.replaceproperty!) and [`Core.memoryrefse """ Core.memoryrefsetonce! + +""" + Core.Intrinsics.pointerref(p::Ptr{T}, i::Int, align::Int) + +Load a value of type `T` from the address of the `i`th element (1-indexed) +starting at `p`. This is equivalent to the C expression `p[i-1]`. + +The alignment must be a power of two, or 0, indicating the default alignment +for `T`. If `p[i-1]` is out of bounds, invalid, or is not aligned, the behavior +is undefined. An alignment of 1 is always safe. + +See also [`unsafe_load`](@ref). +""" +Core.Intrinsics.pointerref + +""" + Core.Intrinsics.pointerset(p::Ptr{T}, x::T, i::Int, align::Int) + +Store a value of type `T` to the address of the `i`th element (1-indexed) +starting at `p`. This is equivalent to the C expression `p[i-1] = x`. + +The alignment must be a power of two, or `0`, indicating the default alignment +for `T`. If `p[i-1]` is out of bounds, invalid, or is not aligned, the behavior +is undefined. An alignment of 1 is always safe. + +See also [`unsafe_store!`](@ref). +""" +Core.Intrinsics.pointerset + """ Core.Intrinsics.atomic_pointerref(pointer::Ptr{T}, order::Symbol) --> T diff --git a/base/error.jl b/base/error.jl index 3ea7210652dad..e5eecf453ee75 100644 --- a/base/error.jl +++ b/base/error.jl @@ -135,8 +135,8 @@ function catch_backtrace() return _reformat_bt(bt::Vector{Ptr{Cvoid}}, bt2::Vector{Any}) end -struct ExceptionStack <: AbstractArray{Any,1} - stack::Array{Any,1} +struct ExceptionStack <: AbstractArray{NamedTuple{(:exception, :backtrace)},1} + stack::Array{NamedTuple{(:exception, :backtrace)},1} end """ @@ -159,7 +159,7 @@ uncaught exceptions. """ function current_exceptions(task::Task=current_task(); backtrace::Bool=true) raw = ccall(:jl_get_excstack, Any, (Any,Cint,Cint), task, backtrace, typemax(Cint))::Vector{Any} - formatted = Any[] + formatted = NamedTuple{(:exception, :backtrace)}[] stride = backtrace ? 3 : 1 for i = reverse(1:stride:length(raw)) exc = raw[i] diff --git a/base/errorshow.jl b/base/errorshow.jl index c53c605cbfb86..1ae98378ff542 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -307,7 +307,7 @@ function showerror(io::IO, ex::MethodError) iob = IOContext(buf, io) # for type abbreviation as in #49795; some, like `convert(T, x)`, should not abbreviate show_signature_function(iob, Core.Typeof(f)) show_tuple_as_call(iob, :function, arg_types; hasfirst=false, kwargs = isempty(kwargs) ? nothing : kwargs) - str = String(take!(buf)) + str = takestring!(buf) str = type_limited_string_from_context(io, str) print(io, str) end @@ -327,13 +327,33 @@ function showerror(io::IO, ex::MethodError) if ft <: AbstractArray print(io, "\nIn case you're trying to index into the array, use square brackets [] instead of parentheses ().") end - # Check for local functions that shadow methods in Base - let name = ft.name.mt.name - if f_is_function && isdefined(Base, name) - basef = getfield(Base, name) - if basef !== f && hasmethod(basef, arg_types) - print(io, "\nYou may have intended to import ") - show_unquoted(io, Expr(:., :Base, QuoteNode(name))) + # Check for functions with the same name in other modules + if f_is_function && ex.world != typemax(UInt) + let name = ft.name.singletonname + modules_to_check = Set{Module}() + push!(modules_to_check, Base) + for T in san_arg_types_param + modulesof!(modules_to_check, T) + end + + # Check all modules (sorted for consistency) + sorted_modules = sort!(collect(modules_to_check), by=nameof) + for mod in sorted_modules + if isdefinedglobal(mod, name) + candidate = getglobal(mod, name) + if candidate !== f && hasmethod(candidate, arg_types; world=ex.world) + if mod === Base + print(io, "\nYou may have intended to import ") + show_unquoted(io, Expr(:., :Base, QuoteNode(name))) + else + print(io, "\nThe definition in ") + show_unquoted(io, mod) + print(io, " may have intended to extend ") + f_module = parentmodule(ft) + show_unquoted(io, Expr(:., f_module, QuoteNode(name))) + end + end + end end end end @@ -382,7 +402,7 @@ end function showerror(io::IO, exc::FieldError) @nospecialize - print(io, "FieldError: type $(exc.type |> nameof) has no field `$(exc.field)`") + print(io, "FieldError: type $(exc.type.name.wrapper) has no field `$(exc.field)`") Base.Experimental.show_error_hints(io, exc) end @@ -445,6 +465,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) line_score = Int[] # These functions are special cased to only show if first argument is matched. special = f === convert || f === getindex || f === setindex! + f isa Core.Builtin && return # `methods` isn't very useful for a builtin funcs = Tuple{Any,Vector{Any}}[(f, arg_types_param)] # An incorrect call method produces a MethodError for convert. @@ -452,7 +473,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) # pool MethodErrors for these two functions. if f === convert && !isempty(arg_types_param) at1 = arg_types_param[1] - if isType(at1) && !has_free_typevars(at1) + if isType(at1) && !has_free_typevars(at1) && at1.parameters[1] isa Type push!(funcs, (at1.parameters[1], arg_types_param[2:end])) end end @@ -474,8 +495,8 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) end sig0 = sig0::DataType s1 = sig0.parameters[1] - if sig0 === Tuple || !isa(func, rewrap_unionall(s1, method.sig)) - # function itself doesn't match or is a builtin + if !isa(func, rewrap_unionall(s1, method.sig)) + # function itself doesn't match continue else print(iob, " ") @@ -592,15 +613,13 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) end if ex.world < reinterpret(UInt, method.primary_world) print(iob, " (method too new to be called from this world context.)") - elseif ex.world > reinterpret(UInt, method.deleted_world) - print(iob, " (method deleted before this world age.)") end println(iob) m = parentmodule_before_main(method) modulecolor = get!(() -> popfirst!(STACKTRACE_MODULECOLORS), STACKTRACE_FIXEDCOLORS, m) print_module_path_file(iob, m, string(file), line; modulecolor, digit_align_width = 3) - push!(lines, String(take!(buf))) + push!(lines, takestring!(buf)) push!(line_score, -(right_matches * 2 + (length(arg_types_param) < 2 ? 1 : 0))) end end @@ -622,6 +641,7 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs=[]) println(io) # extra newline for spacing to stacktrace end end + nothing end # In case the line numbers in the source code have changed since the code was compiled, @@ -638,107 +658,146 @@ const update_stackframes_callback = Ref{Function}(identity) const STACKTRACE_MODULECOLORS = Iterators.Stateful(Iterators.cycle([:magenta, :cyan, :green, :yellow])) const STACKTRACE_FIXEDCOLORS = IdDict(Base => :light_black, Core => :light_black) -function show_full_backtrace(io::IO, trace::Vector; print_linebreaks::Bool) - num_frames = length(trace) - ndigits_max = ndigits(num_frames) - - println(io, "\nStacktrace:") - - for (i, (frame, n)) in enumerate(trace) - print_stackframe(io, i, frame, n, ndigits_max, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS) - if i < num_frames - println(io) - print_linebreaks && println(io) - end - end -end - const BIG_STACKTRACE_SIZE = 50 # Arbitrary constant chosen here -function show_reduced_backtrace(io::IO, t::Vector) +function _backtrace_find_and_remove_cycles(t) recorded_positions = IdDict{UInt, Vector{Int}}() #= For each frame of hash h, recorded_positions[h] is the list of indices i such that hash(t[i-1]) == h, ie the list of positions in which the frame appears just before. =# + max_nested_cycles = 0 displayed_stackframes = [] - repeated_cycle = Tuple{Int,Int,Int}[] - # First: line to introuce the "cycle repetition" message - # Second: length of the cycle - # Third: number of repetitions + repeated_cycles = Tuple{Int,Int,Int}[] + # First: index into `display_stackframes` to introuce the cycle bracket on + # Second: length of the cycle as a count in the trace + # Third: number of cycle repetitions + + t_curr = 1 frame_counter = 1 - while frame_counter < length(t) - (last_frame, n) = t[frame_counter] - frame_counter += 1 # Indicating the next frame - - current_hash = hash(last_frame) - positions = get(recorded_positions, current_hash, Int[]) - recorded_positions[current_hash] = push!(positions, frame_counter) - - repetitions = 0 - for index_p in length(positions)-1:-1:1 # More recent is more likely - p = positions[index_p] - cycle_length = frame_counter - p - i = frame_counter - j = p - while i < length(t) && t[i] == t[j] - i += 1 - j += 1 + + while t_curr ≤ length(t) + (last_frame, n) = t[t_curr] + current_hash = hash(t[t_curr]) + positions = get(recorded_positions, current_hash, Int[]) + + t_curr += 1 + recorded_positions[current_hash] = push!(positions, t_curr) + + # Check previous positions for cycles + ncycles = 0 + nnested_cycles = n > 0 + for k ∈ reverse(eachindex(positions))[2:end] # More recent is more likely + t_prev = positions[k] + t_cycle_length = t_curr - t_prev + + # walk trace at current and previous matching positions until matching stops + t_curr_end = t_curr + t_prev_end = t_prev + while t_curr_end < length(t) && t[t_curr_end] == t[t_prev_end] + t_curr_end += 1 + t_prev_end += 1 end - if j >= frame_counter-1 + + if t_prev_end ≥ t_curr - 1 #= At least one cycle repeated =# - repetitions = div(i - frame_counter + 1, cycle_length) - push!(repeated_cycle, (length(displayed_stackframes), cycle_length, repetitions)) - frame_counter += cycle_length * repetitions - 1 - break + ncycles = div(t_curr_end - t_prev + 1, t_cycle_length) + push!(repeated_cycles, (length(displayed_stackframes) - 1, t_cycle_length, ncycles)) + t_curr += t_cycle_length * (ncycles - 1) - 1 + nnested_cycles += 1 end end - if repetitions==0 + # ensure an outer cycle comes before a contained inner cycle + sort!(repeated_cycles, by = x -> (x[1], -x[2])) + max_nested_cycles = max(max_nested_cycles, nnested_cycles) + + if ncycles == 0 push!(displayed_stackframes, (last_frame, n)) end end + return displayed_stackframes, repeated_cycles, max_nested_cycles +end + +function _backtrace_print_repetition_closings!(io::IO, i, current_cycles, frame_counter, max_nested_cycles, nactive_cycles, ndigits_max; prefix = nothing) + while !isempty(current_cycles) + start_line = current_cycles[end][1] + cycle_length = current_cycles[end][2] + end_line = start_line + cycle_length - 1 + repetitions = current_cycles[end][3] + frame_counter_advance = current_cycles[end][4] + + i != end_line && break + + println(io) + prefix === nothing || print(io, prefix) + line_length = (max_nested_cycles - nactive_cycles) + ndigits_max + 2 + nactive_cycles -= 1 + printstyled(io, " ", "│" ^ nactive_cycles, "╰", "─" ^ (line_length); color = :light_black) + printstyled(io, " repeated $repetitions times"; color = :light_black, italic = true) + + pop!(current_cycles) + + if cycle_length > 1 + # adjust cycle_length in outer cycles to reflect displayed frames consumed by this inner cycle + for j ∈ eachindex(current_cycles) + current_cycles[j] = (current_cycles[j][1], current_cycles[j][2] - cycle_length * (repetitions - 1), current_cycles[j][3:4]...) + end + else + # adjust frame_counter_advance in outer cycles to reflect frames consumed by a single repeated frame + for j ∈ eachindex(current_cycles) + current_cycles[j] = (current_cycles[j][1:3]..., current_cycles[j][4] + (frame_counter_advance * (current_cycles[j][3] - 1))) + end + end + + frame_counter += frame_counter_advance + end + return frame_counter, nactive_cycles +end - try invokelatest(update_stackframes_callback[], displayed_stackframes) catch end +function show_processed_backtrace(io::IO, trace::Vector, num_frames::Int, repeated_cycles::Vector{NTuple{3, Int}}, max_nested_cycles::Int; print_linebreaks::Bool, prefix = nothing) + println(io) + prefix === nothing || print(io, prefix) + println(io, "Stacktrace:") - println(io, "\nStacktrace:") + ndigits_max = ndigits(num_frames) - ndigits_max = ndigits(length(t)) + push!(repeated_cycles, (0,0,0)) # repeated_cycles is never empty - push!(repeated_cycle, (0,0,0)) # repeated_cycle is never empty frame_counter = 1 - for i in eachindex(displayed_stackframes) - (frame, n) = displayed_stackframes[i] + current_cycles = NTuple{4, Int}[] # adding a value to track amount to advance frame_counter when cycle is closed - print_stackframe(io, frame_counter, frame, n, ndigits_max, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS) + for i in eachindex(trace) + (frame, n) = trace[i] - if i < length(displayed_stackframes) - println(io) - stacktrace_linebreaks() && println(io) + ncycle_starts = 0 + while repeated_cycles[1][1] == i + cycle = popfirst!(repeated_cycles) + push!(current_cycles, (cycle..., cycle[2] * (cycle[3] - 1))) + ncycle_starts += 1 end - - while repeated_cycle[1][1] == i # never empty because of the initial (0,0,0) - cycle_length = repeated_cycle[1][2] - repetitions = repeated_cycle[1][3] - popfirst!(repeated_cycle) - printstyled(io, - "--- the above ", cycle_length, " lines are repeated ", - repetitions, " more time", repetitions>1 ? "s" : "", " ---", color = :light_black) - if i < length(displayed_stackframes) - println(io) - stacktrace_linebreaks() && println(io) - end - frame_counter += cycle_length * repetitions + if n > 1 + push!(current_cycles, (i, 1, n, n - 1)) + ncycle_starts += 1 end + nactive_cycles = length(current_cycles) + + print_stackframe(io, frame_counter, frame, ndigits_max, max_nested_cycles, nactive_cycles, ncycle_starts, STACKTRACE_FIXEDCOLORS, STACKTRACE_MODULECOLORS; prefix) + + frame_counter, nactive_cycles = _backtrace_print_repetition_closings!(io, i, current_cycles, frame_counter, max_nested_cycles, nactive_cycles, ndigits_max; prefix) frame_counter += 1 + + if i < length(trace) + println(io) + print_linebreaks && println(io) + end end end - # Print a stack frame where the module color is determined by looking up the parent module in # `modulecolordict`. If the module does not have a color, yet, a new one can be drawn # from `modulecolorcycler`. -function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulecolordict, modulecolorcycler) +function print_stackframe(io, i, frame::StackFrame, ndigits_max::Int, max_nested_cycles::Int, nactive_cycles::Int, ncycle_starts::Int, modulecolordict, modulecolorcycler; prefix = nothing) m = Base.parentmodule(frame) modulecolor = if m !== nothing m = parentmodule_before_main(m) @@ -746,7 +805,7 @@ function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulec else :default end - print_stackframe(io, i, frame, n, ndigits_max, modulecolor) + print_stackframe(io, i, frame, ndigits_max, max_nested_cycles, nactive_cycles, ncycle_starts, modulecolor; prefix) end # Gets the topmost parent module that isn't Main @@ -761,7 +820,7 @@ end parentmodule_before_main(x) = parentmodule_before_main(parentmodule(x)) # Print a stack frame where the module color is set manually with `modulecolor`. -function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulecolor) +function print_stackframe(io, i, frame::StackFrame, ndigits_max::Int, max_nested_cycles::Int, nactive_cycles::Int, ncycle_starts::Int, modulecolor; prefix = nothing) file, line = string(frame.file), frame.line # Used by the REPL to make it possible to open @@ -773,20 +832,29 @@ function print_stackframe(io, i, frame::StackFrame, n::Int, ndigits_max, modulec inlined = getfield(frame, :inlined) modul = parentmodule(frame) - digit_align_width = ndigits_max + 2 + digit_align_width = ndigits_max + 2 + max_nested_cycles - nactive_cycles + + # repeated section bracket line 1 + prefix === nothing || print(io, prefix) + print(io, " ") + printstyled(io, "├" ^ (nactive_cycles - ncycle_starts); color = :light_black) + printstyled(io, "┌" ^ ncycle_starts; color = :light_black) # frame number - print(io, " ", lpad("[" * string(i) * "]", digit_align_width)) + print(io, lpad("[" * string(i) * "]", digit_align_width)) print(io, " ") + # func name and arguments StackTraces.show_spec_linfo(IOContext(io, :backtrace=>true), frame) - if n > 1 - printstyled(io, " (repeats $n times)"; color=Base.warn_color(), bold=true) - end println(io) + # repeated section bracket line 2 + prefix === nothing || print(io, prefix) + print(io, " ") + printstyled(io, "│" ^ nactive_cycles; color = :light_black) + # @ Module path / file : line - print_module_path_file(io, modul, file, line; modulecolor, digit_align_width) + print_module_path_file(io, modul, file, line; modulecolor, digit_align_width = digit_align_width - 1) # inlined printstyled(io, inlined ? " [inlined]" : "", color = :light_black) @@ -813,42 +881,112 @@ function print_module_path_file(io, modul, file, line; modulecolor = :light_blac printstyled(io, basename(file), ":", line; color = :light_black, underline = true) end -function show_backtrace(io::IO, t::Vector) +#= + +Stacktrace processing pipeline: +1. Raw traces extracted with `backtrace` or `catch_backtrace` as vector of instruction pointers. +2. IP traces converted to frames with `stacktrace`, which may or may not include C frames. +3. Originator trims frames related to itself (e.g. REPL removes REPL-specific frames) + - CapturedException only keeps a limit of 100 frames by processing before display +4. `process_backtrace` filters a trace for internal implementation or redundant frames and summarizes repeated single frames: + - `kwcall` frames removed + - `include`-related stack frames removed + - Some frames that have the same location info are merged + - Repeated frames are removed and summarized with a count + - Output is an Any[] containing (StackFrame, count) tuple elements and this form is exposed to e.g. Revise +5. If a trace is too long, cycles are identified and summarized +6. `update_stackframes_callback[]` provides e.g. Revise an opportunity to edit line info + +=# + +function show_backtrace(io::IO, t::Vector; prefix = nothing) if haskey(io, :last_shown_line_infos) empty!(io[:last_shown_line_infos]) end - # t is a pre-processed backtrace (ref #12856) + # Process backtrace if it has not yet been. A processed backtrace is a Vector{Any} + # with elements of type Tuple{StackFrame, Int}. (ref #12856) if t isa Vector{Any} && (length(t) == 0 || t[1] isa Tuple{StackFrame,Int}) filtered = t else - filtered = process_backtrace(t) + # t is a raw trace requiring lookup + if t isa Vector{<:Union{Base.InterpreterIP,Ptr{Cvoid}}} + frametrace = stacktrace(t) + else + frametrace = t + end + filtered = process_backtrace(frametrace) end isempty(filtered) && return - if length(filtered) == 1 && StackTraces.is_top_level_frame(filtered[1][1]) + nframes = sum(last(x) for x ∈ filtered) + + # don't show a single top-level frame with no location info + if nframes == 1 && StackTraces.is_top_level_frame(filtered[1][1]) f = filtered[1][1]::StackFrame if f.line == 0 && f.file === :var"" - # don't show a single top-level frame with no location info return end end + # Find repeated cycles if trace is too long if length(filtered) > BIG_STACKTRACE_SIZE - show_reduced_backtrace(IOContext(io, :backtrace => true), filtered) - return + filtered, repeated_cycles, max_nested_cycles = _backtrace_find_and_remove_cycles(filtered) else - try invokelatest(update_stackframes_callback[], filtered) catch end - # process_backtrace returns a Vector{Tuple{Frame, Int}} - show_full_backtrace(io, filtered; print_linebreaks = stacktrace_linebreaks()) + repeated_cycles = NTuple{3, Int}[] + max_nested_cycles = any(x -> last(x) > 1, filtered) ? 1 : 0 end + + # Allow external code to edit information in the frames (e.g. line numbers with Revise) + try invokelatest(update_stackframes_callback[], filtered) catch end + + show_processed_backtrace(IOContext(io, :backtrace => true), filtered, nframes, repeated_cycles, max_nested_cycles; print_linebreaks = stacktrace_linebreaks(), prefix) nothing end +function _backtrace_collapse_and_count_repeated_frames(frames::Vector{StackFrame}) + n = 0 + last_frame = StackTraces.UNKNOWN + tracecount = Any[] + for frame in frames + if frame.file != last_frame.file || frame.line != last_frame.line || frame.func != last_frame.func || frame.linfo !== last_frame.linfo + if n > 0 + push!(tracecount, (last_frame, n)) + end + n = 1 + last_frame = frame + else + n += 1 + end + end + if n > 0 + push!(tracecount, (last_frame, n)) + end + return tracecount +end + +function _backtrace_remove_kwcall_frames!(trace) + todelete = findall(trace) do (frame, _) + code = frame.linfo + if code isa MethodInstance + def = code.def + if def isa Method && def.name !== :kwcall && def.sig <: Tuple{typeof(Core.kwcall),NamedTuple,Any,Vararg} + # hide kwcall() methods, which are probably internal keyword sorter methods + # (we print the internal method instead, after demangling + # the argument list, since it has the right line number info) + return true + end + else + frame.func === :kwcall && return true + end + return false + end + deleteat!(trace, todelete) +end # For improved user experience, filter out frames for include() implementation # - see #33065. See also #35371 for extended discussion of internal frames. -function _simplify_include_frames(trace) +function _backtrace_simplify_include_frames!(trace) kept_frames = trues(length(trace)) first_ignored = nothing for i in length(trace):-1:1 @@ -880,11 +1018,11 @@ function _simplify_include_frames(trace) if first_ignored !== nothing kept_frames[1:first_ignored] .= false end - return trace[kept_frames] + keepat!(trace, kept_frames) end # Collapse frames that have the same location (in some cases) -function _collapse_repeated_frames(trace) +function _backtrace_collapse_repeated_locations!(trace) kept_frames = trues(length(trace)) last_frame = nothing for i in eachindex(trace) @@ -945,63 +1083,19 @@ function _collapse_repeated_frames(trace) end last_frame = frame end - return trace[kept_frames] + keepat!(trace, kept_frames) end -function process_backtrace(t::Vector, limit::Int=typemax(Int); skipC = true) - n = 0 - last_frame = StackTraces.UNKNOWN - count = 0 - ret = Any[] - for i in eachindex(t) - lkups = t[i] - if lkups isa StackFrame - lkups = [lkups] - else - lkups = StackTraces.lookup(lkups) - end - for lkup in lkups - if lkup === StackTraces.UNKNOWN - continue - end - - if (lkup.from_c && skipC) - continue - end - if lkup.linfo isa Union{MethodInstance, CodeInstance} - def = StackTraces.frame_method_or_module(lkup) - if def isa Method && def.name !== :kwcall && def.sig <: Tuple{typeof(Core.kwcall),NamedTuple,Any,Vararg} - # hide kwcall() methods, which are probably internal keyword sorter methods - # (we print the internal method instead, after demangling - # the argument list, since it has the right line number info) - continue - end - elseif !lkup.from_c - lkup.func === :kwcall && continue - end - count += 1 - if count > limit - break - end +function process_backtrace(t::Vector{StackFrame}) + tracecount = _backtrace_collapse_and_count_repeated_frames(t) + process_backtrace(tracecount) +end - if lkup.file != last_frame.file || lkup.line != last_frame.line || lkup.func != last_frame.func || lkup.linfo !== last_frame.linfo - if n > 0 - push!(ret, (last_frame, n)) - end - n = 1 - last_frame = lkup - else - n += 1 - end - end - count > limit && break - end - if n > 0 - push!(ret, (last_frame, n)) - end - trace = _simplify_include_frames(ret) - trace = _collapse_repeated_frames(trace) - return trace +function process_backtrace(tracecount::Vector{Any}) + _backtrace_remove_kwcall_frames!(tracecount) + _backtrace_simplify_include_frames!(tracecount) + _backtrace_collapse_repeated_locations!(tracecount) + return tracecount end function show_exception_stack(io::IO, stack) @@ -1051,7 +1145,7 @@ Experimental.register_error_hint(noncallable_number_hint_handler, MethodError) # eg: d = Dict; d["key"] = 2 function nonsetable_type_hint_handler(io, ex, arg_types, kwargs) @nospecialize - if ex.f == setindex! + if ex.f === setindex! T = arg_types[1] if T <: Number print(io, "\nAre you trying to index into an array? For multi-dimensional arrays, separate the indices with commas: ") @@ -1070,9 +1164,8 @@ Experimental.register_error_hint(nonsetable_type_hint_handler, MethodError) # Display a hint in case the user tries to use the + operator on strings # (probably attempting concatenation) -function string_concatenation_hint_handler(io, ex, arg_types, kwargs) - @nospecialize - if (ex.f === +) && !isempty(arg_types) && all(i -> i <: AbstractString, arg_types) +function string_concatenation_hint_handler(@nospecialize(io::IO), ex::MethodError, arg_types::Vector{Any}, kwargs::Vector{Any}) + if (ex.f === +) && !isempty(arg_types) && all(@nospecialize(a) -> unwrapva(a) <: AbstractString, arg_types) print(io, "\nString concatenation is performed with ") printstyled(io, "*", color=:cyan) print(io, " (See also: https://docs.julialang.org/en/v1/manual/strings/#man-concatenation).") @@ -1123,7 +1216,7 @@ Experimental.register_error_hint(fielderror_dict_hint_handler, FieldError) function fielderror_listfields_hint_handler(io, exc) fields = fieldnames(exc.type) if isempty(fields) - print(io, "; $(nameof(exc.type)) has no fields at all.") + print(io, "; $(exc.type.name.wrapper) has no fields at all.") else print(io, ", available fields: $(join(map(k -> "`$k`", fields), ", "))") end @@ -1150,19 +1243,41 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) if isdefined(ex, :scope) scope = ex.scope if scope isa Module - bpart = Base.lookup_binding_partition(ex.world, GlobalRef(scope, var)) - kind = Base.binding_kind(bpart) - if kind === Base.PARTITION_KIND_GLOBAL || kind === Base.PARTITION_KIND_UNDEF_CONST || kind == Base.PARTITION_KIND_DECLARED + bpart = lookup_binding_partition(ex.world, GlobalRef(scope, var)) + kind = binding_kind(bpart) + + # Get the current world's binding partition for comparison + curworld = tls_world_age() + cur_bpart = lookup_binding_partition(curworld, GlobalRef(scope, var)) + cur_kind = binding_kind(cur_bpart) + + # Track if we printed the "too new" message + printed_too_new = false + + # Check if the binding exists in the current world but was undefined in the error's world + if kind === PARTITION_KIND_GUARD + if isdefinedglobal(scope, var) + print(io, "\nThe binding may be too new: running in world age $(ex.world), while current world is $(curworld).") + printed_too_new = true + else + print(io, "\nSuggestion: check for spelling errors or missing imports.") + end + elseif kind === PARTITION_KIND_GLOBAL || kind === PARTITION_KIND_UNDEF_CONST || kind == PARTITION_KIND_DECLARED print(io, "\nSuggestion: add an appropriate import or assignment. This global was declared but not assigned.") - elseif kind === Base.PARTITION_KIND_FAILED + elseif kind === PARTITION_KIND_FAILED print(io, "\nHint: It looks like two or more modules export different ", "bindings with this name, resulting in ambiguity. Try explicitly ", "importing it from a particular module, or qualifying the name ", "with the module it should come from.") - elseif kind === Base.PARTITION_KIND_GUARD - print(io, "\nSuggestion: check for spelling errors or missing imports.") - elseif Base.is_some_explicit_imported(kind) - print(io, "\nSuggestion: this global was defined as `$(Base.partition_restriction(bpart).globalref)` but not assigned a value.") + elseif is_some_explicit_imported(kind) + print(io, "\nSuggestion: this global was defined as `$(partition_restriction(bpart).globalref)` but not assigned a value.") + elseif kind === PARTITION_KIND_BACKDATED_CONST + print(io, "\nSuggestion: define the const at top-level before running function that uses it (stricter Julia v1.12+ rule).") + end + + # Check if binding kind changed between the error's world and current world + if !printed_too_new && kind !== cur_kind + print(io, "\nNote: the binding state changed since the error occurred (was: $(kind), now: $(cur_kind)).") end elseif scope === :static_parameter print(io, "\nSuggestion: run Test.detect_unbound_args to detect method arguments that do not fully constrain a type parameter.") diff --git a/base/essentials.jl b/base/essentials.jl index 029205666b2bc..f27346c4b43cb 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Core: CodeInfo, SimpleVector, donotdelete, compilerbarrier, memoryrefnew, memoryrefget, memoryrefset! +using Core: CodeInfo, SimpleVector, donotdelete, compilerbarrier, memoryref, memoryrefnew, memoryrefget, memoryrefset! const Callable = Union{Function,Type} @@ -377,13 +377,27 @@ macro _nospecializeinfer_meta() return Expr(:meta, :nospecializeinfer) end +# These special checkbounds methods are defined early for bootstrapping +function checkbounds(::Type{Bool}, A::Union{Array, Memory}, i::Int) + @inline + ult_int(bitcast(UInt, sub_int(i, 1)), bitcast(UInt, length(A))) +end +function checkbounds(A::Union{Array, GenericMemory}, i::Int) + @inline + checkbounds(Bool, A, i) || throw_boundserror(A, (i,)) +end + default_access_order(a::GenericMemory{:not_atomic}) = :not_atomic default_access_order(a::GenericMemory{:atomic}) = :monotonic default_access_order(a::GenericMemoryRef{:not_atomic}) = :not_atomic default_access_order(a::GenericMemoryRef{:atomic}) = :monotonic -getindex(A::GenericMemory, i::Int) = (@_noub_if_noinbounds_meta; - memoryrefget(memoryrefnew(memoryrefnew(A), i, @_boundscheck), default_access_order(A), false)) +function getindex(A::GenericMemory, i::Int) + @_noub_if_noinbounds_meta + (@_boundscheck) && checkbounds(A, i) + memoryrefget(memoryrefnew(A, i, false), default_access_order(A), false) +end + getindex(A::GenericMemoryRef) = memoryrefget(A, default_access_order(A), @_boundscheck) """ @@ -421,7 +435,7 @@ Stacktrace: [...] ``` -If `T` is a [`AbstractFloat`](@ref) type, then it will return the +If `T` is an [`AbstractFloat`](@ref) type, then it will return the closest value to `x` representable by `T`. Inf is treated as one ulp greater than `floatmax(T)` for purposes of determining nearest. @@ -717,12 +731,14 @@ Neither `convert` nor `cconvert` should take a Julia object and turn it into a ` """ function cconvert end -cconvert(T::Type, x) = x isa T ? x : convert(T, x) # do the conversion eagerly in most cases +cconvert(::Type{T}, x) where {T} = x isa T ? x : convert(T, x) # do the conversion eagerly in most cases cconvert(::Type{Union{}}, x...) = convert(Union{}, x...) cconvert(::Type{<:Ptr}, x) = x # but defer the conversion to Ptr to unsafe_convert unsafe_convert(::Type{T}, x::T) where {T} = x # unsafe_convert (like convert) defaults to assuming the convert occurred unsafe_convert(::Type{T}, x::T) where {T<:Ptr} = x # to resolve ambiguity with the next method unsafe_convert(::Type{P}, x::Ptr) where {P<:Ptr} = convert(P, x) +unsafe_convert(::Type{Ptr{UInt8}}, s::String) = ccall(:jl_string_ptr, Ptr{UInt8}, (Any,), s) +unsafe_convert(::Type{Ptr{Int8}}, s::String) = ccall(:jl_string_ptr, Ptr{Int8}, (Any,), s) """ reinterpret(::Type{Out}, x::In) @@ -947,17 +963,17 @@ end # linear indexing function getindex(A::Array, i::Int) @_noub_if_noinbounds_meta - @boundscheck ult_int(bitcast(UInt, sub_int(i, 1)), bitcast(UInt, length(A))) || throw_boundserror(A, (i,)) + @boundscheck checkbounds(A, i) memoryrefget(memoryrefnew(getfield(A, :ref), i, false), :not_atomic, false) end # simple Array{Any} operations needed for bootstrap function setindex!(A::Array{Any}, @nospecialize(x), i::Int) @_noub_if_noinbounds_meta - @boundscheck ult_int(bitcast(UInt, sub_int(i, 1)), bitcast(UInt, length(A))) || throw_boundserror(A, (i,)) + @boundscheck checkbounds(A, i) memoryrefset!(memoryrefnew(getfield(A, :ref), i, false), x, :not_atomic, false) return A end -setindex!(A::Memory{Any}, @nospecialize(x), i::Int) = (memoryrefset!(memoryrefnew(memoryrefnew(A), i, @_boundscheck), x, :not_atomic, @_boundscheck); A) +setindex!(A::Memory{Any}, @nospecialize(x), i::Int) = (memoryrefset!(memoryrefnew(A, i, @_boundscheck), x, :not_atomic, @_boundscheck); A) setindex!(A::MemoryRef{T}, x) where {T} = (memoryrefset!(A, convert(T, x), :not_atomic, @_boundscheck); A) setindex!(A::MemoryRef{Any}, @nospecialize(x)) = (memoryrefset!(A, x, :not_atomic, @_boundscheck); A) @@ -1209,8 +1225,9 @@ Stateful iterators that want to opt into this feature should define an `isdone` method that returns true/false depending on whether the iterator is done or not. Stateless iterators need not implement this function. -If the result is `missing`, callers may go ahead and compute -`iterate(x, state) === nothing` to compute a definite answer. +If the result is `missing`, then `isdone` cannot determine whether the iterator +state is terminal, and callers must compute `iterate(itr, state) === nothing` +to obtain a definitive answer. See also [`iterate`](@ref), [`isempty`](@ref) """ @@ -1298,5 +1315,3 @@ typename(typeof(function <= end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC typename(typeof(function >= end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC typename(typeof(function < end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC typename(typeof(function > end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC -typename(typeof(function << end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC -typename(typeof(function >> end)).constprop_heuristic = Core.SAMETYPE_HEURISTIC diff --git a/base/exports.jl b/base/exports.jl index 2e0bb3ccfe4cf..2c30f095a3998 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -338,7 +338,9 @@ export isinf, isinteger, isnan, + isnegative, isodd, + ispositive, ispow2, isqrt, isreal, @@ -680,6 +682,7 @@ export split, string, strip, + takestring!, textwidth, thisind, titlecase, @@ -807,6 +810,7 @@ export fieldoffset, fieldname, fieldnames, + fieldindex, fieldcount, fieldtypes, hasfield, @@ -1055,6 +1059,7 @@ export @__DIR__, @__LINE__, @__MODULE__, + @__FUNCTION__, @int128_str, @uint128_str, @big_str, diff --git a/base/expr.jl b/base/expr.jl index 634e2ddb7b4de..b44c9336024e5 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -60,7 +60,8 @@ function copy(x::PhiCNode) return PhiCNode(new_values) end -# copy parts of an AST that the compiler mutates +# copy parts of an IR that the compiler mutates +# (this is not a general-purpose copy for an Expr AST) function copy_exprs(@nospecialize(x)) if isa(x, Expr) return copy(x) @@ -91,10 +92,86 @@ function copy(c::CodeInfo) return cnew end +function isequal_exprarg(@nospecialize(x), @nospecialize(y)) + x isa typeof(y) || return false + x === y && return true + # c.f. list of types in copy_expr also + if x isa Expr + x == (y::Expr) && return true + elseif x isa QuoteNode + x == (y::QuoteNode) && return true + elseif x isa PhiNode + x == (y::PhiNode) && return true + elseif x isa PhiCNode + x == (y::PhiCNode) && return true + elseif x isa CodeInfo + x == (y::CodeInfo) && return true + end + return false +end + + +function isequal_exprargs(x::Array{Any,1}, y::Array{Any,1}) + l = length(x) + l == length(y) || return false + for i = 1:l + if !isassigned(x, i) + # phi and phic values are permitted to be undef + isassigned(y, i) && return false + else + isassigned(y, i) || return false + isequal_exprarg(x[i], y[i]) || return false + end + end + return true +end -==(x::Expr, y::Expr) = x.head === y.head && isequal(x.args, y.args) -==(x::QuoteNode, y::QuoteNode) = isequal(x.value, y.value) -==(stmt1::Core.PhiNode, stmt2::Core.PhiNode) = stmt1.edges == stmt2.edges && stmt1.values == stmt2.values +# define == such that == inputs to parsing (including line numbers) yield == outputs from lowering (including all metadata) +# (aside from cases where parsing just returns a number, which are ambiguous here) +==(x::Expr, y::Expr) = x.head === y.head && isequal_exprargs(x.args, y.args) + +==(x::QuoteNode, y::QuoteNode) = isequal_exprarg(x.value, y.value) + +==(stmt1::Core.PhiNode, stmt2::Core.PhiNode) = isequal(stmt1.edges, stmt2.edges) && isequal_exprargs(stmt1.values, stmt2.values) + +==(stmt1::Core.PhiCNode, stmt2::Core.PhiCNode) = isequal_exprargs(stmt1.values, stmt2.values) + +function ==(stmt1::CodeInfo, stmt2::CodeInfo) + for i in 1:nfields(stmt1) + if !isdefined(stmt1, i) + isdefined(stmt2, i) && return false + else + isdefined(stmt2, i) || return false + f1 = getfield(stmt1, i) + f2 = getfield(stmt2, i) + f1 isa typeof(f2) || return false + if f1 isa Vector{Any} + # code or types vectors + isequal_exprargs(f1, f2::Vector{Any}) || return false + elseif f1 isa DebugInfo + f1 == f2::DebugInfo || return false + elseif f1 isa Vector + # misc data + l = length(f1) + l == length(f2::Vector) || return false + for i = 1:l + f1[i] === f2[i] || return false + end + else + # misc fields + f1 === f2 || return false + end + end + end + return true +end + +function ==(x::DebugInfo, y::DebugInfo) + for i in 1:nfields(x) + getfield(x, i) == getfield(y, i) || return false + end + return true +end """ macroexpand(m::Module, x; recursive=true) @@ -144,7 +221,7 @@ There are differences between `@macroexpand` and [`macroexpand`](@ref). expands with respect to the module in which it is called. This is best seen in the following example: -```julia-repl +```jldoctest julia> module M macro m() 1 @@ -152,7 +229,7 @@ julia> module M function f() (@macroexpand(@m), macroexpand(M, :(@m)), - macroexpand(Main, :(@m)) + macroexpand(parentmodule(M), :(@m)) ) end end @@ -540,16 +617,20 @@ The `:consistent` setting asserts that for egal (`===`) inputs: contents) are not egal. !!! note - The `:consistent`-cy assertion is made world-age wise. More formally, write - ``fᵢ`` for the evaluation of ``f`` in world-age ``i``, then this setting requires: + The `:consistent`-cy assertion is made with respect to a particular world range `R`. + More formally, write ``fᵢ`` for the evaluation of ``f`` in world-age ``i``, then this setting requires: ```math - ∀ i, x, y: x ≡ y → fᵢ(x) ≡ fᵢ(y) + ∀ i ∈ R, j ∈ R, x, y: x ≡ y → fᵢ(x) ≡ fⱼ(y) ``` - However, for two world ages ``i``, ``j`` s.t. ``i ≠ j``, we may have ``fᵢ(x) ≢ fⱼ(y)``. + + For `@assume_effects`, the range `R` is `m.primary_world:m.deleted_world` of + the annotated or containing method. + + For ordinary code instances, `R` is `ci.min_world:ci.max_world`. A further implication is that `:consistent` functions may not make their return value dependent on the state of the heap or any other global state - that is not constant for a given world age. + that is not constant over the given world age range. !!! note The `:consistent`-cy includes all legal rewrites performed by the optimizer. @@ -1658,21 +1739,54 @@ end is_meta_expr_head(head::Symbol) = head === :boundscheck || head === :meta || head === :loopinfo is_meta_expr(@nospecialize x) = isa(x, Expr) && is_meta_expr_head(x.head) -function is_self_quoting(@nospecialize(x)) - return isa(x,Number) || isa(x,AbstractString) || isa(x,Tuple) || isa(x,Type) || - isa(x,Char) || x === nothing || isa(x,Function) -end +""" + isa_ast_node(x) -function quoted(@nospecialize(x)) - return is_self_quoting(x) ? x : QuoteNode(x) -end +Return false if `x` is not interpreted specially by any of inference, lowering, +or codegen as either an AST or IR special form. +""" +function isa_ast_node(@nospecialize x) + # c.f. Core.IR module, augmented with AST types + return x isa NewvarNode || + x isa CodeInfo || + x isa LineNumberNode || + x isa GotoNode || + x isa GotoIfNot || + x isa EnterNode || + x isa ReturnNode || + x isa SSAValue || + x isa SlotNumber || + x isa Argument || + x isa QuoteNode || + x isa GlobalRef || + x isa Symbol || + x isa PiNode || + x isa PhiNode || + x isa PhiCNode || + x isa UpsilonNode || + x isa Expr +end + +is_self_quoting(@nospecialize(x)) = !isa_ast_node(x) + +""" + quoted(x) + +Return `x` made safe for inserting as a constant into IR. Note that this does +not make it safe for inserting into an AST, since eval will sometimes copy some +types of AST object inside, and even may sometimes evaluate and interpolate any +`\$` inside, depending on the context. +""" +quoted(@nospecialize(x)) = isa_ast_node(x) ? QuoteNode(x) : x # Implementation of generated functions function generated_body_to_codeinfo(ex::Expr, defmod::Module, isva::Bool) - ci = ccall(:jl_expand, Any, (Any, Any), ex, defmod) + ci = ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), + ex, defmod, "none", 0, typemax(Csize_t), 0)[1] if !isa(ci, CodeInfo) if isa(ci, Expr) && ci.head === :error - error("syntax: $(ci.args[1])") + msg = ci.args[1] + error(msg isa String ? strcat("syntax: ", msg) : msg) end error("The function body AST defined by this @generated function is not pure. This likely means it contains a closure, a comprehension or a generator.") end diff --git a/base/file.jl b/base/file.jl index 7fbb08c4b6fe6..24a21396721d3 100644 --- a/base/file.jl +++ b/base/file.jl @@ -164,7 +164,7 @@ required intermediate directories. Return `path`. # Examples -```julia-repl +```jldoctest; setup = :(curdir = pwd(); testdir = mktempdir(); cd(testdir)), teardown = :(cd(curdir); rm(testdir, recursive=true)), filter = r"^\\".*testingdir\\"\$" julia> mkdir("testingdir") "testingdir" @@ -516,7 +516,7 @@ If the file does not exist a new file is created. Return `path`. # Examples -```julia-repl +```jldoctest; setup = :(curdir = pwd(); testdir = mktempdir(); cd(testdir)), teardown = :(cd(curdir); rm(testdir, recursive=true)), filter = r"[\\d\\.]+e[\\+\\-]?\\d+" julia> write("my_little_file", 2); julia> mtime("my_little_file") @@ -1134,7 +1134,7 @@ for (path, dirs, files) in walkdir(".") end ``` -```julia-repl +```jldoctest; setup = :(prevdir = pwd(); tmpdir = mktempdir(); cd(tmpdir)), teardown = :(cd(prevdir); rm(tmpdir, recursive=true)) julia> mkpath("my/test/dir"); julia> itr = walkdir("my"); diff --git a/base/filesystem.jl b/base/filesystem.jl index 87b3552c80a2f..a5f1327b5cff1 100644 --- a/base/filesystem.jl +++ b/base/filesystem.jl @@ -139,9 +139,9 @@ export File, import .Base: IOError, _UVError, _sizeof_uv_fs, check_open, close, closewrite, eof, eventloop, fd, isopen, bytesavailable, position, read, read!, readbytes!, readavailable, seek, seekend, show, - skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error, + skip, stat, unsafe_read, unsafe_write, write, transcode, uv_error, _uv_error, setup_stdio, rawhandle, OS_HANDLE, INVALID_OS_HANDLE, windowserror, filesize, - isexecutable, isreadable, iswritable, MutableDenseArrayType, truncate + isexecutable, isreadable, iswritable, MutableDenseArrayType, truncate, unsafe_takestring! import .Base.RefValue diff --git a/base/flparse.jl b/base/flfrontend.jl similarity index 71% rename from base/flparse.jl rename to base/flfrontend.jl index 8b474cf148fb2..86b291cf7328b 100644 --- a/base/flparse.jl +++ b/base/flfrontend.jl @@ -17,3 +17,10 @@ end function fl_parse(text::AbstractString, filename::AbstractString, lineno, offset, options) fl_parse(String(text), String(filename), lineno, offset, options) end + +function fl_lower(ex, mod::Module, filename::Union{String,Ptr{UInt8}}="none", + lineno=0, world::Unsigned=typemax(Csize_t), warn::Bool=false) + warn = warn ? 1 : 0 + ccall(:jl_fl_lower, Any, (Any, Any, Ptr{UInt8}, Csize_t, Csize_t, Cint), + ex, mod, filename, lineno, world, warn) +end diff --git a/base/float.jl b/base/float.jl index ff64ca93760ed..d5bfd044cf3b0 100644 --- a/base/float.jl +++ b/base/float.jl @@ -83,7 +83,7 @@ julia> NaN == NaN, isequal(NaN, NaN), isnan(NaN) !!! note Always use [`isnan`](@ref) or [`isequal`](@ref) for checking for `NaN`. Using `x === NaN` may give unexpected results: - ```julia-repl + ```jldoctest julia> reinterpret(UInt32, NaN32) 0x7fc00000 @@ -603,7 +603,10 @@ function rem(x::T, y::T) where {T<:IEEEFloat} end end -function mod(x::T, y::T) where {T<:AbstractFloat} +function mod(x::T, y::T) where T<:AbstractFloat + if isinf(y) && isfinite(x) + return x + end r = rem(x,y) if r == 0 copysign(r,y) @@ -1061,6 +1064,13 @@ julia> 1.0 + eps() julia> 1.0 + eps()/2 1.0 ``` + +More generally, for any floating-point numeric type, `eps` corresponds to an +upper bound on the distance to the nearest floating-point complex value: if ``\text{fl}(x)`` is the closest +floating-point value to a number ``x`` (e.g. an arbitrary real number), then ``\text{fl}(x)`` +satisfies ``|x - \text{fl}(x)| ≤ \text{eps}(x)/2``, not including overflow cases. +This allows the definition of `eps` to be extended to complex numbers, +for which ``\text{fl}(a + ib) = \text{fl}(a) + i \text{fl}(b)``. """ eps(::Type{<:AbstractFloat}) diff --git a/base/generator.jl b/base/generator.jl index eb1465d7e1e11..26bb7c7d91b5d 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -98,7 +98,7 @@ IteratorSize(::Type{Any}) = SizeUnknown() IteratorSize(::Type{<:Tuple}) = HasLength() IteratorSize(::Type{<:AbstractArray{<:Any,N}}) where {N} = HasShape{N}() -IteratorSize(::Type{Generator{I,F}}) where {I,F} = IteratorSize(I) +IteratorSize(::Type{<:Generator{I}}) where {I} = (@isdefined I) ? IteratorSize(I) : SizeUnknown() haslength(iter) = IteratorSize(iter) isa Union{HasShape, HasLength} diff --git a/base/genericmemory.jl b/base/genericmemory.jl index 2a33336c0aad6..b180462115f41 100644 --- a/base/genericmemory.jl +++ b/base/genericmemory.jl @@ -106,7 +106,7 @@ sizeof(a::GenericMemory) = Core.sizeof(a) # multi arg case will be overwritten later. This is needed for bootstrapping function isassigned(a::GenericMemory, i::Int) @inline - @boundscheck (i - 1)%UInt < length(a)%UInt || return false + @boundscheck checkbounds(Bool, a, i) || return false return @inbounds memoryref_isassigned(memoryref(a, i), default_access_order(a), false) end @@ -223,10 +223,6 @@ Memory{T}(x::AbstractArray{S,1}) where {T,S} = copyto_axcheck!(Memory{T}(undef, ## copying iterators to containers -## Iteration ## - -iterate(A::Memory, i=1) = (@inline; (i - 1)%UInt < length(A)%UInt ? (@inbounds A[i], i + 1) : nothing) - ## Indexing: getindex ## # Faster contiguous indexing using copyto! for AbstractUnitRange and Colon diff --git a/base/gmp.jl b/base/gmp.jl index 97488551f60f6..e4d9294766aaa 100644 --- a/base/gmp.jl +++ b/base/gmp.jl @@ -10,8 +10,8 @@ import .Base: *, +, -, /, <, <<, >>, >>>, <=, ==, >, >=, ^, (~), (&), (|), xor, trailing_zeros, trailing_ones, count_ones, count_zeros, tryparse_internal, bin, oct, dec, hex, isequal, invmod, _prevpow2, _nextpow2, ndigits0zpb, widen, signed, unsafe_trunc, trunc, iszero, isone, big, flipsign, signbit, - sign, hastypemax, isodd, iseven, digits!, hash, hash_integer, top_set_bit, - clamp, unsafe_takestring + sign, isodd, iseven, digits!, hash, hash_integer, top_set_bit, + ispositive, isnegative, clamp, unsafe_takestring import Core: Signed, Float16, Float32, Float64 @@ -257,6 +257,7 @@ function export!(a::AbstractVector{T}, n::BigInt; order::Integer=-1, nails::Inte stride(a, 1) == 1 || throw(ArgumentError("a must have stride 1")) ndigits = cld(sizeinbase(n, 2), 8*sizeof(T) - nails) length(a) < ndigits && resize!(a, ndigits) + fill!(a, zero(T)) count = Ref{Csize_t}() ccall((:__gmpz_export, libgmp), Ptr{T}, (Ptr{T}, Ref{Csize_t}, Cint, Csize_t, Cint, Csize_t, mpz_t), a, count, order, sizeof(T), endian, nails, n) @@ -285,8 +286,6 @@ signed(x::BigInt) = x BigInt(x::BigInt) = x Signed(x::BigInt) = x -hastypemax(::Type{BigInt}) = false - function tryparse_internal(::Type{BigInt}, s::AbstractString, startpos::Int, endpos::Int, base_::Integer, raise::Bool) # don't make a copy in the common case where we are parsing a whole String bstr = startpos == firstindex(s) && endpos == lastindex(s) ? String(s) : String(SubString(s,startpos,endpos)) @@ -384,7 +383,7 @@ function (::Type{T})(x::BigInt) where T<:Base.BitSigned else 0 <= n <= cld(sizeof(T),sizeof(Limb)) || throw(InexactError(nameof(T), T, x)) y = x % T - ispos(x) ⊻ (y > 0) && throw(InexactError(nameof(T), T, x)) # catch overflow + ispositive(x) ⊻ (y > 0) && throw(InexactError(nameof(T), T, x)) # catch overflow y end end @@ -607,7 +606,7 @@ Number of ones in the binary representation of abs(x). count_ones_abs(x::BigInt) = iszero(x) ? 0 : MPZ.mpn_popcount(x) function top_set_bit(x::BigInt) - isneg(x) && throw(DomainError(x, "top_set_bit only supports negative arguments when they have type BitSigned.")) + isnegative(x) && throw(DomainError(x, "top_set_bit only supports negative arguments when they have type BitSigned.")) iszero(x) && return 0 x.size * sizeof(Limb) << 3 - leading_zeros(GC.@preserve x unsafe_load(x.d, x.size)) end @@ -701,7 +700,7 @@ function prod(arr::AbstractArray{BigInt}) foldl(MPZ.mul!, arr; init) end -factorial(n::BigInt) = !isneg(n) ? MPZ.fac_ui(n) : throw(DomainError(n, "`n` must not be negative.")) +factorial(n::BigInt) = !isnegative(n) ? MPZ.fac_ui(n) : throw(DomainError(n, "`n` must not be negative.")) function binomial(n::BigInt, k::Integer) k < 0 && return BigInt(0) @@ -733,17 +732,17 @@ isone(x::BigInt) = x == Culong(1) <(i::Integer, x::BigInt) = cmp(x,i) > 0 <(x::BigInt, f::CdoubleMax) = isnan(f) ? false : cmp(x,f) < 0 <(f::CdoubleMax, x::BigInt) = isnan(f) ? false : cmp(x,f) > 0 -isneg(x::BigInt) = x.size < 0 -ispos(x::BigInt) = x.size > 0 +isnegative(x::BigInt) = x.size < 0 +ispositive(x::BigInt) = x.size > 0 -signbit(x::BigInt) = isneg(x) +signbit(x::BigInt) = isnegative(x) flipsign!(x::BigInt, y::Integer) = (signbit(y) && (x.size = -x.size); x) flipsign( x::BigInt, y::Integer) = signbit(y) ? -x : x flipsign( x::BigInt, y::BigInt) = signbit(y) ? -x : x # above method to resolving ambiguities with flipsign(::T, ::T) where T<:Signed function sign(x::BigInt) - isneg(x) && return -one(x) - ispos(x) && return one(x) + isnegative(x) && return -one(x) + ispositive(x) && return one(x) return x end @@ -755,12 +754,12 @@ function string(n::BigInt; base::Integer = 10, pad::Integer = 1) iszero(n) && pad < 1 && return "" nd1 = ndigits(n, base=base) nd = max(nd1, pad) - sv = Base.StringMemory(nd + isneg(n)) + sv = Base.StringMemory(nd + isnegative(n)) GC.@preserve sv MPZ.get_str!(pointer(sv) + nd - nd1, base, n) - @inbounds for i = (1:nd-nd1) .+ isneg(n) + @inbounds for i = (1:nd-nd1) .+ isnegative(n) sv[i] = '0' % UInt8 end - isneg(n) && (sv[1] = '-' % UInt8) + isnegative(n) && (sv[1] = '-' % UInt8) unsafe_takestring(sv) end @@ -770,7 +769,7 @@ function digits!(a::AbstractVector{T}, n::BigInt; base::Integer = 10) where {T<: # fast path using mpz_get_str via string(n; base) s = codeunits(string(n; base)) i, j = firstindex(a)-1, length(s)+1 - lasti = min(lastindex(a), firstindex(a) + length(s)-1 - isneg(n)) + lasti = min(lastindex(a), firstindex(a) + length(s)-1 - isnegative(n)) while i < lasti # base ≤ 36: 0-9, plus a-z for 10-35 # base > 36: 0-9, plus A-Z for 10-35 and a-z for 36..61 @@ -779,14 +778,14 @@ function digits!(a::AbstractVector{T}, n::BigInt; base::Integer = 10) where {T<: end lasti = lastindex(a) while i < lasti; a[i+=1] = zero(T); end - return isneg(n) ? map!(-,a,a) : a + return isnegative(n) ? map!(-,a,a) : a elseif a isa StridedVector{<:Base.BitInteger} && stride(a,1) == 1 && ispow2(base) && base-1 ≤ typemax(T) # fast path using mpz_export origlen = length(a) _, writelen = MPZ.export!(a, n; nails = 8sizeof(T) - trailing_zeros(base)) length(a) != origlen && resize!(a, origlen) # truncate to least-significant digits a[begin+writelen:end] .= zero(T) - return isneg(n) ? map!(-,a,a) : a + return isnegative(n) ? map!(-,a,a) : a end end return invoke(digits!, Tuple{typeof(a), Integer}, a, n; base) # slow generic fallback @@ -844,24 +843,29 @@ Base.deepcopy_internal(x::BigInt, stackdict::IdDict) = get!(() -> MPZ.set(x), st ## streamlined hashing for BigInt, by avoiding allocation from shifts ## +Base._hash_shl!(x::BigInt, n) = MPZ.mul_2exp!(x, n) + if Limb === UInt64 === UInt # On 64 bit systems we can define # an optimized version for BigInt of hash_integer (used e.g. for Rational{BigInt}), # and of hash - using .Base: hash_uint + using .Base: HASH_SECRET, hash_bytes, hash_finalizer function hash_integer(n::BigInt, h::UInt) + iszero(n) && return hash_integer(0, h) GC.@preserve n begin s = n.size - s == 0 && return hash_integer(0, h) - p = convert(Ptr{UInt64}, n.d) - b = unsafe_load(p) - h ⊻= hash_uint(ifelse(s < 0, -b, b) ⊻ h) - for k = 2:abs(s) - h ⊻= hash_uint(unsafe_load(p, k) ⊻ h) - end - return h + h ⊻= (s < 0) + + us = abs(s) + leading_zero_bytes = div(leading_zeros(unsafe_load(n.d, us)), 8) + hash_bytes( + Ptr{UInt8}(n.d), + 8 * us - leading_zero_bytes, + h, + HASH_SECRET + ) end end @@ -893,23 +897,16 @@ if Limb === UInt64 === UInt return hash(ldexp(flipsign(Float64(limb), sz), pow), h) end h = hash_integer(pow, h) - h ⊻= hash_uint(flipsign(limb, sz) ⊻ h) - for idx = idx+1:asz - if shift == 0 - limb = unsafe_load(ptr, idx) - else - limb1 = limb2 - if idx == asz - limb = limb1 >> shift - limb == 0 && break # don't hash leading zeros - else - limb2 = unsafe_load(ptr, idx+1) - limb = limb2 << upshift | limb1 >> shift - end - end - h ⊻= hash_uint(limb ⊻ h) - end - return h + + h ⊻= (sz < 0) + leading_zero_bytes = div(leading_zeros(unsafe_load(x.d, asz)), 8) + trailing_zero_bytes = div(pow, 8) + return hash_bytes( + Ptr{UInt8}(x.d) + trailing_zero_bytes, + 8 * asz - (leading_zero_bytes + trailing_zero_bytes), + h, + HASH_SECRET + ) end end end @@ -918,7 +915,7 @@ module MPQ # Rational{BigInt} import .Base: unsafe_rational, __throw_rational_argerror_zero -import ..GMP: BigInt, MPZ, Limb, isneg, libgmp +import ..GMP: BigInt, MPZ, Limb, libgmp gmpq(op::Symbol) = (Symbol(:__gmpq_, op), libgmp) @@ -996,7 +993,7 @@ end # define add, sub, mul, div, and their inplace versions function add!(z::Rational{BigInt}, x::Rational{BigInt}, y::Rational{BigInt}) if iszero(x.den) || iszero(y.den) - if iszero(x.den) && iszero(y.den) && isneg(x.num) != isneg(y.num) + if iszero(x.den) && iszero(y.den) && isnegative(x.num) != isnegative(y.num) throw(DivideError()) end return set!(z, iszero(x.den) ? x : y) @@ -1009,7 +1006,7 @@ end function sub!(z::Rational{BigInt}, x::Rational{BigInt}, y::Rational{BigInt}) if iszero(x.den) || iszero(y.den) - if iszero(x.den) && iszero(y.den) && isneg(x.num) == isneg(y.num) + if iszero(x.den) && iszero(y.den) && isnegative(x.num) == isnegative(y.num) throw(DivideError()) end iszero(x.den) && return set!(z, x) @@ -1026,7 +1023,7 @@ function mul!(z::Rational{BigInt}, x::Rational{BigInt}, y::Rational{BigInt}) if iszero(x.num) || iszero(y.num) throw(DivideError()) end - return set_si!(z, ifelse(xor(isneg(x.num), isneg(y.num)), -1, 1), 0) + return set_si!(z, ifelse(xor(isnegative(x.num), isnegative(y.num)), -1, 1), 0) end zq = _MPQ(z) ccall((:__gmpq_mul, libgmp), Cvoid, @@ -1039,7 +1036,7 @@ function div!(z::Rational{BigInt}, x::Rational{BigInt}, y::Rational{BigInt}) if iszero(y.den) throw(DivideError()) end - isneg(y.num) || return set!(z, x) + isnegative(y.num) || return set!(z, x) return set_si!(z, flipsign(-1, x.num), 0) elseif iszero(y.den) return set_si!(z, 0, 1) diff --git a/base/hashing.jl b/base/hashing.jl index a2b00844ecaf5..897a0d73cb874 100644 --- a/base/hashing.jl +++ b/base/hashing.jl @@ -1,6 +1,11 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -## hashing a single value ## +const HASH_SEED = UInt == UInt64 ? 0xbdd89aa982704029 : 0xeabe9406 +const HASH_SECRET = tuple( + 0x2d358dccaa6c78a5, + 0x8bb84b93962eacc9, + 0x4b33a62ed433d4a3, +) """ hash(x[, h::UInt])::UInt @@ -17,90 +22,131 @@ The hash value may change when a new Julia process is started. ```jldoctest; filter = r"0x[0-9a-f]{16}" julia> a = hash(10) -0x95ea2955abd45275 +0x759d18cc5346a65f julia> hash(10, a) # only use the output of another hash function as the second argument -0xd42bad54a8575b16 +0x03158cd61b1b0bd1 ``` See also: [`objectid`](@ref), [`Dict`](@ref), [`Set`](@ref). """ -hash(x::Any) = hash(x, zero(UInt)) +hash(data::Any) = hash(data, HASH_SEED) hash(w::WeakRef, h::UInt) = hash(w.value, h) # Types can't be deleted, so marking as total allows the compiler to look up the hash -hash(T::Type, h::UInt) = hash_uint(3h - @assume_effects :total ccall(:jl_type_hash, UInt, (Any,), T)) +@noinline _jl_type_hash(T::Type) = @assume_effects :total ccall(:jl_type_hash, UInt, (Any,), T) +hash(T::Type, h::UInt) = hash(_jl_type_hash(T), h) +hash(@nospecialize(data), h::UInt) = hash(objectid(data), h) -## hashing general objects ## - -hash(@nospecialize(x), h::UInt) = hash_uint(3h - objectid(x)) - -hash(x::Symbol) = objectid(x) - -## core data hashing functions ## - -function hash_64_64(n::UInt64) - a::UInt64 = n - a = ~a + a << 21 - a = a ⊻ a >> 24 - a = a + a << 3 + a << 8 - a = a ⊻ a >> 14 - a = a + a << 2 + a << 4 - a = a ⊻ a >> 28 - a = a + a << 31 - return a +function mul_parts(a::UInt64, b::UInt64) + p = widemul(a, b) + return (p >> 64) % UInt64, p % UInt64 end - -function hash_64_32(n::UInt64) - a::UInt64 = n - a = ~a + a << 18 - a = a ⊻ a >> 31 - a = a * 21 - a = a ⊻ a >> 11 - a = a + a << 6 - a = a ⊻ a >> 22 - return a % UInt32 +hash_mix(a::UInt64, b::UInt64) = ⊻(mul_parts(a, b)...) + +# faster-but-weaker than hash_mix intended for small keys +hash_mix_linear(x::Union{UInt64, UInt32}, h::UInt) = 3h - x +function hash_finalizer(x::UInt64) + x ⊻= (x >> 32) + x *= 0x63652a4cd374b267 + x ⊻= (x >> 33) + return x end -function hash_32_32(n::UInt32) - a::UInt32 = n - a = a + 0x7ed55d16 + a << 12 - a = a ⊻ 0xc761c23c ⊻ a >> 19 - a = a + 0x165667b1 + a << 5 - a = a + 0xd3a2646c ⊻ a << 9 - a = a + 0xfd7046c5 + a << 3 - a = a ⊻ 0xb55a4f09 ⊻ a >> 16 - return a -end +hash_64_64(data::UInt64) = hash_finalizer(data) +hash_64_32(data::UInt64) = hash_64_64(data) % UInt32 +hash_32_32(data::UInt32) = hash_64_32(UInt64(data)) if UInt === UInt64 - hash_uint64(x::UInt64) = hash_64_64(x) - hash_uint(x::UInt) = hash_64_64(x) + const hash_uint64 = hash_64_64 + const hash_uint = hash_64_64 else - hash_uint64(x::UInt64) = hash_64_32(x) - hash_uint(x::UInt) = hash_32_32(x) + const hash_uint64 = hash_64_32 + const hash_uint = hash_32_32 end -## efficient value-based hashing of integers ## - -hash(x::Int64, h::UInt) = hash_uint64(bitcast(UInt64, x)) - 3h -hash(x::UInt64, h::UInt) = hash_uint64(x) - 3h -hash(x::Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32}, h::UInt) = hash(Int64(x), h) - -function hash_integer(n::Integer, h::UInt) - h ⊻= hash_uint((n % UInt) ⊻ h) - n = abs(n) - n >>>= sizeof(UInt) << 3 - while n != 0 - h ⊻= hash_uint((n % UInt) ⊻ h) - n >>>= sizeof(UInt) << 3 +hash(x::UInt64, h::UInt) = hash_uint64(hash_mix_linear(x, h)) +hash(x::Int64, h::UInt) = hash(bitcast(UInt64, x), h) +hash(x::Union{Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32}, h::UInt) = hash(Int64(x), h) + +hash_integer(x::Integer, h::UInt) = _hash_integer(x, UInt64(h)) % UInt +function _hash_integer( + x::Integer, + seed::UInt64 = HASH_SEED, + secret::NTuple{3, UInt64} = HASH_SECRET + ) + seed ⊻= (x < 0) + u = abs(x) + + # always left-pad to full byte + buflen = UInt(max(cld(top_set_bit(u), 8), 1)) + seed = seed ⊻ (hash_mix(seed ⊻ secret[1], secret[2]) ⊻ buflen) + + a = zero(UInt64) + b = zero(UInt64) + + if buflen ≤ 16 + if buflen ≥ 4 + a = (UInt64(u % UInt32) << 32) | + UInt64((u >>> ((buflen - 4) * 8)) % UInt32) + + delta = (buflen & 24) >>> (buflen >>> 3) + + b = (UInt64((u >>> (8 * delta)) % UInt32) << 32) | + UInt64((u >>> (8 * (buflen - 4 - delta))) % UInt32) + else # buflen > 0 + b0 = u % UInt8 + b1 = (u >>> (8 * div(buflen, 2))) % UInt8 + b2 = (u >>> (8 * (buflen - 1))) % UInt8 + a = (UInt64(b0) << 56) | + (UInt64(b1) << 32) | + UInt64(b2) + end + else + a = (u >>> 8(buflen - 16)) % UInt + b = (u >>> 8(buflen - 8)) % UInt + + i = buflen + if i > 48 + see1 = seed + see2 = seed + while i ≥ 48 + l0 = u % UInt; u >>>= 64 + l1 = u % UInt; u >>>= 64 + l2 = u % UInt; u >>>= 64 + l3 = u % UInt; u >>>= 64 + l4 = u % UInt; u >>>= 64 + l5 = u % UInt; u >>>= 64 + + seed = hash_mix(l0 ⊻ secret[1], l1 ⊻ seed) + see1 = hash_mix(l2 ⊻ secret[2], l3 ⊻ see1) + see2 = hash_mix(l4 ⊻ secret[3], l5 ⊻ see2) + i -= 48 + end + seed = seed ⊻ see1 ⊻ see2 + end + if i > 16 + l0 = u % UInt; u >>>= 64 + l1 = u % UInt; u >>>= 64 + seed = hash_mix(l0 ⊻ secret[3], l1 ⊻ seed ⊻ secret[2]) + if i > 32 + l2 = u % UInt; u >>>= 64 + l3 = u % UInt; u >>>= 64 + seed = hash_mix(l2 ⊻ secret[3], l3 ⊻ seed) + end + end end - return h + + a = a ⊻ secret[2] + b = b ⊻ seed + b, a = mul_parts(a, b) + return hash_mix(a ⊻ secret[1] ⊻ buflen, b ⊻ secret[2]) end + ## efficient value-based hashing of floats ## -const hx_NaN = hash_uint64(reinterpret(UInt64, NaN)) +const hx_NaN = hash(reinterpret(UInt64, NaN)) function hash(x::Float64, h::UInt) # see comments on trunc and hash(Real, UInt) if typemin(Int64) <= x < typemax(Int64) @@ -116,7 +162,7 @@ function hash(x::Float64, h::UInt) elseif isnan(x) return hx_NaN ⊻ h # NaN does not have a stable bit pattern end - return hash_uint64(bitcast(UInt64, x)) - 3h + return hash(bitcast(UInt64, x), h) end hash(x::Float32, h::UInt) = hash(Float64(x), h) @@ -131,10 +177,11 @@ function hash(x::Float16, h::UInt) elseif isnan(x) return hx_NaN ⊻ h # NaN does not have a stable bit pattern end - return hash_uint64(bitcast(UInt64, Float64(x))) - 3h + return hash(bitcast(UInt64, Float64(x)), h) end ## generic hashing for rational values ## +_hash_shl!(x, n) = (x << n) function hash(x::Real, h::UInt) # decompose x as num*2^pow/den num, pow, den = decompose(x) @@ -150,6 +197,7 @@ function hash(x::Real, h::UInt) den = -den end num_z = trailing_zeros(num) + num >>= num_z den_z = trailing_zeros(den) den >>= den_z @@ -174,27 +222,130 @@ function hash(x::Real, h::UInt) end # handle generic rational values h = hash_integer(pow, h) - h = hash_integer(num, h) + + # trimming only whole bytes of trailing zeros simplifies greatly + # some specializations for memory-backed bitintegers + h = hash_integer((pow > 0) ? _hash_shl!(num, pow % 8) : num, h) return h end ## symbol & expression hashing ## - if UInt === UInt64 - hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h + 0x83c7900696d26dc6)) - hash(x::QuoteNode, h::UInt) = hash(x.value, h + 0x2c97bf8b3de87020) + # conservatively hash using == equality of all of the data, even though == often uses === internally + hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h ⊻ 0x83c7900696d26dc6)) + hash(x::QuoteNode, h::UInt) = hash(x.value, h ⊻ 0x2c97bf8b3de87020) + hash(x::PhiNode, h::UInt) = hash(x.edges, hash(x.values, h ⊻ 0x2c97bf8b3de87020)) + hash(x::PhiCNode, h::UInt) = hash(x.values, h ⊻ 0x2c97bf8b3de87020) else - hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h + 0x96d26dc6)) - hash(x::QuoteNode, h::UInt) = hash(x.value, h + 0x469d72af) + hash(x::Expr, h::UInt) = hash(x.args, hash(x.head, h ⊻ 0x469d72af)) + hash(x::QuoteNode, h::UInt) = hash(x.value, h ⊻ 0x469d72af) + hash(x::PhiNode, h::UInt) = hash(x.edges, hash(x.values, h ⊻ 0x469d72af)) + hash(x::PhiCNode, h::UInt) = hash(x.values, h ⊻ 0x469d72af) end -## hashing strings ## +function hash(x::CodeInfo, h::UInt) + h ⊻= UInt === UInt64 ? 0x2c97bf8b3de87020 : 0x469d72af + for i in 1:nfields(x) + h = hash(isdefined(x, i) ? getfield(x, i) : missing, h) + end + return h +end -const memhash = UInt === UInt64 ? :memhash_seed : :memhash32_seed -const memhash_seed = UInt === UInt64 ? 0x71e729fd56419c81 : 0x56419c81 +function hash(x::DebugInfo, h::UInt) + h ⊻= UInt === UInt64 ? 0x2c97bf8b3de87020 : 0x469d72af + for i in 1:nfields(x) + h = hash(getfield(x, i), h) + end + return h +end + +hash(x::Symbol) = objectid(x) -@assume_effects :total function hash(s::String, h::UInt) - h += memhash_seed - ccall(memhash, UInt, (Ptr{UInt8}, Csize_t, UInt32), s, sizeof(s), h % UInt32) + h + +load_le(::Type{T}, ptr::Ptr{UInt8}, i) where {T <: Union{UInt32, UInt64}} = + unsafe_load(convert(Ptr{T}, ptr + i - 1)) + +function read_small(ptr::Ptr{UInt8}, n::Int) + return (UInt64(unsafe_load(ptr)) << 56) | + (UInt64(unsafe_load(ptr, div(n, 2) + 1)) << 32) | + UInt64(unsafe_load(ptr, n)) end + +@assume_effects :terminates_globally function hash_bytes( + ptr::Ptr{UInt8}, + n::Int, + seed::UInt64, + secret::NTuple{3, UInt64} + ) + # Adapted with gratitude from [rapidhash](https://github.com/Nicoshev/rapidhash) + buflen = UInt64(n) + seed = seed ⊻ (hash_mix(seed ⊻ secret[1], secret[2]) ⊻ buflen) + + a = zero(UInt64) + b = zero(UInt64) + + if buflen ≤ 16 + if buflen ≥ 4 + a = (UInt64(load_le(UInt32, ptr, 1)) << 32) | + UInt64(load_le(UInt32, ptr, n - 3)) + + delta = (buflen & 24) >>> (buflen >>> 3) + b = (UInt64(load_le(UInt32, ptr, delta + 1)) << 32) | + UInt64(load_le(UInt32, ptr, n - 3 - delta)) + elseif buflen > 0 + a = read_small(ptr, n) + end + else + pos = 1 + i = buflen + if i > 48 + see1 = seed + see2 = seed + while i ≥ 48 + seed = hash_mix( + load_le(UInt64, ptr, pos) ⊻ secret[1], + load_le(UInt64, ptr, pos + 8) ⊻ seed + ) + see1 = hash_mix( + load_le(UInt64, ptr, pos + 16) ⊻ secret[2], + load_le(UInt64, ptr, pos + 24) ⊻ see1 + ) + see2 = hash_mix( + load_le(UInt64, ptr, pos + 32) ⊻ secret[3], + load_le(UInt64, ptr, pos + 40) ⊻ see2 + ) + pos += 48 + i -= 48 + end + seed = seed ⊻ see1 ⊻ see2 + end + if i > 16 + seed = hash_mix( + load_le(UInt64, ptr, pos) ⊻ secret[3], + load_le(UInt64, ptr, pos + 8) ⊻ seed ⊻ secret[2] + ) + if i > 32 + seed = hash_mix( + load_le(UInt64, ptr, pos + 16) ⊻ secret[3], + load_le(UInt64, ptr, pos + 24) ⊻ seed + ) + end + end + + a = load_le(UInt64, ptr, n - 15) + b = load_le(UInt64, ptr, n - 7) + end + + a = a ⊻ secret[2] + b = b ⊻ seed + b, a = mul_parts(a, b) + return hash_mix(a ⊻ secret[1] ⊻ buflen, b ⊻ secret[2]) +end + +@assume_effects :total hash(data::String, h::UInt) = + GC.@preserve data hash_bytes(pointer(data), sizeof(data), UInt64(h), HASH_SECRET) % UInt + +# no longer used in Base, but a lot of packages access these internals +const memhash = UInt === UInt64 ? :memhash_seed : :memhash32_seed +const memhash_seed = UInt === UInt64 ? 0x71e729fd56419c81 : 0x56419c81 diff --git a/base/iddict.jl b/base/iddict.jl index f1632e93427a8..ec5392cf7b5b8 100644 --- a/base/iddict.jl +++ b/base/iddict.jl @@ -12,7 +12,7 @@ the same, so they get overwritten. The `IdDict` hashes by object-id, and thus preserves the 3 different keys. # Examples -```julia-repl +```jldoctest; filter = r" \\S+ +=> \\S+" => " KEY => VALUE" julia> Dict(true => "yes", 1 => "no", 1.0 => "maybe") Dict{Real, String} with 1 entry: 1.0 => "maybe" @@ -58,12 +58,12 @@ IdDict(kv) = dict_with_eltype((K, V) -> IdDict{K, V}, kv, eltype(kv)) empty(d::IdDict, ::Type{K}, ::Type{V}) where {K, V} = IdDict{K,V}() -function rehash!(d::IdDict, newsz = length(d.ht)%UInt) +function rehash!(d::IdDict, newsz::Integer = length(d.ht)%UInt) d.ht = ccall(:jl_idtable_rehash, Memory{Any}, (Any, Csize_t), d.ht, newsz) d end -function sizehint!(d::IdDict, newsz) +function sizehint!(d::IdDict, newsz::Integer) newsz = _tablesz(newsz*2) # *2 for keys and values in same array oldsz = length(d.ht) # grow at least 25% diff --git a/base/idset.jl b/base/idset.jl index c2f26b528020d..4ad9426172909 100644 --- a/base/idset.jl +++ b/base/idset.jl @@ -78,7 +78,7 @@ pop!(s::IdSet, @nospecialize(x)) = _pop!(s, x) == -1 ? throw(KeyError(x)) : x pop!(s::IdSet, @nospecialize(x), @nospecialize(default)) = _pop!(s, x) == -1 ? default : x delete!(s::IdSet, @nospecialize(x)) = (_pop!(s, x); s) -function sizehint!(s::IdSet, newsz) +function sizehint!(s::IdSet, newsz::Integer) # TODO: grow/compact list and perform rehash, if profitable? # TODO: shrink? # s.list = resize(s.list, newsz) @@ -92,8 +92,17 @@ function sizehint!(s::IdSet, newsz) nothing end +function _zero!(a::Memory{<:BitInteger}) + t = @_gc_preserve_begin a + p = unsafe_convert(Ptr{Cvoid}, a) + T = eltype(a) + memset(p, 0x0, (sizeof(T) * length(a)) % UInt) + @_gc_preserve_end t + return a +end + function empty!(s::IdSet) - fill!(s.idxs, 0x00) + _zero!(s.idxs) list = s.list for i = 1:s.max _unsetindex!(list, i) diff --git a/base/indices.jl b/base/indices.jl index 45f3495e51191..88e48a2b331ee 100644 --- a/base/indices.jl +++ b/base/indices.jl @@ -132,7 +132,7 @@ function throw_promote_shape_mismatch(a::Tuple, b::Union{Nothing,Tuple}, i = not if i ≢ nothing print(msg, ", mismatch at dim ", i) end - throw(DimensionMismatch(String(take!(msg)))) + throw(DimensionMismatch(takestring!(msg))) end function promote_shape(a::Tuple{Int,}, b::Tuple{Int,}) diff --git a/base/initdefs.jl b/base/initdefs.jl index ef1b40796f1ad..cc829f1823dd4 100644 --- a/base/initdefs.jl +++ b/base/initdefs.jl @@ -284,22 +284,14 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing} env == "@temp" && return mktempdir() env == "@stdlib" && return Sys.STDLIB if startswith(env, "@script") - if @isdefined(PROGRAM_FILE) - dir = dirname(PROGRAM_FILE) - else - cmds = unsafe_load_commands(JLOptions().commands) - if any(cmd::Pair{Char, String}->cmd_suppresses_program(first(cmd)), cmds) - # Usage error. The user did not pass a script. - return nothing - end - dir = dirname(ARGS[1]) - end - if env == "@script" # complete match, not startswith, so search upwards - return current_project(dir) - else - # starts with, so assume relative path is after - return abspath(replace(env, "@script" => dir)) - end + program_file = JLOptions().program_file + program_file = program_file != C_NULL ? unsafe_string(program_file) : nothing + isnothing(program_file) && return nothing # User did not pass a script + + # Expand trailing relative path + dir = dirname(program_file) + dir = env != "@script" ? (dir * env[length("@script")+1:end]) : dir + return current_project(dir) end env = replace(env, '#' => VERSION.major, count=1) env = replace(env, '#' => VERSION.minor, count=1) diff --git a/base/int.jl b/base/int.jl index 6d6251ac1d64b..57ebeda4812ad 100644 --- a/base/int.jl +++ b/base/int.jl @@ -139,6 +139,8 @@ iseven(n::Real) = isinteger(n) && iszero(rem(Integer(n), 2)) signbit(x::Integer) = x < 0 signbit(x::Unsigned) = false +isnegative(x::Unsigned) = false + flipsign(x::T, y::T) where {T<:BitSigned} = flipsign_int(x, y) flipsign(x::BitSigned, y::BitSigned) = flipsign_int(promote(x, y)...) % typeof(x) @@ -250,7 +252,7 @@ end The reduction of `x` modulo `y`, or equivalently, the remainder of `x` after floored division by `y`, i.e. `x - y*fld(x,y)` if computed without intermediate rounding. -The result will have the same sign as `y`, and magnitude less than `abs(y)` (with some +The result will have the same sign as `y` if `isfinite(y)`, and magnitude less than `abs(y)` (with some exceptions, see note below). !!! note @@ -720,7 +722,7 @@ macro big_str(s::String) is_prev_dot = (c == '.') end print(bf, s[end]) - s = String(take!(bf)) + s = unsafe_takestring!(bf) end n = tryparse(BigInt, s) n === nothing || return n @@ -842,7 +844,7 @@ widen(::Type{UInt64}) = UInt128 # |x|<=2^(k-1), |y|<=2^k-1 => |x*y|<=2^(2k-1)-1 widemul(x::Signed,y::Unsigned) = widen(x) * signed(widen(y)) widemul(x::Unsigned,y::Signed) = signed(widen(x)) * widen(y) -# multplication by Bool doesn't require widening +# multiplication by Bool doesn't require widening widemul(x::Bool,y::Bool) = x * y widemul(x::Bool,y::Number) = x * y widemul(x::Number,y::Bool) = x * y diff --git a/base/intfuncs.jl b/base/intfuncs.jl index 9733790b03b04..925dafdd8f378 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -286,7 +286,7 @@ function invmod(n::Integer, m::Integer) g, x, y = gcdx(n, m) g != 1 && throw(DomainError((n, m), LazyString("Greatest common divisor is ", g, "."))) # Note that m might be negative here. - if n isa Unsigned && hastypemax(typeof(n)) && x > typemax(n)>>1 + if x isa Unsigned && hastypemax(typeof(x)) && x > typemax(x)>>1 # x might have wrapped if it would have been negative # adding back m forces a correction x += m @@ -335,11 +335,6 @@ function invmod(n::T) where {T<:BitInteger} end # ^ for any x supporting * -function to_power_type(x::Number) - T = promote_type(typeof(x), typeof(x*x)) - convert(T, x) -end -to_power_type(x) = oftype(x*x, x) @noinline throw_domerr_powbysq(::Any, p) = throw(DomainError(p, LazyString( "Cannot raise an integer x to a negative power ", p, ".", "\nConvert input to float."))) @@ -355,12 +350,23 @@ to_power_type(x) = oftype(x*x, x) "or write float(x)^", p, " or Rational.(x)^", p, "."))) # The * keyword supports `*=checked_mul` for `checked_pow` @assume_effects :terminates_locally function power_by_squaring(x_, p::Integer; mul=*) - x = to_power_type(x_) + x_squared_ = x_ * x_ + x_squared_type = typeof(x_squared_) + T = if x_ isa Number + promote_type(typeof(x_), x_squared_type) + else + x_squared_type + end + x = convert(T, x_) + square_is_useful = mul === * if p == 1 return copy(x) elseif p == 0 return one(x) elseif p == 2 + if square_is_useful # avoid performing the same multiplication a second time when possible + return convert(T, x_squared_) + end return mul(x, x) elseif p < 0 isone(x) && return copy(x) @@ -369,6 +375,11 @@ to_power_type(x) = oftype(x*x, x) end t = trailing_zeros(p) + 1 p >>= t + if square_is_useful # avoid performing the same multiplication a second time when possible + if (t -= 1) > 0 + x = convert(T, x_squared_) + end + end while (t -= 1) > 0 x = mul(x, x) end @@ -570,7 +581,8 @@ function nextpow(a::Real, x::Real) n = ceil(Integer,log(a, x)) # round-off error of log can go either direction, so need some checks p = a^(n-1) - x > typemax(p) && throw(DomainError(x,"argument is beyond the range of type of the base")) + hastypemax(typeof(p)) && x > typemax(p) && + throw(DomainError(x,"argument is beyond the range of type of the base")) p >= x && return p wp = a^n wp > p || throw(OverflowError("result is beyond the range of type of the base")) @@ -611,9 +623,10 @@ function prevpow(a::T, x::Real) where T <: Real n = floor(Integer,log(a, x)) # round-off error of log can go either direction, so need some checks p = a^n - x > typemax(p) && throw(DomainError(x,"argument is beyond the range of type of the base")) + hastypemax(typeof(p)) && x > typemax(p) && + throw(DomainError(x,"argument is beyond the range of type of the base")) if a isa Integer - wp, overflow = mul_with_overflow(a, p) + wp, overflow = mul_with_overflow(promote(a, p)...) wp <= x && !overflow && return wp else wp = a^(n+1) @@ -845,7 +858,7 @@ function append_c_digits(olength::Int, digits::Unsigned, buf, pos::Int) while i >= 2 d, c = divrem(digits, 0x64) digits = oftype(digits, d) - @inbounds d100 = _dec_d100[(c % Int) + 1] + @inbounds d100 = _dec_d100[(c % Int)::Int + 1] @inbounds buf[pos + i - 2] = d100 % UInt8 @inbounds buf[pos + i - 1] = (d100 >> 0x8) % UInt8 i -= 2 @@ -1066,8 +1079,6 @@ end Return `true` if and only if the extrema `typemax(T)` and `typemin(T)` are defined. """ -hastypemax(::Base.BitIntegerType) = true -hastypemax(::Type{Bool}) = true hastypemax(::Type{T}) where {T} = applicable(typemax, T) && applicable(typemin, T) """ diff --git a/base/invalidation.jl b/base/invalidation.jl index 14b88e71b9def..0a44449748c2f 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -15,36 +15,6 @@ function iterate(gri::GlobalRefIterator, i = 1) return ((b::Core.Binding).globalref, i+1) end -const TYPE_TYPE_MT = Type.body.name.mt -const NONFUNCTION_MT = Core.MethodTable.name.mt -function foreach_module_mtable(visit, m::Module, world::UInt) - for gb in globalrefs(m) - binding = gb.binding - bpart = lookup_binding_partition(world, binding) - if is_defined_const_binding(binding_kind(bpart)) - v = partition_restriction(bpart) - uw = unwrap_unionall(v) - name = gb.name - if isa(uw, DataType) - tn = uw.name - if tn.module === m && tn.name === name && tn.wrapper === v && isdefined(tn, :mt) - # this is the original/primary binding for the type (name/wrapper) - mt = tn.mt - if mt !== nothing && mt !== TYPE_TYPE_MT && mt !== NONFUNCTION_MT - @assert mt.module === m - visit(mt) || return false - end - end - elseif isa(v, Core.MethodTable) && v.module === m && v.name === name - # this is probably an external method table here, so let's - # assume so as there is no way to precisely distinguish them - visit(v) || return false - end - end - end - return true -end - function foreachgr(visit, src::CodeInfo) stmts = src.code for i = 1:length(stmts) @@ -97,20 +67,28 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid binding = convert(Core.Binding, gr) if isdefined(method, :source) src = _uncompressed_ir(method) - old_stmts = src.code invalidate_all = should_invalidate_code_for_globalref(gr, src) end + if invalidate_all && !Base.generating_output() + @atomic method.did_scan_source |= 0x4 + end + invalidated_any = false for mi in specializations(method) isdefined(mi, :cache) || continue ci = mi.cache + invalidated = false while true if ci.max_world > new_max_world && (invalidate_all || scan_edge_list(ci, binding)) ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), ci, new_max_world) + invalidated = true end isdefined(ci, :next) || break ci = ci.next end + invalidated && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), mi) + invalidated_any |= invalidated end + return invalidated_any end export_affecting_partition_flags(bpart::Core.BindingPartition) = @@ -124,40 +102,43 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core (_, (ib, ibpart)) = Compiler.walk_binding_partition(b, invalidated_bpart, new_max_world) (_, (nb, nbpart)) = Compiler.walk_binding_partition(b, new_bpart, new_max_world+1) - # abstract_eval_globalref_partition is the maximum amount of information that inference + # `abstract_eval_partition_load` is the maximum amount of information that inference # reads from a binding partition. If this information does not change - we do not need to # invalidate any code that inference created, because we know that the result will not change. need_to_invalidate_code = - Compiler.abstract_eval_globalref_partition(nothing, ib, ibpart) !== - Compiler.abstract_eval_globalref_partition(nothing, nb, nbpart) + Compiler.abstract_eval_partition_load(nothing, ib, ibpart) !== + Compiler.abstract_eval_partition_load(nothing, nb, nbpart) need_to_invalidate_export = export_affecting_partition_flags(invalidated_bpart) !== export_affecting_partition_flags(new_bpart) + invalidated_any = false + queued_bindings = Tuple{Core.Binding, Core.BindingPartition, Core.BindingPartition}[] # defer handling these to keep the logging coherent if need_to_invalidate_code if (b.flags & BINDING_FLAG_ANY_IMPLICIT_EDGES) != 0 nmethods = ccall(:jl_module_scanned_methods_length, Csize_t, (Any,), gr.mod) for i = 1:nmethods method = ccall(:jl_module_scanned_methods_getindex, Any, (Any, Csize_t), gr.mod, i)::Method - invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) + invalidated_any |= invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) end end - if isdefined(b, :backedges) - for edge in b.backedges - if isa(edge, CodeInstance) - ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world) - elseif isa(edge, Core.Binding) - isdefined(edge, :partitions) || continue - latest_bpart = edge.partitions - latest_bpart.max_world == typemax(UInt) || continue - is_some_imported(binding_kind(latest_bpart)) || continue - if is_some_binding_imported(binding_kind(latest_bpart)) - partition_restriction(latest_bpart) === b || continue - end - invalidate_code_for_globalref!(edge, latest_bpart, latest_bpart, new_max_world) - else - invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) + nbackedges = ccall(:jl_binding_backedges_length, Csize_t, (Any,), b) + for i = 1:nbackedges + edge = ccall(:jl_binding_backedges_getindex, Any, (Any, Csize_t), b, i) + if isa(edge, CodeInstance) + ccall(:jl_invalidate_code_instance, Cvoid, (Any, UInt), edge, new_max_world) + invalidated_any = true + elseif isa(edge, Core.Binding) + isdefined(edge, :partitions) || continue + latest_bpart = edge.partitions + latest_bpart.max_world == typemax(UInt) || continue + is_some_imported(binding_kind(latest_bpart)) || continue + if is_some_binding_imported(binding_kind(latest_bpart)) + partition_restriction(latest_bpart) === b || continue end + push!(queued_bindings, (edge, latest_bpart, latest_bpart)) + else + invalidated_any |= invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) end end end @@ -168,7 +149,7 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core usings_backedges = ccall(:jl_get_module_usings_backedges, Any, (Any,), gr.mod) if usings_backedges !== nothing for user::Module in usings_backedges::Vector{Any} - user_binding = ccall(:jl_get_module_binding_or_nothing, Any, (Any, Any), user, gr.name) + user_binding = ccall(:jl_get_module_binding_or_nothing, Any, (Any, Any), user, gr.name)::Union{Core.Binding, Nothing} user_binding === nothing && continue isdefined(user_binding, :partitions) || continue latest_bpart = user_binding.partitions @@ -178,17 +159,22 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world) : latest_bpart if need_to_invalidate_code || new_bpart !== latest_bpart - invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, new_bpart, new_max_world) + push!(queued_bindings, (convert(Core.Binding, user_binding), latest_bpart, new_bpart)) end end end end + invalidated_any && ccall(:jl_maybe_log_binding_invalidation, Cvoid, (Any,), invalidated_bpart) + for (edge, invalidated_bpart, new_bpart) in queued_bindings + invalidated_any |= invalidate_code_for_globalref!(edge, invalidated_bpart, new_bpart, new_max_world) + end + return invalidated_any end invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) = invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world) function maybe_add_binding_backedge!(b::Core.Binding, edge::Union{Method, CodeInstance}) - meth = isa(edge, Method) ? edge : Compiler.get_ci_mi(edge).def + meth = isa(edge, Method) ? edge : get_ci_mi(edge).def ccall(:jl_maybe_add_binding_backedge, Cint, (Any, Any, Any), b, edge, meth) return nothing end @@ -199,7 +185,7 @@ function binding_was_invalidated(b::Core.Binding) b.partitions.min_world > unsafe_load(cglobal(:jl_require_world, UInt)) end -function scan_new_method!(methods_with_invalidated_source::IdSet{Method}, method::Method, image_backedges_only::Bool) +function scan_new_method!(method::Method, image_backedges_only::Bool) isdefined(method, :source) || return if image_backedges_only && !has_image_globalref(method) return @@ -210,23 +196,26 @@ function scan_new_method!(methods_with_invalidated_source::IdSet{Method}, method b = convert(Core.Binding, gr) if binding_was_invalidated(b) # TODO: We could turn this into an addition if condition. For now, use it as a reasonably cheap - # additional consistency chekc + # additional consistency check @assert !image_backedges_only - push!(methods_with_invalidated_source, method) + @atomic method.did_scan_source |= 0x4 end maybe_add_binding_backedge!(b, method) end + @atomic method.did_scan_source |= 0x1 end -function scan_new_methods(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool) - methods_with_invalidated_source = IdSet{Method}() +function scan_new_methods!(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool) + if image_backedges_only && Base.generating_output(true) + # Replacing image bindings is forbidden during incremental precompilation - skip backedge insertion + return + end for method in internal_methods if isa(method, Method) - scan_new_method!(methods_with_invalidated_source, method, image_backedges_only) + scan_new_method!(method, image_backedges_only) end end for tme::Core.TypeMapEntry in extext_methods - scan_new_method!(methods_with_invalidated_source, tme.func::Method, image_backedges_only) + scan_new_method!(tme.func::Method, image_backedges_only) end - return methods_with_invalidated_source end diff --git a/base/io.jl b/base/io.jl index 2fb8ce0c95f06..71e4c404dfeac 100644 --- a/base/io.jl +++ b/base/io.jl @@ -41,11 +41,9 @@ buffer_writes(x::IO, bufsize=SZ_UNBUFFERED_IO) = x """ isopen(object)::Bool -Determine whether an object - such as a stream or timer --- is not yet closed. Once an object is closed, it will never produce a new event. -However, since a closed stream may still have data to read in its buffer, -use [`eof`](@ref) to check for the ability to read data. -Use the `FileWatching` package to be notified when a stream might be writable or readable. +Determine whether an object, such as an IO or timer, is still open and hence active. + +See also: [`close`](@ref) # Examples ```jldoctest @@ -63,9 +61,19 @@ false function isopen end """ - close(stream) + close(io::IO) + +Close `io`. Performs a [`flush`](@ref) first. + +Closing an IO signals that its underlying resources (OS handle, network +connections, etc) should be destroyed. +A closed IO is in an undefined state and should not be written to or read from. +When attempting to do so, the IO may throw an exception, continue to behave +normally, or read/write zero bytes, depending on the implementation. +However, implementations should make sure that reading to or writing from a +closed IO does not cause undefined behaviour. -Close an I/O stream. Performs a [`flush`](@ref) first. +See also: [`isopen`](@ref) """ function close end @@ -97,9 +105,11 @@ julia> read(io, String) function closewrite end """ - flush(stream) + flush(io::IO) -Commit all currently buffered writes to the given stream. +Commit all currently buffered writes to the given io. +This has a default implementation `flush(::IO) = nothing`, so may be called +in generic IO code. """ function flush end @@ -277,13 +287,13 @@ julia> io = IOBuffer(); julia> write(io, "JuliaLang is a GitHub organization.", " It has many members.") 56 -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a GitHub organization. It has many members." julia> write(io, "Sometimes those members") + write(io, " write documentation.") 44 -julia> String(take!(io)) +julia> takestring!(io) "Sometimes those members write documentation." ``` User-defined plain-data types without `write` methods can be written when wrapped in a `Ref`: @@ -544,7 +554,7 @@ julia> rm("my_file.txt") """ readuntil(filename::AbstractString, delim; kw...) = open(io->readuntil(io, delim; kw...), convert(String, filename)::String) readuntil(stream::IO, delim::UInt8; kw...) = _unsafe_take!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...)) -readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; kw...) = String(_unsafe_take!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...))) +readuntil(stream::IO, delim::Union{AbstractChar, AbstractString}; kw...) = takestring!(copyuntil(IOBuffer(sizehint=16), stream, delim; kw...)) readuntil(stream::IO, delim::T; keep::Bool=false) where T = _copyuntil(Vector{T}(), stream, delim, keep) @@ -566,10 +576,10 @@ Similar to [`readuntil`](@ref), which returns a `String`; in contrast, ```jldoctest julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); -julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", 'L'))) +julia> takestring!(copyuntil(IOBuffer(), "my_file.txt", 'L')) "Julia" -julia> String(take!(copyuntil(IOBuffer(), "my_file.txt", '.', keep = true))) +julia> takestring!(copyuntil(IOBuffer(), "my_file.txt", '.', keep = true)) "JuliaLang is a GitHub organization." julia> rm("my_file.txt") @@ -616,8 +626,7 @@ Logan """ readline(filename::AbstractString; keep::Bool=false) = open(io -> readline(io; keep), filename) -readline(s::IO=stdin; keep::Bool=false) = - String(_unsafe_take!(copyline(IOBuffer(sizehint=16), s; keep))) +readline(s::IO=stdin; keep::Bool=false) = takestring!(copyline(IOBuffer(sizehint=16), s; keep)) """ copyline(out::IO, io::IO=stdin; keep::Bool=false) @@ -642,10 +651,10 @@ See also [`copyuntil`](@ref) for reading until more general delimiters. ```jldoctest julia> write("my_file.txt", "JuliaLang is a GitHub organization.\\nIt has many members.\\n"); -julia> String(take!(copyline(IOBuffer(), "my_file.txt"))) +julia> takestring!(copyline(IOBuffer(), "my_file.txt")) "JuliaLang is a GitHub organization." -julia> String(take!(copyline(IOBuffer(), "my_file.txt", keep=true))) +julia> takestring!(copyline(IOBuffer(), "my_file.txt", keep=true)) "JuliaLang is a GitHub organization.\\n" julia> rm("my_file.txt") @@ -1290,7 +1299,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state) buf.size = _stripnewline(r.itr.keep, buf.size, buf.data) empty!(chunks) # will cause next iteration to terminate seekend(r.itr.stream) # reposition to end of stream for isdone - s = String(_unsafe_take!(buf)) + s = unsafe_takestring!(buf) else # extract the string from chunks[ichunk][inewline+1] to chunks[jchunk][jnewline] if ichunk == jchunk # common case: current and previous newline in same chunk @@ -1307,7 +1316,7 @@ function iterate(r::Iterators.Reverse{<:EachLine}, state) end write(buf, view(chunks[jchunk], 1:jnewline)) buf.size = _stripnewline(r.itr.keep, buf.size, buf.data) - s = String(_unsafe_take!(buf)) + s = unsafe_takestring!(buf) # overwrite obsolete chunks (ichunk+1:jchunk) i = jchunk diff --git a/base/iobuffer.jl b/base/iobuffer.jl index ed0cd63183dc1..dd3b757c0e3f0 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -1,45 +1,168 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -## work with AbstractVector{UInt8} via I/O primitives ## +# IOBuffer is a Memory{UInt8} backed IO type for in-memory IO. + +# Here, u represents used bytes (already read), X represents bytes still to read, +# - represents bytes uninitialized data but which can be written to later. +# . represents bytes before offset, which the buffer will not touch, until +# a write operation happens. + +# .....uuuuuuuuuuuuuXXXXXXXXXXXXX------------ +# | | | | | | +# | offset ptr size | maxsize +# 1 lastindex(data) + +# N.B: `mark` does not correspond to any index in the buffer. Instead, it stores +# the mark at virtual offset in the buffer. + +# AFTER COMPACTION + +# XXXXXXXXXXXXX-------------------------- +# || | | | | +# |1 ptr size | maxsize +# | lastindex(data) +# offset (set to zero) + +# * The underlying array is always 1-indexed +# * The IOBuffer has full control (ownership) of the underlying array, only when +# buffer.write == true. +# * Unreachable data can be deleted in the buffer's data, shifting the whole thing to the left +# to make room for more data, without replacing or resizing data. +# This can be done only if the buffer is not seekable -# Stateful string mutable struct GenericIOBuffer{T<:AbstractVector{UInt8}} <: IO - data::T # T should support: getindex, setindex!, length, copyto!, similar, and (optionally) resize! - reinit::Bool # if true, data needs to be re-allocated (after take!) + # T should support: getindex, setindex!, length, copyto!, similar, size and (optionally) resize! + data::T + + # The user can take control of `data` out of this struct. When that happens, instead of eagerly allocating + # a new array, we set `.reinit` to true, and then allocate a new one when needed. + # If reinit is true, the buffer is writable, and offset_or_compacted and size is zero. See `take!` + reinit::Bool readable::Bool writable::Bool - seekable::Bool # if not seekable, implementation is free to destroy (compact) past read data - append::Bool # add data at end instead of at pointer - size::Int # end pointer (and write pointer if append == true) + offset - maxsize::Int # fixed array size (typically pre-allocated) - ptr::Int # read (and maybe write) pointer + offset - offset::Int # offset of ptr and size from actual start of data and actual size - mark::Int # reset mark location for ptr (or <0 for no mark) - function GenericIOBuffer{T}(data::T, readable::Bool, writable::Bool, seekable::Bool, append::Bool, - maxsize::Integer) where T<:AbstractVector{UInt8} - require_one_based_indexing(data) - return new(data, false, readable, writable, seekable, append, length(data), maxsize, 1, 0, -1) + # If not seekable, implementation is free to destroy (compact) data before ptr, unless + # it can be recovered using the mark by using `reset`. + # If it IS seekable, the user may always recover any data in 1:size by seeking, + # so no data can be destroyed. + # Non-seekable IOBuffers can only be constructed with `PipeBuffer`, which are writable, + # readable and append. + seekable::Bool + + # If true, write new data to the index size+1 instead of the index ptr. + append::Bool + + # Last index of `data` that has been written to. Data in size+1:end has not yet been used, + # and may contain arbitrary values. + # This value is always in 0 : lastindex(data) + size::Int + + # When the buffer is resized, or a new buffer allocated, this is the maximum size of the buffer. + # A new GenericIOBuffer may be constructed with an existing data larger than `maxsize`. + # When that happensm we must make sure to not have more than `maxsize` bytes in the buffer, + # else reallocating will lose data. So, never write to indices > `maxsize + get_offset(io)` + # This value is always in 0:typemax(Int). + maxsize::Int + + # Data is read/written from/to ptr, except in situations where append is true, in which case + # data is still read from ptr, but written to size+1. + # This value is always in offset + 1 : size+1 + ptr::Int + + # This field has two distinct meanings: + # If the value is positive, it encodes an offset of the start of the data in `data`. + # This is used if the buffer is instantiated from a Vector with non-zero memory offset. + # Then, the IOBuffer stores the underlying memory, and so the first data in the buffer + # is not at index 1. + # If the value is negative, then `-io.offset_or_compacted` gets the number of compacted + # bytes. That's the number of unused bytes deleted from a non-seekable stream to make space. + # We need to keep track of it in order to make `mark` and `position` etc work, that is, + # we need to know the virtual position of the mark even when an arbitrary number + # of unused bytes has been deleted due to compaction. + # Since compaction will move data in the buffer and thereby zero the offset, either the + # offset or the number of compacted bytes will be zero at any point, so both can be + # stored in one field. + # If offset: Value is always in 0:lastindex(data) + # If compacted: Value is in typemin(Int):0 + offset_or_compacted::Int + + # The mark is -1 if not set, else the zero-indexed virtual position of ptr in the buffer. + # Due to compaction and offset, this value is not an index into the buffer, but may be translated + # to an index. + # This value is in -1:typemax(Int) + mark::Int + + # Unsafe constructor which does not do any checking + global function _new_generic_iobuffer( + ::Type{T}, + data::T, + readable::Bool, + writable::Bool, + seekable::Bool, + append::Bool, + maxsize::Int, + ) where T<:AbstractVector{UInt8} + len = Int(length(data))::Int + return new{T}(data, false, readable, writable, seekable, append, len, maxsize, 1, 0, -1) + end +end + +function GenericIOBuffer{T}( + data::T, + readable::Bool, + writable::Bool, + seekable::Bool, + append::Bool, + maxsize::Integer, + truncate::Bool, + ) where T<:AbstractVector{UInt8} + require_one_based_indexing(data) + mz = Int(maxsize)::Int + len = Int(length(data))::Int + if !truncate && mz < len + throw(ArgumentError("maxsize must not be smaller than data length")) end + buf = _new_generic_iobuffer(T, data, readable, writable, seekable, append, mz) + if truncate + buf.size = buf.offset_or_compacted + end + buf end const IOBuffer = GenericIOBuffer{Memory{UInt8}} function GenericIOBuffer(data::T, readable::Bool, writable::Bool, seekable::Bool, append::Bool, - maxsize::Integer) where T<:AbstractVector{UInt8} - GenericIOBuffer{T}(data, readable, writable, seekable, append, maxsize) + maxsize::Integer, truncate::Bool) where T<:AbstractVector{UInt8} + GenericIOBuffer{T}(data, readable, writable, seekable, append, maxsize, truncate) end + +# For this method, we use the underlying Memory of the vector. Therefore, we need to set the, +# ptr and size accordingly, so the buffer only uses the part of the memory that the vector does. function GenericIOBuffer(data::Vector{UInt8}, readable::Bool, writable::Bool, seekable::Bool, append::Bool, - maxsize::Integer) + maxsize::Integer, truncate::Bool) ref = data.ref - buf = GenericIOBuffer(ref.mem, readable, writable, seekable, append, maxsize) + mem = ref.mem offset = memoryrefoffset(ref) - 1 - buf.ptr += offset - buf.size = length(data) + offset - buf.offset = offset + # The user may pass a vector of length <= maxsize, but where the underlying memory + # is larger than maxsize. Don't throw an error in that case. + mz = Int(maxsize)::Int + if !truncate && mz < length(data) + throw(ArgumentError("maxsize must not be smaller than data length")) + end + buf = _new_generic_iobuffer(Memory{UInt8}, mem, readable, writable, seekable, append, mz) + buf.offset_or_compacted = offset + buf.ptr = offset + 1 + if truncate + buf.size = offset + else + buf.size = length(data) + offset + end return buf end +get_offset(io::GenericIOBuffer) = max(0, io.offset_or_compacted) +get_compacted(io::GenericIOBuffer) = max(0, -io.offset_or_compacted) + # allocate Vector{UInt8}s for IOBuffer storage that can efficiently become Strings StringMemory(n::Integer) = unsafe_wrap(Memory{UInt8}, _string_n(n)) StringVector(n::Integer) = wrap(Array, StringMemory(n)) @@ -75,7 +198,7 @@ julia> io = IOBuffer(); julia> write(io, "JuliaLang is a GitHub organization.", " It has many members.") 56 -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a GitHub organization. It has many members." julia> io = IOBuffer(b"JuliaLang is a GitHub organization.") @@ -93,7 +216,7 @@ IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=fa julia> write(io, "JuliaLang is a GitHub organization.") 34 -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a GitHub organization" julia> length(read(IOBuffer(b"data", read=true, truncate=false))) @@ -111,17 +234,11 @@ function IOBuffer( truncate::Union{Bool,Nothing}=nothing, maxsize::Integer=typemax(Int), sizehint::Union{Integer,Nothing}=nothing) - if maxsize < 0 - throw(ArgumentError("negative maxsize")) - end if sizehint !== nothing sizehint!(data, sizehint) end flags = open_flags(read=read, write=write, append=append, truncate=truncate) - buf = GenericIOBuffer(data, flags.read, flags.write, true, flags.append, Int(maxsize)) - if flags.truncate - buf.size = buf.offset - end + buf = GenericIOBuffer(data, flags.read, flags.write, true, flags.append, maxsize, flags.truncate) return buf end @@ -131,17 +248,23 @@ function IOBuffer(; append::Union{Bool,Nothing}=nothing, truncate::Union{Bool,Nothing}=true, maxsize::Integer=typemax(Int), - sizehint::Union{Integer,Nothing}=nothing) - size = sizehint !== nothing ? Int(sizehint) : maxsize != typemax(Int) ? Int(maxsize) : 32 + sizehint::Union{Integer,Nothing}=nothing, + ) + mz = Int(maxsize)::Int + if mz < 0 + throw(ArgumentError("negative maxsize")) + end + size = if sizehint !== nothing + # Allow negative sizehint, just like `sizehint!` does + min(mz, max(0, Int(sizehint)::Int)) + else + min(mz, 32) + end flags = open_flags(read=read, write=write, append=append, truncate=truncate) - buf = IOBuffer( - StringMemory(size), - read=flags.read, - write=flags.write, - append=flags.append, - truncate=flags.truncate, - maxsize=maxsize) - fill!(buf.data, 0) + # A common usecase of IOBuffer is to incrementally construct strings. By using StringMemory + # as the default storage, we can turn the result into a string without copying. + buf = _new_generic_iobuffer(Memory{UInt8}, StringMemory(size), flags.read, flags.write, true, flags.append, mz) + buf.size = 0 return buf end @@ -158,21 +281,53 @@ If `data` is given, creates a `PipeBuffer` to operate on a data vector, optionally specifying a size beyond which the underlying `Array` may not be grown. """ PipeBuffer(data::AbstractVector{UInt8}=Memory{UInt8}(); maxsize::Int = typemax(Int)) = - GenericIOBuffer(data, true, true, false, true, maxsize) + GenericIOBuffer(data, true, true, false, true, maxsize, false) PipeBuffer(maxsize::Integer) = (x = PipeBuffer(StringMemory(maxsize), maxsize = maxsize); x.size = 0; x) +# Internal method where truncation IS supported +function _truncated_pipebuffer(data::AbstractVector{UInt8}=Memory{UInt8}(); maxsize::Int = typemax(Int)) + buf = PipeBuffer(data) + buf.size = get_offset(buf) + buf.maxsize = maxsize + buf +end + _similar_data(b::GenericIOBuffer, len::Int) = similar(b.data, len) _similar_data(b::IOBuffer, len::Int) = StringMemory(len) -function copy(b::GenericIOBuffer) - ret = typeof(b)(b.reinit ? _similar_data(b, 0) : b.writable ? - copyto!(_similar_data(b, length(b.data)), b.data) : b.data, - b.readable, b.writable, b.seekable, b.append, b.maxsize) - ret.size = b.size - ret.ptr = b.ptr - ret.mark = b.mark - ret.offset = b.offset - return ret +# Note: Copying may change the value of the position (and mark) for un-seekable streams. +# However, these values are not stable anyway due to compaction. + +function copy(b::GenericIOBuffer{T}) where T + if b.reinit + # If buffer is used up, allocate a new size-zero buffer + # Reinit implies writable, and that ptr, size, offset and mark are already the default values + return typeof(b)(_similar_data(b, 0), b.readable, b.writable, b.seekable, b.append, b.maxsize, false) + elseif b.writable + # Else, we just copy the reachable bytes. If buffer is seekable, all bytes + # after offset are reachable, since they can be seeked to + used_span = get_used_span(b) + compacted = first(used_span) - get_offset(b) - 1 + len = length(used_span) + data = copyto!(_similar_data(b, len), view(b.data, used_span)) + ret = typeof(b)(data, b.readable, b.writable, b.seekable, b.append, b.maxsize, false) + ret.size = len + # Copying data over implicitly compacts, and may add compaction + ret.offset_or_compacted = -get_compacted(b) - compacted + ret.ptr = b.ptr - first(used_span) + 1 + ret.mark = b.mark + return ret + else + # When the buffer is just readable, they can share the same data, so we just make + # a shallow copy of the IOBuffer struct. + # Use internal constructor because we want to allow b.maxsize to be larger than data, + # in case that is the case for `b`. + ret = _new_generic_iobuffer(T, b.data, b.readable, b.writable, b.seekable, b.append, b.maxsize) + ret.offset_or_compacted = b.offset_or_compacted + ret.ptr = b.ptr + ret.mark = b.mark + return ret + end end show(io::IO, b::GenericIOBuffer) = print(io, "IOBuffer(data=UInt8[...], ", @@ -180,9 +335,9 @@ show(io::IO, b::GenericIOBuffer) = print(io, "IOBuffer(data=UInt8[...], ", "writable=", b.writable, ", ", "seekable=", b.seekable, ", ", "append=", b.append, ", ", - "size=", b.size - b.offset, ", ", + "size=", b.size - get_offset(b), ", ", "maxsize=", b.maxsize == typemax(Int) ? "Inf" : b.maxsize, ", ", - "ptr=", b.ptr - b.offset, ", ", + "ptr=", b.ptr - get_offset(b), ", ", "mark=", b.mark, ")") @noinline function _throw_not_readable() @@ -192,7 +347,7 @@ end function unsafe_read(from::GenericIOBuffer, p::Ptr{UInt8}, nb::UInt) from.readable || _throw_not_readable() - avail = bytesavailable(from) + avail = bytesavailable(from) % UInt adv = min(avail, nb) unsafe_read!(p, from.data, from.ptr, adv) from.ptr += adv @@ -221,7 +376,45 @@ function unsafe_read!(dest::Ptr{UInt8}, src::DenseBytes, so::Integer, nbytes::UI nothing end -function peek(from::GenericIOBuffer, T::Union{Type{Int16},Type{UInt16},Type{Int32},Type{UInt32},Type{Int64},Type{UInt64},Type{Int128},Type{UInt128},Type{Float16},Type{Float32},Type{Float64}}) +const MultiByteBitNumberType = Union{ + Type{UInt16}, + Type{Int16}, + Type{UInt32}, + Type{Int32}, + Type{UInt64}, + Type{Int64}, + Type{UInt128}, + Type{Int128}, + Type{Float16}, + Type{Float32}, + Type{Float64}, +} + +function load_from_array(T::MultiByteBitNumberType, data::AbstractArray{UInt8}, from::Int) + x = if T <: AbstractFloat + uinttype(T)(0) + else + unsigned(T)(0) + end + for i in 0:sizeof(x)-1 + x |= typeof(x)(data[from + i]) << (8 * i) + end + reinterpret(T, ltoh(x)) +end + +function peek(from::GenericIOBuffer, T::MultiByteBitNumberType) + from.readable || _throw_not_readable() + avail = bytesavailable(from) + nb = sizeof(T) + if nb > avail + throw(EOFError()) + end + return load_from_array(T, from.data, from.ptr) +end + +# This method can use a pointer, since the underlying buffer is dense +# and memory backed +function peek(from::GenericIOBuffer{<:MutableDenseArrayType}, T::MultiByteBitNumberType) from.readable || _throw_not_readable() avail = bytesavailable(from) nb = sizeof(T) @@ -235,29 +428,12 @@ function peek(from::GenericIOBuffer, T::Union{Type{Int16},Type{UInt16},Type{Int3 return x end -function read(from::GenericIOBuffer, T::Union{Type{Int16},Type{UInt16},Type{Int32},Type{UInt32},Type{Int64},Type{UInt64},Type{Int128},Type{UInt128},Type{Float16},Type{Float32},Type{Float64}}) +function read(from::GenericIOBuffer, T::MultiByteBitNumberType) x = peek(from, T) from.ptr += sizeof(T) return x end -function read_sub(from::GenericIOBuffer, a::AbstractArray{T}, offs, nel) where T - require_one_based_indexing(a) - from.readable || _throw_not_readable() - if offs+nel-1 > length(a) || offs < 1 || nel < 0 - throw(BoundsError()) - end - if isa(a, MutableDenseArrayType{UInt8}) - nb = UInt(nel * sizeof(T)) - GC.@preserve a unsafe_read(from, pointer(a, offs), nb) - else - for i = offs:offs+nel-1 - a[i] = read(from, T) - end - end - return a -end - @inline function read(from::GenericIOBuffer, ::Type{UInt8}) from.readable || _throw_not_readable() ptr = from.ptr @@ -283,18 +459,33 @@ read(from::GenericIOBuffer, ::Type{Ptr{T}}) where {T} = convert(Ptr{T}, read(fro isreadable(io::GenericIOBuffer) = io.readable iswritable(io::GenericIOBuffer) = io.writable -filesize(io::GenericIOBuffer) = (io.seekable ? io.size - io.offset : bytesavailable(io)) +# Number of bytes that can be read from the buffer, if you seek to the start first. +filesize(io::GenericIOBuffer) = (io.seekable ? io.size - get_offset(io) : bytesavailable(io)) + +# Number of bytes that can be read from the buffer. bytesavailable(io::GenericIOBuffer) = io.size - io.ptr + 1 -position(io::GenericIOBuffer) = io.ptr - io.offset - 1 + +# TODO: Document that position for an unmarked and unseekable stream is invalid (and make it error?) +function position(io::GenericIOBuffer) + # Position is zero-indexed, but ptr is one-indexed, hence the -1 + io.ptr - io.offset_or_compacted - 1 +end function skip(io::GenericIOBuffer, n::Integer) skip(io, clamp(n, Int)) end + function skip(io::GenericIOBuffer, n::Int) + # In both cases, the result will never go to before the first position, + # nor beyond the last position, and will not throw an error unless the stream + # is not seekable and try to skip a negative number of bytes. if signbit(n) + # Skipping a negative number of bytes is equivalent to seeking backwards. seekto = clamp(widen(position(io)) + widen(n), Int) seek(io, seekto) # Does error checking else + # Don't use seek in order to allow a non-seekable IO to still skip bytes. + # Handle overflow. n_max = io.size + 1 - io.ptr io.ptr += min(n, n_max) io @@ -304,16 +495,30 @@ end function seek(io::GenericIOBuffer, n::Integer) seek(io, clamp(n, Int)) end + +function translate_seek_position(io::GenericIOBuffer, n::Int) + # If there is an offset (the field F is positive), then there are F unused bytes at the beginning + # of the data, and we need to seek to n + F + 1. (Also compensate for `seek` being zero- + # indexed) + + # If bytes has been compacted (field F is negative), then F bytes has been deleted from + # the buffer, and a virtual position n means a position n + F in the data. + # Remember that F is negative, so n + F is subtracting from n. So we also end up with + # n + F + 1. + clamp(widen(n) + widen(io.offset_or_compacted) + widen(1), Int) +end + function seek(io::GenericIOBuffer, n::Int) if !io.seekable ismarked(io) || throw(ArgumentError("seek failed, IOBuffer is not seekable and is not marked")) n == io.mark || throw(ArgumentError("seek failed, IOBuffer is not seekable and n != mark")) end + # TODO: REPL.jl relies on the fact that this does not throw (by seeking past the beginning or end # of an GenericIOBuffer), so that would need to be fixed in order to throw an error here - #(n < 0 || n > io.size - io.offset) && throw(ArgumentError("Attempted to seek outside IOBuffer boundaries.")) - #io.ptr = n + io.offset + 1 - io.ptr = clamp(n, 0, io.size - io.offset) + io.offset + 1 + max_ptr = io.size + 1 + min_ptr = get_offset(io) + 1 + io.ptr = clamp(translate_seek_position(io, n), min_ptr, max_ptr) return io end @@ -322,113 +527,163 @@ function seekend(io::GenericIOBuffer) return io end -# choose a resize strategy based on whether `resize!` is defined: -# for a Vector, we use `resize!`, but for most other types, -# this calls `similar`+copy -function _resize!(io::GenericIOBuffer, sz::Int) - a = io.data - offset = io.offset - if applicable(resize!, a, sz) - if offset != 0 - size = io.size - size > offset && copyto!(a, 1, a, offset + 1, min(sz, size - offset)) - io.ptr -= offset - io.size -= offset - io.offset = 0 - end - resize!(a, sz) +# Resize the io's data to `new_size`, which must not be > io.maxsize. +# Use `resize!` if the data supports it, else reallocate a new one and +# copy the old data over. +# If not `exact` and resizing is not supported, overallocate in order to +# prevent excessive resizing. +function _resize!(io::GenericIOBuffer, new_size::Int, exact::Bool) + old_data = io.data + if applicable(resize!, old_data, new_size) + resize!(old_data, new_size) else - size = io.size - if size >= sz && sz != 0 - b = a - else - b = _similar_data(io, sz == 0 ? 0 : max(overallocation(size - io.offset), sz)) - end - size > offset && copyto!(b, 1, a, offset + 1, min(sz, size - offset)) - io.data = b - io.ptr -= offset - io.size -= offset - io.offset = 0 + new_size = exact ? new_size : min(io.maxsize, overallocation(new_size)) + used_span = get_used_span(io) + deleted = first(used_span) - 1 + compacted = deleted - get_offset(io) + new_data = _similar_data(io, new_size) + io.data = new_data + iszero(new_size) && return io + len_used = length(used_span) + iszero(len_used) || copyto!(new_data, 1, old_data, first(used_span), len_used) + # Copying will implicitly compact, and so compaction must be updated + io.offset_or_compacted = -get_compacted(io) - compacted + io.ptr -= deleted + io.size = len_used end return io end function truncate(io::GenericIOBuffer, n::Integer) io.writable || throw(ArgumentError("truncate failed, IOBuffer is not writeable")) + # Non-seekable buffers can only be constructed with `PipeBuffer`, which is explicitly + # documented to not be truncatable. io.seekable || throw(ArgumentError("truncate failed, IOBuffer is not seekable")) n < 0 && throw(ArgumentError("truncate failed, n bytes must be ≥ 0, got $n")) n > io.maxsize && throw(ArgumentError("truncate failed, $(n) bytes is exceeds IOBuffer maxsize $(io.maxsize)")) - n = Int(n) + n = Int(n)::Int + offset = get_offset(io) + current_size = io.size - offset if io.reinit - io.data = _similar_data(io, n) + # If reinit, we don't need to truncate anything but just reinitializes + # the buffer with zeros. Mark, ptr and offset has already been reset. + io.data = fill!(_similar_data(io, n), 0x00) io.reinit = false - elseif n > length(io.data) + io.offset - _resize!(io, n) - end - ismarked(io) && io.mark > n && unmark(io) - n += io.offset - io.data[io.size+1:n] .= 0 - io.size = n - io.ptr = min(io.ptr, n+1) + io.size = n + elseif n < current_size + # Else, if we need to shrink the iobuffer, we simply change the pointers without + # actually shrinking the underlying storage, or copying data. + + # Clear the mark if it points to data that has now been deleted. + if translate_seek_position(io, io.mark) > n+offset + io.mark = -1 + end + io.size = n + offset + io.ptr = min(io.ptr, n + offset + 1) + elseif n > current_size + if n + offset > io.maxsize + compact!(io) + end + _resize!(io, n + get_offset(io), false) + fill!(view(io.data, io.size + 1:min(length(io.data), n + get_offset(io))), 0x00) + io.size = min(length(io.data), n + get_offset(io)) + end return io end -function compact(io::GenericIOBuffer) - io.writable || throw(ArgumentError("compact failed, IOBuffer is not writeable")) - io.seekable && throw(ArgumentError("compact failed, IOBuffer is seekable")) - io.reinit && return - local ptr::Int, bytes_to_move::Int - if ismarked(io) && io.mark < position(io) - io.mark == 0 && return - ptr = io.mark + io.offset - bytes_to_move = bytesavailable(io) + (io.ptr - ptr) - else - ptr = io.ptr - bytes_to_move = bytesavailable(io) +# Ensure that the buffer has room for at least `nshort` more bytes, except when +# doing that would exceed maxsize. +@inline ensureroom(io::GenericIOBuffer, nshort::Int) = ensureroom(io, UInt(nshort)) + +@inline function ensureroom(io::GenericIOBuffer, nshort::UInt) + # If the IO is not writable, we call the slow path only to error. + # If reinit, the data has been handed out to the user, and the IOBuffer + # no longer controls it, so we need to allocate a new one. + if !io.writable || io.reinit + return ensureroom_reallocate(io, nshort) + end + # The fast path here usually checks there is already room, then does nothing. + # When append is true, new data is added after io.size, not io.ptr + existing_space = min(lastindex(io.data), io.maxsize + get_offset(io)) - (io.append ? io.size : io.ptr - 1) + if existing_space < nshort % Int + # Outline this function to make it more likely that ensureroom inlines itself + return ensureroom_slowpath(io, nshort, existing_space) end - copyto!(io.data, 1, io.data, ptr, bytes_to_move) - io.size -= ptr - 1 - io.ptr -= ptr - 1 - io.offset = 0 - return + return io end -@noinline function ensureroom_slowpath(io::GenericIOBuffer, nshort::UInt) +# Throw error (placed in this function to outline it) or reinit the buffer +@noinline function ensureroom_reallocate(io::GenericIOBuffer, nshort::UInt) io.writable || throw(ArgumentError("ensureroom failed, IOBuffer is not writeable")) - if io.reinit - io.data = _similar_data(io, nshort % Int) - io.reinit = false + io.data = _similar_data(io, min(io.maxsize, nshort % Int)) + io.reinit = false + io.offset_or_compacted = -get_compacted(io) + return io +end + +# Here, we already know there is not enough room at the end of the io's data. +@noinline function ensureroom_slowpath(io::GenericIOBuffer, nshort::UInt, available_bytes::Int) + reclaimable_bytes = first(get_used_span(io)) - 1 + # Avoid resizing and instead compact the buffer, only if we gain enough bytes from + # doing so (at least 32 bytes and 1/8th of the data length). Also, if we would have + # to resize anyway, there would be no point in compacting, so also check that. + if ( + reclaimable_bytes ≥ 32 && + reclaimable_bytes ≥ length(io.data) >>> 3 && + (reclaimable_bytes + available_bytes) % UInt ≥ nshort + ) + compact!(io) + return io end - if !io.seekable - if !ismarked(io) && io.ptr > io.offset+1 && io.size <= io.ptr - 1 - io.ptr = 1 - io.size = 0 - io.offset = 0 - else - datastart = (ismarked(io) ? io.mark : io.ptr - io.offset) - if (io.size-io.offset+nshort > io.maxsize) || - (datastart > 4096 && datastart > io.size - io.ptr) || - (datastart > 262144) - # apply somewhat arbitrary heuristics to decide when to destroy - # old, read data to make more room for new data - compact(io) - end + + desired_size = length(io.data) + Int(nshort) - available_bytes + if desired_size > io.maxsize + # If we can't fit all the requested data in the new buffer, we need to + # fit as much as possible, so we must compact + if !iszero(reclaimable_bytes) + desired_size -= compact!(io) end + # Max out the buffer size if we want more than the buffer size + if length(io.data) < io.maxsize + _resize!(io, io.maxsize, true) + end + else + # Else, we request only the requested size, but set `exact` to `false`, + # in order to overallocate to avoid growing the buffer by too little + _resize!(io, desired_size, false) end - return + + return io end -@inline ensureroom(io::GenericIOBuffer, nshort::Int) = ensureroom(io, UInt(nshort)) -@inline function ensureroom(io::GenericIOBuffer, nshort::UInt) - if !io.writable || (!io.seekable && io.ptr > io.offset+1) || io.reinit - ensureroom_slowpath(io, nshort) - end - n = min((nshort % Int) + (io.append ? io.size : io.ptr-1) - io.offset, io.maxsize) - l = length(io.data) + io.offset - if n > l - _resize!(io, Int(n)) +# Get the indices in data which cannot be deleted +function get_used_span(io::IOBuffer) + # A seekable buffer can recover data before ptr + return if io.seekable + get_offset(io) + 1 : io.size + # If non-seekable, the mark can be used to recover data before ptr, + # so data at the mark and after must also be saved + elseif io.mark > -1 + min(io.ptr, translate_seek_position(io, io.mark)) : io.size + else + io.ptr : io.size end - return io +end + +# Delete any offset, and also compact data if buffer is not seekable. +# Return the number of bytes deleted +function compact!(io::GenericIOBuffer)::Int + offset = get_offset(io) + used_span = get_used_span(io) + deleted = first(used_span) - 1 + compacted = deleted - offset + iszero(deleted) && return 0 + data = io.data + copyto!(data, 1, data, deleted + 1, length(used_span)) + io.offset_or_compacted = -get_compacted(io) - compacted + io.ptr -= deleted + io.size -= deleted + return deleted end eof(io::GenericIOBuffer) = (io.ptr - 1 >= io.size) @@ -439,17 +694,17 @@ function closewrite(io::GenericIOBuffer) end @noinline function close(io::GenericIOBuffer{T}) where T + if io.writable && !io.reinit + _resize!(io, 0, true) + end io.readable = false io.writable = false io.seekable = false io.size = 0 - io.offset = 0 io.maxsize = 0 io.ptr = 1 io.mark = -1 - if io.writable && !io.reinit - io.data = _resize!(io, 0) - end + io.offset_or_compacted = -get_compacted(io) nothing end @@ -472,31 +727,42 @@ julia> String(take!(io)) ``` """ function take!(io::GenericIOBuffer) - ismarked(io) && unmark(io) + io.mark = -1 if io.seekable - nbytes = io.size - io.offset - data = copyto!(StringVector(nbytes), 1, io.data, io.offset + 1, nbytes) + # If the buffer is seekable, then the previously consumed bytes from ptr+1:size + # must still be output, as they are not truly gone. + # Hence, we output all bytes from 1:io.size + offset = get_offset(io) + nbytes = io.size - offset + data = copyto!(StringVector(nbytes), 1, io.data, offset + 1, nbytes) else + # Else, if not seekable, bytes from 1:ptr-1 are truly gone and should not + # be output. Hence, we output `bytesavailable`, which is ptr:size nbytes = bytesavailable(io) data = read!(io, StringVector(nbytes)) end if io.writable + io.reinit = true io.ptr = 1 io.size = 0 - io.offset = 0 + io.offset_or_compacted = 0 end return data end + +# This method is specialized because we know the underlying data is a Memory, so we can +# e.g. wrap directly in an array without copying. Otherwise the logic is the same as +# the generic method function take!(io::IOBuffer) - ismarked(io) && unmark(io) + io.mark = -1 if io.seekable nbytes = filesize(io) if nbytes == 0 || io.reinit data = StringVector(0) elseif io.writable - data = wrap(Array, memoryref(io.data, io.offset + 1), nbytes) + data = wrap(Array, memoryref(io.data, get_offset(io) + 1), nbytes) else - data = copyto!(StringVector(nbytes), 1, io.data, io.offset + 1, nbytes) + data = copyto!(StringVector(nbytes), 1, io.data, get_offset(io) + 1, nbytes) end else nbytes = bytesavailable(io) @@ -505,18 +771,92 @@ function take!(io::IOBuffer) elseif io.writable data = wrap(Array, memoryref(io.data, io.ptr), nbytes) else - data = read!(io, data) + error("Unreachable IOBuffer state") end end if io.writable io.reinit = true io.ptr = 1 io.size = 0 - io.offset = 0 + io.offset_or_compacted = 0 end return data end +"Internal method. This method can be faster than takestring!, because it does not +reset the buffer to a usable state, and it does not check for io.reinit. +Using the buffer after calling unsafe_takestring! may cause undefined behaviour. +This function is meant to be used when the buffer is only used as a temporary +string builder, which is discarded after the string is built." +function unsafe_takestring!(io::IOBuffer) + used_span = get_used_span(io) + nbytes = length(used_span) + from = first(used_span) + isempty(used_span) && return "" + # The C function can only copy from the start of the memory. + # Fortunately, in most cases, the offset will be zero. + return if isone(from) + ccall(:jl_genericmemory_to_string, Ref{String}, (Any, Int), io.data, nbytes) + else + mem = StringMemory(nbytes % UInt) + unsafe_copyto!(mem, 1, io.data, from, nbytes) + unsafe_takestring(mem) + end +end + +""" + takestring!(io::IOBuffer) -> String + +Return the content of `io` as a `String`, resetting the buffer to its initial +state. +This is preferred over calling `String(take!(io))` to create a string from +an `IOBuffer`. + +# Examples +```jldoctest +julia> io = IOBuffer(); + +julia> write(io, [0x61, 0x62, 0x63]); + +julia> s = takestring!(io) +"abc" + +julia> isempty(take!(io)) # io is now empty +true +``` + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. +""" +function takestring!(io::IOBuffer) + # If the buffer has been used up and needs to be replaced, there are no bytes, and + # we can return an empty string without interacting with the buffer at all. + io.reinit && return "" + + # If the iobuffer is writable, taking will remove the buffer from `io`. + # So, we reset the iobuffer, and directly unsafe takestring. + return if io.writable + s = unsafe_takestring!(io) + io.reinit = true + io.mark = -1 + io.ptr = 1 + io.size = 0 + io.offset_or_compacted = 0 + s + else + # If the buffer is not writable, taking will NOT remove the buffer, + # so if we just converted the buffer to a string, garbage collecting + # the string would free the memory underneath the iobuffer + used_span = get_used_span(io) + mem = StringMemory(length(used_span)) + unsafe_copyto!(mem, 1, io.data, first(used_span), length(used_span)) + unsafe_takestring(mem) + end +end + +# Fallback methods +takestring!(io::GenericIOBuffer) = String(take!(io)) + """ _unsafe_take!(io::IOBuffer) @@ -529,46 +869,79 @@ state. This should only be used internally for performance-critical It might save an allocation compared to `take!` (if the compiler elides the Array allocation), as well as omits some checks. """ -_unsafe_take!(io::IOBuffer) = - wrap(Array, io.size == io.offset ? - memoryref(Memory{UInt8}()) : - memoryref(io.data, io.offset + 1), - io.size - io.offset) +function _unsafe_take!(io::IOBuffer) + offset = get_offset(io) + mem = if io.size == offset + memoryref(Memory{UInt8}()) + else + memoryref(io.data, offset + 1) + end + wrap(Array, mem, io.size - offset) +end function write(to::IO, from::GenericIOBuffer) - written::Int = bytesavailable(from) + # This would cause an infinite loop, as it should read until the end, but more + # data is being written into it continuously. if to === from - from.ptr = from.size + 1 + throw(ArgumentError("Writing all content fron an IOBuffer into itself in invalid")) else - written = GC.@preserve from unsafe_write(to, pointer(from.data, from.ptr), UInt(written)) - from.ptr += written + from.readable || _throw_not_readable() + written = write(to, view(from.data, from.ptr:from.size)) + from.ptr = from.size + 1 end return written end function unsafe_write(to::GenericIOBuffer, p::Ptr{UInt8}, nb::UInt) ensureroom(to, nb) - ptr = (to.append ? to.size+1 : to.ptr) - written = Int(min(nb, Int(length(to.data))::Int - ptr + 1)) - towrite = written - d = to.data - while towrite > 0 - @inbounds d[ptr] = unsafe_load(p) - ptr += 1 + size = to.size + append = to.append + ptr = append ? size+1 : to.ptr + data = to.data + to_write = min(nb, (min(Int(length(data))::Int, to.maxsize + get_offset(to)) - ptr + 1) % UInt) % Int + # Dispatch based on the type of data, to possibly allow using memcpy + _unsafe_write(data, p, ptr, to_write % UInt) + # Update to.size only if the ptr has advanced to higher than + # the previous size. Otherwise, we just overwrote existing data + to.size = max(size, ptr + to_write - 1) + # If to.append, we only update size, not ptr. + if !append + to.ptr = ptr + to_write + end + return to_write +end + +@inline function _unsafe_write(data::AbstractVector{UInt8}, p::Ptr{UInt8}, from::Int, nb::UInt) + for i in 0:nb-1 + data[from + i] = unsafe_load(p) p += 1 - towrite -= 1 end - to.size = max(to.size, ptr - 1) - if !to.append - to.ptr += written +end + +@inline function _unsafe_write(data::MutableDenseArrayType{UInt8}, p::Ptr{UInt8}, from::Int, nb::UInt) + # Calling `unsafe_copyto!` is very efficient for large arrays, but has some overhead + # for small (< 5 bytes) arrays. + # Since a common use case of IOBuffer is to construct strings incrementally, often + # one char at a time, it's crucial to be fast in the case of small arrays. + # This optimization only gives a minor 10% speed boost in the best case. + if nb < 5 + @inbounds for i in UInt(1):nb + data[from + (i % Int) - 1] = unsafe_load(p, i) + end + else + GC.@preserve data begin + ptr = Ptr{UInt8}(pointer(data, from))::Ptr{UInt8} + @inline unsafe_copyto!(ptr, p, nb) + end end - return written end @inline function write(to::GenericIOBuffer, a::UInt8) ensureroom(to, UInt(1)) ptr = (to.append ? to.size+1 : to.ptr) - if ptr > to.maxsize + # We have just ensured there is room for 1 byte, EXCEPT if we were to exceed + # maxsize. So, we just need to check that here. + if ptr > to.maxsize + get_offset(to) return 0 else to.data[ptr] = a @@ -581,31 +954,26 @@ end end readbytes!(io::GenericIOBuffer, b::MutableDenseArrayType{UInt8}, nb=length(b)) = readbytes!(io, b, Int(nb)) + function readbytes!(io::GenericIOBuffer, b::MutableDenseArrayType{UInt8}, nb::Int) - nr = min(nb, bytesavailable(io)) - if length(b) < nr - resize!(b, nr) + io.readable || _throw_not_readable() + to_read = min(nb, bytesavailable(io)) + if length(b) < to_read + resize!(b, to_read) end - read_sub(io, b, 1, nr) - return nr + checkbounds(b, 1:to_read) + GC.@preserve b unsafe_read(io, pointer(b), to_read) + to_read end read(io::GenericIOBuffer) = read!(io, StringVector(bytesavailable(io))) + +# For IO buffers, all the data is immediately available. readavailable(io::GenericIOBuffer) = read(io) -read(io::GenericIOBuffer, nb::Integer) = read!(io, StringVector(min(nb, bytesavailable(io)))) -function occursin(delim::UInt8, buf::IOBuffer) - p = pointer(buf.data, buf.ptr) - q = GC.@preserve buf ccall(:memchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p, delim, bytesavailable(buf)) - return q != C_NULL -end +read(io::GenericIOBuffer, nb::Integer) = read!(io, StringVector(min(nb, bytesavailable(io)))) function occursin(delim::UInt8, buf::GenericIOBuffer) - data = buf.data - for i = buf.ptr:buf.size - @inbounds b = data[i] - b == delim && return true - end - return false + return in(delim, view(buf.data, buf.ptr:buf.size)) end function copyuntil(out::IO, io::GenericIOBuffer, delim::UInt8; keep::Bool=false) @@ -622,21 +990,45 @@ function copyuntil(out::IO, io::GenericIOBuffer, delim::UInt8; keep::Bool=false) end function copyline(out::GenericIOBuffer, s::IO; keep::Bool=false) - copyuntil(out, s, 0x0a, keep=true) - line = out.data - i = out.size # XXX: this is only correct for appended data. if the data was inserted, only ptr should change - if keep || i == out.offset || line[i] != 0x0a + # If the data is copied into the middle of the buffer of `out` instead of appended to the end, + # and !keep, and the line copied ends with \r\n, then the copyuntil (even if keep=false) + # will overwrite one too many bytes with the new \r byte. + # Work around this by making a new temporary buffer. + # Could perhaps be done better + if !out.append && out.ptr < out.size + 1 + newbuf = IOBuffer() + copyuntil(newbuf, s, 0x0a, keep=true) + v = take!(newbuf) + # Remove \r\n or \n if present + if !keep + if length(v) > 1 && last(v) == UInt8('\n') + pop!(v) + end + if length(v) > 1 && last(v) == UInt8('\r') + pop!(v) + end + end + write(out, v) return out - elseif i < 2 || line[i-1] != 0x0d - i -= 1 else - i -= 2 - end - out.size = i - if !out.append - out.ptr = i+1 + # Else, we can just copy the data directly into the buffer, and then + # subtract the last one or two bytes depending on `keep`. + copyuntil(out, s, 0x0a, keep=true) + line = out.data + i = out.size + if keep || i == out.offset_or_compacted || line[i] != 0x0a + return out + elseif i < 2 || line[i-1] != 0x0d + i -= 1 + else + i -= 2 + end + out.size = i + if !out.append + out.ptr = i+1 + end + return out end - return out end function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) @@ -644,6 +1036,7 @@ function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) # note: findfirst + copyto! is much faster than a single loop # except for nout ≲ 20. A single loop is 2x faster for nout=5. nout = nread = something(findfirst(==(0x0a), data), length(data))::Int + # Remove the 0x0a (newline) if not keep, and also remove the 0x0d (\r) if it is there if !keep && nout > 0 && data[nout] == 0x0a nout -= 1 nout > 0 && data[nout] == 0x0d && (nout -= 1) @@ -652,6 +1045,7 @@ function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) io.ptr += nread return out end + copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) copyline(out::GenericIOBuffer, io::GenericIOBuffer; keep::Bool=false) = _copyline(out, io; keep) diff --git a/base/iostream.jl b/base/iostream.jl index 7a06e7d1e237b..c58818968c0b1 100644 --- a/base/iostream.jl +++ b/base/iostream.jl @@ -75,6 +75,16 @@ fd(s::IOStream) = RawFD(ccall(:jl_ios_fd, Clong, (Ptr{Cvoid},), s.ios)) stat(s::IOStream) = stat(fd(s)) +""" + isopen(s::IOStream) + +Check if the stream is not yet closed. + +A closed `IOStream` may still have data to read in its buffer, +use [`eof`](@ref) to check for the ability to read data. + +Use the `FileWatching` package to be notified when a file might be writable or readable. +""" isopen(s::IOStream) = ccall(:ios_isopen, Cint, (Ptr{Cvoid},), s.ios) != 0 function close(s::IOStream) @@ -111,7 +121,7 @@ julia> write(io, "JuliaLang is a GitHub organization.") julia> truncate(io, 15) IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=false, size=15, maxsize=Inf, ptr=16, mark=-1) -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a " julia> io = IOBuffer(); @@ -120,7 +130,7 @@ julia> write(io, "JuliaLang is a GitHub organization."); julia> truncate(io, 40); -julia> String(take!(io)) +julia> takestring!(io) "JuliaLang is a GitHub organization.\\0\\0\\0\\0\\0" ``` """ @@ -469,7 +479,7 @@ function readuntil_string(s::IOStream, delim::UInt8, keep::Bool) end readuntil(s::IOStream, delim::AbstractChar; keep::Bool=false) = isascii(delim) ? readuntil_string(s, delim % UInt8, keep) : - String(_unsafe_take!(copyuntil(IOBuffer(sizehint=70), s, delim; keep))) + takestring!(copyuntil(IOBuffer(sizehint=70), s, delim; keep)) function readline(s::IOStream; keep::Bool=false) @_lock_ios s ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, '\n', 1, keep ? 0 : 2) diff --git a/base/irrationals.jl b/base/irrationals.jl index f86b55e4faaa7..e0e7fc3bc2e1d 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -182,7 +182,7 @@ isinteger(::AbstractIrrational) = false iszero(::AbstractIrrational) = false isone(::AbstractIrrational) = false -hash(x::Irrational, h::UInt) = 3*objectid(x) - h +hash(x::Irrational, h::UInt) = 3h - objectid(x) widen(::Type{T}) where {T<:Irrational} = T diff --git a/base/iterators.jl b/base/iterators.jl index c7450781c4928..9812feb2d62cd 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -6,7 +6,7 @@ Methods for working with Iterators. baremodule Iterators # small dance to make this work from Base or Intrinsics -import ..@__MODULE__, ..parentmodule +import Base: @__MODULE__, parentmodule const Base = parentmodule(@__MODULE__) using .Base: @inline, Pair, Pairs, AbstractDict, IndexLinear, IndexStyle, AbstractVector, Vector, @@ -16,15 +16,15 @@ using .Base: (:), |, +, -, *, !==, !, ==, !=, <=, <, >, >=, =>, missing, any, _counttuple, eachindex, ntuple, zero, prod, reduce, in, firstindex, lastindex, tail, fieldtypes, min, max, minimum, zero, oneunit, promote, promote_shape, LazyString, - afoldl -using Core + afoldl, mod1 +using .Core using Core: @doc -using .Base: - cld, fld, resize!, IndexCartesian -using .Base.Checked: checked_mul +using Base: + cld, fld, resize!, IndexCartesian, Checked +using .Checked: checked_mul -import .Base: +import Base: first, last, isempty, length, size, axes, ndims, eltype, IteratorSize, IteratorEltype, promote_typejoin, @@ -32,7 +32,7 @@ import .Base: getindex, setindex!, get, iterate, popfirst!, isdone, peek, intersect -export enumerate, zip, rest, countfrom, take, drop, takewhile, dropwhile, cycle, repeated, product, flatten, flatmap, partition +export enumerate, zip, rest, countfrom, take, drop, takewhile, dropwhile, cycle, repeated, product, flatten, flatmap, partition, nth public accumulate, filter, map, peel, reverse, Stateful """ @@ -534,10 +534,15 @@ filter(flt, itr) = Filter(flt, itr) function iterate(f::Filter, state...) y = iterate(f.itr, state...) while y !== nothing - if f.flt(y[1]) - return y + v, s = y + if f.flt(v) + if y isa Tuple{Any,Any} + return (v, s) # incorporate type information that may be improved by user-provided `f.flt` + else + return y + end end - y = iterate(f.itr, y[2]) + y = iterate(f.itr, s) end nothing end @@ -625,13 +630,19 @@ end """ rest(iter, state) -An iterator that yields the same elements as `iter`, but starting at the given `state`. +An iterator that yields the same elements as `iter`, but starting at the given `state`, which +must be a state obtainable via a sequence of one or more calls to `iterate(iter[, state])` See also: [`Iterators.drop`](@ref), [`Iterators.peel`](@ref), [`Base.rest`](@ref). # Examples ```jldoctest -julia> collect(Iterators.rest([1,2,3,4], 2)) +julia> iter = [1,2,3,4]; + +julia> val, state = iterate(iter) +(1, 2) + +julia> collect(Iterators.rest(iter, state)) 3-element Vector{Int64}: 2 3 @@ -991,6 +1002,7 @@ end reverse(it::Cycle) = Cycle(reverse(it.xs)) last(it::Cycle) = last(it.xs) + # Repeated - repeat an object infinitely many times struct Repeated{O} @@ -1238,20 +1250,48 @@ flatten_length(f, T) = throw(ArgumentError( length(f::Flatten{I}) where {I} = flatten_length(f, eltype(I)) length(f::Flatten{Tuple{}}) = 0 -@propagate_inbounds function iterate(f::Flatten, state=()) - if state !== () - y = iterate(tail(state)...) - y !== nothing && return (y[1], (state[1], state[2], y[2])) +@propagate_inbounds function iterate(fl::Flatten) + it_result = iterate(fl.it) + it_result === nothing && return nothing + + inner_iterator, next_outer_state = it_result + inner_it_result = iterate(inner_iterator) + + while inner_it_result === nothing + it_result = iterate(fl.it, next_outer_state) + it_result === nothing && return nothing + + inner_iterator, next_outer_state = it_result + inner_it_result = iterate(inner_iterator) end - x = (state === () ? iterate(f.it) : iterate(f.it, state[1])) - x === nothing && return nothing - y = iterate(x[1]) - while y === nothing - x = iterate(f.it, x[2]) - x === nothing && return nothing - y = iterate(x[1]) + + item, next_inner_state = inner_it_result + return item, (next_outer_state, inner_iterator, next_inner_state) +end + +@propagate_inbounds function iterate(fl::Flatten, state) + next_outer_state, inner_iterator, next_inner_state = state + + # try to advance the inner iterator + inner_it_result = iterate(inner_iterator, next_inner_state) + if inner_it_result !== nothing + item, next_inner_state = inner_it_result + return item, (next_outer_state, inner_iterator, next_inner_state) + end + + # advance the outer iterator + while true + outer_it_result = iterate(fl.it, next_outer_state) + outer_it_result === nothing && return nothing + + inner_iterator, next_outer_state = outer_it_result + inner_it_result = iterate(inner_iterator) + + if inner_it_result !== nothing + item, next_inner_state = inner_it_result + return item, (next_outer_state, inner_iterator, next_inner_state) + end end - return y[1], (x[2], x[1], y[2]) end reverse(f::Flatten) = Flatten(reverse(itr) for itr in reverse(f.it)) @@ -1602,4 +1642,83 @@ end # be the same as the keys, so this is a valid optimization (see #51631) pairs(s::AbstractString) = IterableStatePairs(s) +""" + nth(itr, n::Integer) + +Get the `n`th element of an iterable collection. Throw a `BoundsError`[@ref] if not existing. +Will advance any `Stateful`[@ref] iterator. + +See also: [`first`](@ref), [`last`](@ref) + +# Examples +```jldoctest +julia> Iterators.nth(2:2:10, 4) +8 + +julia> Iterators.nth(reshape(1:30, (5,6)), 6) +6 + +julia> stateful = Iterators.Stateful(1:10); Iterators.nth(stateful, 7) +7 + +julia> first(stateful) +8 +``` +""" +nth(itr, n::Integer) = _nth(IteratorSize(itr), itr, n) +nth(itr::Cycle{I}, n::Integer) where I = _nth(IteratorSize(I), itr, n) +nth(itr::Flatten{Take{Repeated{O}}}, n::Integer) where O = _nth(IteratorSize(O), itr, n) +@propagate_inbounds nth(itr::AbstractArray, n::Integer) = itr[begin + n - 1] + +function _nth(::Union{HasShape, HasLength}, itr::Cycle{I}, n::Integer) where {I} + N = length(itr.xs) + N == 0 && throw(BoundsError(itr, n)) + + # prevents wrap around behaviour and inherit the error handling + return nth(itr.xs, n > 0 ? mod1(n, N) : n) +end + +# Flatten{Take{Repeated{O}}} is the actual type of an Iterators.cycle(iterable::O, m) iterator +function _nth(::Union{HasShape, HasLength}, itr::Flatten{Take{Repeated{O}}}, n::Integer) where {O} + cycles = itr.it.n + torepeat = itr.it.xs.x + k = length(torepeat) + (n > k*cycles || k == 0) && throw(BoundsError(itr, n)) + + # prevent wrap around behaviour and inherit the error handling + return nth(torepeat, n > 0 ? mod1(n, k) : n) +end + +function _nth(::IteratorSize, itr, n::Integer) + # unrolled version of `first(drop)` + n > 0 || throw(BoundsError(itr, n)) + y = iterate(itr) + for _ in 1:n-1 + y === nothing && break + y = iterate(itr, y[2]) + end + y === nothing && throw(BoundsError(itr, n)) + y[1] +end +""" + nth(n::Integer) + +Return a function that gets the `n`-th element from any iterator passed to it. +Equivalent to `Base.Fix2(nth, n)` or `itr -> nth(itr, n)`. + +See also: [`nth`](@ref), [`Base.Fix2`](@ref) +# Examples +```jldoctest +julia> fifth_element = Iterators.nth(5) +(::Base.Fix2{typeof(Base.Iterators.nth), Int64}) (generic function with 2 methods) + +julia> fifth_element(reshape(1:30, (5,6))) +5 + +julia> map(fifth_element, ("Willis", "Jovovich", "Oldman")) +('i', 'v', 'a') +``` +""" +nth(n::Integer) = Base.Fix2(nth, n) + end diff --git a/base/libc.jl b/base/libc.jl index 49e9d67590597..f575690c370d6 100644 --- a/base/libc.jl +++ b/base/libc.jl @@ -40,7 +40,7 @@ Base.cconvert(::Type{Cint}, fd::RawFD) = bitcast(Cint, fd) dup(src::RawFD[, target::RawFD])::RawFD Duplicate the file descriptor `src` so that the duplicate refers to the same OS -resource (e.g. a file or socket). A `target` file descriptor may be optionally +resource (e.g. a file or socket). A `target` file descriptor may optionally be passed to use for the new duplicate. """ dup(x::RawFD) = ccall((@static Sys.iswindows() ? :_dup : :dup), RawFD, (RawFD,), x) diff --git a/base/libdl.jl b/base/libdl.jl index 0b65ae8b381b8..73fd9d9c26871 100644 --- a/base/libdl.jl +++ b/base/libdl.jl @@ -5,7 +5,7 @@ module Libdl Interface to libdl. Provides dynamic linking support. """ Libdl -import Base.DL_LOAD_PATH +import Base: DL_LOAD_PATH, isdebugbuild export DL_LOAD_PATH, RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW, dlclose, dlopen, dlopen_e, dlsym, dlsym_e, @@ -341,7 +341,8 @@ Base.print(io::IO, llp::LazyLibraryPath) = print(io, string(llp)) # Helper to get `$(private_shlibdir)` at runtime struct PrivateShlibdirGetter; end const private_shlibdir = Base.OncePerProcess{String}() do - dirname(dlpath("libjulia-internal")) + libname = ifelse(isdebugbuild(), "libjulia-internal-debug", "libjulia-internal") + dirname(dlpath(libname)) end Base.string(::PrivateShlibdirGetter) = private_shlibdir() diff --git a/base/libuv.jl b/base/libuv.jl index 306854e9f4436..5e9bdfaf1e75c 100644 --- a/base/libuv.jl +++ b/base/libuv.jl @@ -39,8 +39,15 @@ macro handle_as(hand, typ) end end -associate_julia_struct(handle::Ptr{Cvoid}, @nospecialize(jlobj)) = +function _uv_hook_close end + +function associate_julia_struct(handle::Ptr{Cvoid}, jlobj::T) where T + # This `cfunction` is not used anywhere, but it triggers compilation of this + # MethodInstance for `--trim` so that it will be available when dispatched to + # by `jl_uv_call_close_callback()` + _ = @cfunction(Base._uv_hook_close, Cvoid, (Ref{T},)) ccall(:jl_uv_associate_julia_struct, Cvoid, (Ptr{Cvoid}, Any), handle, jlobj) +end disassociate_julia_struct(uv) = disassociate_julia_struct(uv.handle) disassociate_julia_struct(handle::Ptr{Cvoid}) = handle != C_NULL && ccall(:jl_uv_disassociate_julia_struct, Cvoid, (Ptr{Cvoid},), handle) @@ -52,14 +59,14 @@ iolock_end() = ccall(:jl_iolock_end, Cvoid, ()) # and should thus not be garbage collected const uvhandles = IdDict() const preserve_handle_lock = Threads.SpinLock() -function preserve_handle(x) +@nospecializeinfer function preserve_handle(@nospecialize(x)) lock(preserve_handle_lock) v = get(uvhandles, x, 0)::Int uvhandles[x] = v + 1 unlock(preserve_handle_lock) nothing end -function unpreserve_handle(x) +@nospecializeinfer function unpreserve_handle(@nospecialize(x)) lock(preserve_handle_lock) v = get(uvhandles, x, 0)::Int if v == 0 @@ -103,7 +110,8 @@ struverror(err::Int32) = unsafe_string(ccall(:uv_strerror, Cstring, (Int32,), er uverrorname(err::Int32) = unsafe_string(ccall(:uv_err_name, Cstring, (Int32,), err)) uv_error(prefix::Symbol, c::Integer) = uv_error(string(prefix), c) -uv_error(prefix::AbstractString, c::Integer) = c < 0 ? throw(_UVError(prefix, c)) : nothing +uv_error(prefix::AbstractString, c::Integer) = c < 0 ? _uv_error(prefix, c) : nothing +_uv_error(prefix::AbstractString, c::Integer) = throw(_UVError(prefix, c)) ## event loop ## diff --git a/base/linking.jl b/base/linking.jl index f3dbe6abba3ec..edb70e8061119 100644 --- a/base/linking.jl +++ b/base/linking.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license module Linking +import Base: isdebugbuild import Base.Libc: Libdl # from LLD_jll @@ -123,7 +124,6 @@ else "-shared" end -is_debug() = ccall(:jl_is_debugbuild, Cint, ()) == 1 libdir() = abspath(Sys.BINDIR, Base.LIBDIR) private_libdir() = abspath(Sys.BINDIR, Base.PRIVATE_LIBDIR) if Sys.iswindows() @@ -137,10 +137,18 @@ verbose_linking() = something(Base.get_bool_env("JULIA_VERBOSE_LINKING", false), function link_image_cmd(path, out) PRIVATE_LIBDIR = "-L$(private_libdir())" SHLIBDIR = "-L$(shlibdir())" - LIBS = is_debug() ? ("-ljulia-debug", "-ljulia-internal-debug") : + LIBS = isdebugbuild() ? ("-ljulia-debug", "-ljulia-internal-debug") : ("-ljulia", "-ljulia-internal") @static if Sys.iswindows() - LIBS = (LIBS..., "-lopenlibm", "-lssp", "-lgcc_s", "-lgcc", "-lmsvcrt") + LIBS = (LIBS..., "-lopenlibm", "-lgcc_s", "-lgcc", "-lmsvcrt") + if isdebugbuild() + LIBS = (LIBS..., "-lssp") + if isfile(joinpath(private_libdir(), "libmingwex.a")) + # In MinGW 11, the ssp implementation was moved from libssp to + # libmingwex with ssp only being a stub. See #59020. + LIBS = (LIBS..., "-lmingwex", "-lkernel32") + end + end end V = verbose_linking() ? "--verbose" : "" diff --git a/base/loading.jl b/base/loading.jl index c7397df8f4515..33799805e0601 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -184,8 +184,8 @@ end const slug_chars = String(['A':'Z'; 'a':'z'; '0':'9']) function slug(x::UInt32, p::Int) - y::UInt32 = x sprint(sizehint=p) do io + y = x n = length(slug_chars) for i = 1:p y, d = divrem(y, n) @@ -1061,6 +1061,7 @@ function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{No for (name, entries) in d entries = entries::Vector{Any} for entry in entries + entry = entry::Dict{String, Any} uuid = get(entry, "uuid", nothing)::Union{Nothing, String} extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}} if extensions !== nothing && haskey(extensions, pkg.name) && uuid !== nothing && uuid5(UUID(uuid), pkg.name) == pkg.uuid @@ -1243,21 +1244,7 @@ const TIMING_IMPORTS = Threads.Atomic{Int}(0) # these return either the array of modules loaded from the path / content given # or an Exception that describes why it couldn't be loaded # and it reconnects the Base.Docs.META -function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}, ignore_native::Union{Nothing,Bool}=nothing; register::Bool=true) - if isnothing(ignore_native) - if JLOptions().code_coverage == 0 && JLOptions().malloc_log == 0 - ignore_native = false - else - io = open(path, "r") - try - iszero(isvalid_cache_header(io)) && return ArgumentError("Incompatible header in cache file $path.") - _, (includes, _, _), _, _, _, _, _, _ = parse_cache_header(io, path) - ignore_native = pkg_tracked(includes) - finally - close(io) - end - end - end +function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}; register::Bool=true) assert_havelock(require_lock) timing_imports = TIMING_IMPORTS[] > 0 try @@ -1276,6 +1263,7 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No depmods[i] = dep end + ignore_native = false unlock(require_lock) # temporarily _unlock_ during these operations sv = try if ocachepath !== nothing @@ -1299,8 +1287,9 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No ext_edges = sv[4]::Union{Nothing,Vector{Any}} extext_methods = sv[5]::Vector{Any} internal_methods = sv[6]::Vector{Any} - StaticData.insert_backedges(edges, ext_edges, extext_methods, internal_methods) - + Compiler.@zone "CC: INSERT_BACKEDGES" begin + StaticData.insert_backedges(edges, ext_edges, extext_methods, internal_methods) + end restored = register_restored_modules(sv, pkg, path) for M in restored @@ -1949,44 +1938,16 @@ function _tryrequire_from_serialized(modkey::PkgId, build_id::UInt128) return ErrorException("Required dependency $modkey failed to load from a cache file.") end -# returns whether the package is tracked in coverage or malloc tracking based on -# JLOptions and includes -function pkg_tracked(includes) - if JLOptions().code_coverage == 0 && JLOptions().malloc_log == 0 - return false - elseif JLOptions().code_coverage == 1 || JLOptions().malloc_log == 1 # user - # Just say true. Pkgimages aren't in Base - return true - elseif JLOptions().code_coverage == 2 || JLOptions().malloc_log == 2 # all - return true - elseif JLOptions().code_coverage == 3 || JLOptions().malloc_log == 3 # tracked path - if JLOptions().tracked_path == C_NULL - return false - else - tracked_path = unsafe_string(JLOptions().tracked_path) - if isempty(tracked_path) - return false - else - return any(includes) do inc - startswith(inc.filename, tracked_path) - end - end - end - end -end - # loads a precompile cache file, ignoring stale_cachefile tests # load all dependent modules first function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}) assert_havelock(require_lock) local depmodnames io = open(path, "r") - ignore_native = false try iszero(isvalid_cache_header(io)) && return ArgumentError("Incompatible header in cache file $path.") _, (includes, _, _), depmodnames, _, _, _, clone_targets, _ = parse_cache_header(io, path) - ignore_native = pkg_tracked(includes) pkgimage = !isempty(clone_targets) if pkgimage @@ -2013,7 +1974,7 @@ function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union depmods[i] = dep end # then load the file - loaded = _include_from_serialized(pkg, path, ocachepath, depmods, ignore_native; register = true) + loaded = _include_from_serialized(pkg, path, ocachepath, depmods; register = true) return loaded end @@ -2258,7 +2219,7 @@ const include_callbacks = Any[] # used to optionally track dependencies when requiring a module: const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", because they are explicitly loaded, and the process should try to avoid invalidating them -const _require_dependencies = Any[] # a list of (mod, abspath, fsize, hash, mtime) tuples that are the file dependencies of the module currently being precompiled +const _require_dependencies = Any[] # a list of (mod::Module, abspath::String, fsize::UInt64, hash::UInt32, mtime::Float64) tuples that are the file dependencies of the module currently being precompiled const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies function _include_dependency(mod::Module, _path::AbstractString; track_content::Bool=true, @@ -2284,9 +2245,9 @@ function _include_dependency!(dep_list::Vector{Any}, track_dependencies::Bool, else @lock require_lock begin if track_content - hash = isdir(path) ? _crc32c(join(readdir(path))) : open(_crc32c, path, "r") + hash = (isdir(path) ? _crc32c(join(readdir(path))) : open(_crc32c, path, "r"))::UInt32 # use mtime=-1.0 here so that fsize==0 && mtime==0.0 corresponds to a missing include_dependency - push!(dep_list, (mod, path, filesize(path), hash, -1.0)) + push!(dep_list, (mod, path, UInt64(filesize(path)), hash, -1.0)) else push!(dep_list, (mod, path, UInt64(0), UInt32(0), mtime(path))) end @@ -2372,7 +2333,7 @@ function require(into::Module, mod::Symbol) if world == typemax(UInt) world = get_world_counter() end - return invoke_in_world(world, __require, into, mod) + return Compiler.@zone "LOAD_Require" invoke_in_world(world, __require, into, mod) end function check_for_hint(into, mod) @@ -2417,7 +2378,8 @@ function __require(into::Module, mod::Symbol) else manifest_warnings = collect_manifest_warnings() throw(ArgumentError(""" - Package $(where.name) does not have $mod in its dependencies: + Cannot load (`using/import`) module $mod into module $into in package $(where.name) + because package $(where.name) does not have $mod in its dependencies: $manifest_warnings- You may have a partially installed environment. Try `Pkg.instantiate()` to ensure all packages in the environment are installed. - Or, if you have $(where.name) checked out for development and have @@ -2663,8 +2625,7 @@ function __require_prelocked(pkg::PkgId, env) if JLOptions().use_compiled_modules == 1 if !generating_output(#=incremental=#false) project = active_project() - if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) && project !== nothing && - isfile(project) && project_file_manifest_path(project) !== nothing + if !generating_output() && !parallel_precompile_attempted && !disable_parallel_precompile && @isdefined(Precompilation) parallel_precompile_attempted = true unlock(require_lock) try @@ -2755,7 +2716,7 @@ end # load a serialized file directly from append_bundled_depot_path for uuidkey without stalechecks """ - require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}=nothing) + require_stdlib(package_uuidkey::PkgId, [ext::String, from::Module]) !!! warning "May load duplicate copies of stdlib packages." @@ -2794,7 +2755,8 @@ end [1] https://github.com/JuliaLang/Pkg.jl/issues/4017#issuecomment-2377589989 [2] https://github.com/JuliaLang/StyledStrings.jl/issues/91#issuecomment-2379602914 """ -function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}=nothing) +require_stdlib(package_uuidkey::PkgId) = require_stdlib(package_uuidkey, nothing, Base) +function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}, from::Module) if generating_output(#=incremental=#true) # Otherwise this would lead to awkward dependency issues by loading a package that isn't in the Project/Manifest error("This interactive function requires a stdlib to be loaded, and package code should instead use it directly from that stdlib.") @@ -2806,15 +2768,29 @@ function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}=noth newm = start_loading(this_uuidkey, UInt128(0), true) newm === nothing || return newm try - # first since this is a stdlib, try to look there directly first - if ext === nothing - sourcepath = normpath(env, this_uuidkey.name, "src", this_uuidkey.name * ".jl") - else - sourcepath = find_ext_path(normpath(joinpath(env, package_uuidkey.name)), ext) - end depot_path = append_bundled_depot_path!(empty(DEPOT_PATH)) - set_pkgorigin_version_path(this_uuidkey, sourcepath) - newm = _require_search_from_serialized(this_uuidkey, sourcepath, UInt128(0), false; DEPOT_PATH=depot_path) + from_stdlib = true # set to false if `from` is a normal package so we do not want the internal loader for the extension either + if ext isa String + from_uuid = PkgId(from) + from_m = get(loaded_modules, from_uuid, nothing) + if from_m === from + # if from_uuid is either nothing or points to something else, assume we should use require_stdlib + # otherwise check cachepath for from to see if it looks like it is from depot_path, since try_build_ids + cachepath = get(PkgOrigin, pkgorigins, from_uuid).cachepath + entrypath, entryfile = cache_file_entry(from_uuid) + from_stdlib = any(x -> startswith(entrypath, x), depot_path) + end + end + if from_stdlib + # first since this is a stdlib, try to look there directly first + if ext === nothing + sourcepath = normpath(env, this_uuidkey.name, "src", this_uuidkey.name * ".jl") + else + sourcepath = find_ext_path(normpath(joinpath(env, package_uuidkey.name)), ext) + end + set_pkgorigin_version_path(this_uuidkey, sourcepath) + newm = _require_search_from_serialized(this_uuidkey, sourcepath, UInt128(0), false; DEPOT_PATH=depot_path) + end finally end_loading(this_uuidkey, newm) end @@ -2824,10 +2800,12 @@ function require_stdlib(package_uuidkey::PkgId, ext::Union{Nothing, String}=noth run_package_callbacks(this_uuidkey) else # if the user deleted their bundled depot, next try to load it completely normally + # if it is an extension, we first need to indicate where to find its parant via EXT_PRIMED + ext isa String && (EXT_PRIMED[this_uuidkey] = PkgId[package_uuidkey]) newm = _require_prelocked(this_uuidkey) end return newm - end + end # release lock end # relative-path load @@ -2849,7 +2827,11 @@ function include_string(mapexpr::Function, mod::Module, code::AbstractString, loc = LineNumberNode(1, Symbol(filename)) try ast = Meta.parseall(code, filename=filename) - @assert Meta.isexpr(ast, :toplevel) + if !Meta.isexpr(ast, :toplevel) + @assert Core._lower != fl_lower + # Only reached when JuliaLowering and alternate parse functions are activated + return Core.eval(mod, ast) + end result = nothing line_and_ex = Expr(:toplevel, loc, nothing) for ex in ast.args @@ -3365,7 +3347,7 @@ mutable struct CacheHeaderIncludes const modpath::Vector{String} # seemingly not needed in Base, but used by Revise end -function CacheHeaderIncludes(dep_tuple::Tuple{Module, String, Int64, UInt32, Float64}) +function CacheHeaderIncludes(dep_tuple::Tuple{Module, String, UInt64, UInt32, Float64}) return CacheHeaderIncludes(PkgId(dep_tuple[1]), dep_tuple[2:end]..., String[]) end @@ -4206,6 +4188,20 @@ function expand_compiler_path(tup) end compiler_chi(tup::Tuple) = CacheHeaderIncludes(expand_compiler_path(tup)) +""" + isprecompilable(f, argtypes::Tuple{Vararg{Any}}) + +Check, as far as is possible without actually compiling, if the given +function `f` can be compiled for the argument tuple (of types) `argtypes`. +""" +function isprecompilable(@nospecialize(f), @nospecialize(argtypes::Tuple)) + isprecompilable(Tuple{Core.Typeof(f), argtypes...}) +end + +function isprecompilable(@nospecialize(argt::Type)) + ccall(:jl_is_compilable, Int32, (Any,), argt) != 0 +end + """ precompile(f, argtypes::Tuple{Vararg{Any}}) diff --git a/base/lock.jl b/base/lock.jl index 40bc9e08bd9b0..461cb2b12a807 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -398,7 +398,7 @@ macro lock_nofail(l, expr) end """ - Lockable(value, lock = ReentrantLock()) + Lockable(value, lock = ReentrantLock()) Creates a `Lockable` object that wraps `value` and associates it with the provided `lock`. This object @@ -431,7 +431,7 @@ Lockable(value) = Lockable(value, ReentrantLock()) getindex(l::Lockable) = (assert_havelock(l.lock); l.value) """ - lock(f::Function, l::Lockable) + lock(f::Function, l::Lockable) Acquire the lock associated with `l`, execute `f` with the lock held, and release the lock when `f` returns. `f` will receive one positional @@ -561,6 +561,36 @@ function acquire(f, s::Semaphore) end end +""" + Base.@acquire s::Semaphore expr + +Macro version of `Base.acquire(f, s::Semaphore)` but with `expr` instead of `f` function. +Expands to: +```julia +Base.acquire(s) +try + expr +finally + Base.release(s) +end +``` +This is similar to using [`acquire`](@ref) with a `do` block, but avoids creating a closure. + +!!! compat "Julia 1.13" + `Base.@acquire` was added in Julia 1.13 +""" +macro acquire(s, expr) + quote + local temp = $(esc(s)) + Base.acquire(temp) + try + $(esc(expr)) + finally + Base.release(temp) + end + end +end + """ release(s::Semaphore) @@ -674,6 +704,9 @@ calls in the same process will return exactly the same value. This is useful in code that will be precompiled, as it allows setting up caches or other state which won't get serialized. +!!! compat "Julia 1.12" + This type requires Julia 1.12 or later. + ## Example ```jldoctest @@ -705,7 +738,9 @@ mutable struct OncePerProcess{T, F} <: Function return once end end +OncePerProcess{T}(initializer::Type{U}) where {T, U} = OncePerProcess{T, Type{U}}(initializer) OncePerProcess{T}(initializer::F) where {T, F} = OncePerProcess{T, F}(initializer) +OncePerProcess(initializer::Type{U}) where U = OncePerProcess{Base.promote_op(initializer), Type{U}}(initializer) OncePerProcess(initializer) = OncePerProcess{Base.promote_op(initializer), typeof(initializer)}(initializer) @inline function (once::OncePerProcess{T,F})() where {T,F} state = (@atomic :acquire once.state) @@ -782,6 +817,9 @@ if that behavior is correct within your library's threading-safety design. See also: [`OncePerTask`](@ref). +!!! compat "Julia 1.12" + This type requires Julia 1.12 or later. + ## Example ```jldoctest @@ -812,7 +850,9 @@ mutable struct OncePerThread{T, F} <: Function return once end end +OncePerThread{T}(initializer::Type{U}) where {T, U} = OncePerThread{T,Type{U}}(initializer) OncePerThread{T}(initializer::F) where {T, F} = OncePerThread{T,F}(initializer) +OncePerThread(initializer::Type{U}) where U = OncePerThread{Base.promote_op(initializer), Type{U}}(initializer) OncePerThread(initializer) = OncePerThread{Base.promote_op(initializer), typeof(initializer)}(initializer) @inline (once::OncePerThread{T,F})() where {T,F} = once[Threads.threadid()] @inline function getindex(once::OncePerThread{T,F}, tid::Integer) where {T,F} @@ -908,6 +948,9 @@ exactly once per Task. All future calls in the same Task will return exactly the See also: [`task_local_storage`](@ref). +!!! compat "Julia 1.12" + This type requires Julia 1.12 or later. + ## Example ```jldoctest @@ -931,8 +974,10 @@ false mutable struct OncePerTask{T, F} <: Function const initializer::F + OncePerTask{T}(initializer::Type{U}) where {T, U} = new{T,Type{U}}(initializer) OncePerTask{T}(initializer::F) where {T, F} = new{T,F}(initializer) OncePerTask{T,F}(initializer::F) where {T, F} = new{T,F}(initializer) + OncePerTask(initializer::Type{U}) where U = new{Base.promote_op(initializer), Type{U}}(initializer) OncePerTask(initializer) = new{Base.promote_op(initializer), typeof(initializer)}(initializer) end @inline function (once::OncePerTask{T,F})() where {T,F} diff --git a/base/locks-mt.jl b/base/locks-mt.jl index 5d355b9ed200c..237e0d9856996 100644 --- a/base/locks-mt.jl +++ b/base/locks-mt.jl @@ -3,7 +3,7 @@ import .Base: unsafe_convert, lock, trylock, unlock, islocked, wait, notify, AbstractLock export SpinLock - +public PaddedSpinLock # Important Note: these low-level primitives defined here # are typically not for general usage @@ -12,33 +12,68 @@ export SpinLock ########################################## """ - SpinLock() + abstract type AbstractSpinLock <: AbstractLock end -Create a non-reentrant, test-and-test-and-set spin lock. +A non-reentrant, test-and-test-and-set spin lock. Recursive use will result in a deadlock. This kind of lock should only be used around code that takes little time to execute and does not block (e.g. perform I/O). In general, [`ReentrantLock`](@ref) should be used instead. Each [`lock`](@ref) must be matched with an [`unlock`](@ref). -If [`!islocked(lck::SpinLock)`](@ref islocked) holds, [`trylock(lck)`](@ref trylock) +If [`!islocked(lck::AbstractSpinLock)`](@ref islocked) holds, [`trylock(lck)`](@ref trylock) succeeds unless there are other tasks attempting to hold the lock "at the same time." Test-and-test-and-set spin locks are quickest up to about 30ish contending threads. If you have more contention than that, different synchronization approaches should be considered. """ -mutable struct SpinLock <: AbstractLock +abstract type AbstractSpinLock <: AbstractLock end + +""" + SpinLock() <: AbstractSpinLock + +Spinlocks are not padded, and so may suffer from false sharing. +See also [`PaddedSpinLock`](@ref). + +See the documentation for [`AbstractSpinLock`](@ref) regarding correct usage. +""" +mutable struct SpinLock <: AbstractSpinLock # we make this much larger than necessary to minimize false-sharing @atomic owned::Int SpinLock() = new(0) end +# TODO: Determine the cache line size using e.g., CPUID. Meanwhile, this is correct for most +# processors. +const CACHE_LINE_SIZE = 64 + +""" + PaddedSpinLock() <: AbstractSpinLock + +PaddedSpinLocks are padded so that each is guaranteed to be on its own cache line, to avoid +false sharing. +See also [`SpinLock`](@ref). + +See the documentation for [`AbstractSpinLock`](@ref) regarding correct usage. +""" +mutable struct PaddedSpinLock <: AbstractSpinLock + # we make this much larger than necessary to minimize false-sharing + _padding_before::NTuple{max(0, CACHE_LINE_SIZE - sizeof(Int)), UInt8} + @atomic owned::Int + _padding_after::NTuple{max(0, CACHE_LINE_SIZE - sizeof(Int)), UInt8} + function PaddedSpinLock() + l = new() + @atomic l.owned = 0 + return l + end +end + # Note: this cannot assert that the lock is held by the correct thread, because we do not # track which thread locked it. Users beware. -Base.assert_havelock(l::SpinLock) = islocked(l) ? nothing : Base.concurrency_violation() +Base.assert_havelock(l::AbstractSpinLock) = islocked(l) ? nothing : Base.concurrency_violation() -function lock(l::SpinLock) +function lock(l::AbstractSpinLock) while true if @inline trylock(l) return @@ -49,7 +84,7 @@ function lock(l::SpinLock) end end -function trylock(l::SpinLock) +function trylock(l::AbstractSpinLock) if l.owned == 0 GC.disable_finalizers() p = @atomicswap :acquire l.owned = 1 @@ -61,7 +96,7 @@ function trylock(l::SpinLock) return false end -function unlock(l::SpinLock) +function unlock(l::AbstractSpinLock) if (@atomicswap :release l.owned = 0) == 0 error("unlock count must match lock count") end @@ -70,6 +105,6 @@ function unlock(l::SpinLock) return end -function islocked(l::SpinLock) +function islocked(l::AbstractSpinLock) return (@atomic :monotonic l.owned) != 0 end diff --git a/base/logging/ConsoleLogger.jl b/base/logging/ConsoleLogger.jl index 818b2272b773c..6521bf49b3e66 100644 --- a/base/logging/ConsoleLogger.jl +++ b/base/logging/ConsoleLogger.jl @@ -9,6 +9,9 @@ interactive work with the Julia REPL. Log levels less than `min_level` are filtered out. +This Logger is thread-safe, with locks for both orchestration of message +limits i.e. `maxlog`, and writes to the stream. + Message formatting can be controlled by setting keyword arguments: * `meta_formatter` is a function which takes the log event metadata @@ -24,6 +27,7 @@ Message formatting can be controlled by setting keyword arguments: """ struct ConsoleLogger <: AbstractLogger stream::IO + lock::ReentrantLock # do not log within this lock min_level::LogLevel meta_formatter show_limited::Bool @@ -33,19 +37,19 @@ end function ConsoleLogger(stream::IO, min_level=Info; meta_formatter=default_metafmt, show_limited=true, right_justify=0) - ConsoleLogger(stream, min_level, meta_formatter, + ConsoleLogger(stream, ReentrantLock(), min_level, meta_formatter, show_limited, right_justify, Dict{Any,Int}()) end function ConsoleLogger(min_level=Info; meta_formatter=default_metafmt, show_limited=true, right_justify=0) - ConsoleLogger(closed_stream, min_level, meta_formatter, + ConsoleLogger(closed_stream, ReentrantLock(), min_level, meta_formatter, show_limited, right_justify, Dict{Any,Int}()) end shouldlog(logger::ConsoleLogger, level, _module, group, id) = - get(logger.message_limits, id, 1) > 0 + @lock logger.lock get(logger.message_limits, id, 1) > 0 min_enabled_level(logger::ConsoleLogger) = logger.min_level @@ -109,9 +113,11 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module hasmaxlog = haskey(kwargs, :maxlog) ? 1 : 0 maxlog = get(kwargs, :maxlog, nothing) if maxlog isa Core.BuiltinInts - remaining = get!(logger.message_limits, id, Int(maxlog)::Int) - logger.message_limits[id] = remaining - 1 - remaining > 0 || return + @lock logger.lock begin + remaining = get!(logger.message_limits, id, Int(maxlog)::Int) + remaining == 0 && return + logger.message_limits[id] = remaining - 1 + end end # Generate a text representation of the message and all key value pairs, @@ -141,7 +147,7 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module for (key, val) in kwargs key === :maxlog && continue showvalue(valio, val) - vallines = split(String(take!(valbuf)), '\n') + vallines = split(takestring!(valbuf), '\n') if length(vallines) == 1 push!(msglines, (indent=2, msg=SubString("$key = $(vallines[1])"))) else @@ -184,6 +190,7 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module println(iob) end - write(stream, take!(buf)) + b = take!(buf) + @lock logger.lock write(stream, b) nothing end diff --git a/base/logging/logging.jl b/base/logging/logging.jl index a9e41777f29b8..25f4dbe4902be 100644 --- a/base/logging/logging.jl +++ b/base/logging/logging.jl @@ -132,6 +132,7 @@ isless(a::LogLevel, b::LogLevel) = isless(a.level, b.level) +(level::LogLevel, inc::Integer) = LogLevel(level.level+inc) -(level::LogLevel, inc::Integer) = LogLevel(level.level-inc) convert(::Type{LogLevel}, level::Integer) = LogLevel(level) +convert(::Type{Int32}, level::LogLevel) = level.level """ BelowMinLevel @@ -171,7 +172,8 @@ Alias for [`LogLevel(1_000_001)`](@ref LogLevel). const AboveMaxLevel = LogLevel( 1000001) # Global log limiting mechanism for super fast but inflexible global log limiting. -const _min_enabled_level = Ref{LogLevel}(Debug) +# Atomic ensures that the value is always consistent across threads. +const _min_enabled_level = Threads.Atomic{Int32}(Debug) function show(io::IO, level::LogLevel) if level == BelowMinLevel print(io, "BelowMinLevel") @@ -394,7 +396,7 @@ function logmsg_code(_module, file, line, level, message, exs...) level = $level # simplify std_level code emitted, if we know it is one of our global constants std_level = $(level isa Symbol ? :level : :(level isa $LogLevel ? level : convert($LogLevel, level)::$LogLevel)) - if std_level >= $(_min_enabled_level)[] + if std_level.level >= $(_min_enabled_level)[] group = $(log_data._group) _module = $(log_data._module) logger = $(current_logger_for_env)(std_level, group, _module) @@ -409,9 +411,13 @@ function logmsg_code(_module, file, line, level, message, exs...) end line = $(log_data._line) local msg, kwargs - $(logrecord) && $handle_message_nothrow( - logger, level, msg, _module, group, id, file, line; - kwargs...) + if $(logrecord) + @assert @isdefined(msg) "Assertion to tell the compiler about the definedness of this variable" + @assert @isdefined(kwargs) "Assertion to tell the compiler about the definedness of this variable" + $handle_message_nothrow( + logger, level, msg, _module, group, id, file, line; + kwargs...) + end end end end @@ -541,7 +547,8 @@ with_logstate(f::Function, logstate) = @with(CURRENT_LOGSTATE => logstate, f()) Disable all log messages at log levels equal to or less than `level`. This is a *global* setting, intended to make debug logging extremely cheap when -disabled. +disabled. Note that this cannot be used to enable logging that is currently disabled +by other mechanisms. # Examples ```julia @@ -663,17 +670,21 @@ close(closed_stream) Simplistic logger for logging all messages with level greater than or equal to `min_level` to `stream`. If stream is closed then messages with log level greater or equal to `Warn` will be logged to `stderr` and below to `stdout`. + +This Logger is thread-safe, with a lock taken around orchestration of message +limits i.e. `maxlog`, and writes to the stream. """ struct SimpleLogger <: AbstractLogger stream::IO + lock::ReentrantLock min_level::LogLevel message_limits::Dict{Any,Int} end -SimpleLogger(stream::IO, level=Info) = SimpleLogger(stream, level, Dict{Any,Int}()) +SimpleLogger(stream::IO, level=Info) = SimpleLogger(stream, ReentrantLock(), level, Dict{Any,Int}()) SimpleLogger(level=Info) = SimpleLogger(closed_stream, level) shouldlog(logger::SimpleLogger, level, _module, group, id) = - get(logger.message_limits, id, 1) > 0 + @lock logger.lock get(logger.message_limits, id, 1) > 0 min_enabled_level(logger::SimpleLogger) = logger.min_level @@ -684,15 +695,14 @@ function handle_message(logger::SimpleLogger, level::LogLevel, message, _module, @nospecialize maxlog = get(kwargs, :maxlog, nothing) if maxlog isa Core.BuiltinInts - remaining = get!(logger.message_limits, id, Int(maxlog)::Int) - logger.message_limits[id] = remaining - 1 - remaining > 0 || return + @lock logger.lock begin + remaining = get!(logger.message_limits, id, Int(maxlog)::Int) + remaining == 0 && return + logger.message_limits[id] = remaining - 1 + end end buf = IOBuffer() stream::IO = logger.stream - if !(isopen(stream)::Bool) - stream = stderr - end iob = IOContext(buf, stream) levelstr = level == Warn ? "Warning" : string(level) msglines = eachsplit(chomp(convert(String, string(message))::String), '\n') @@ -706,7 +716,13 @@ function handle_message(logger::SimpleLogger, level::LogLevel, message, _module, println(iob, "│ ", key, " = ", val) end println(iob, "└ @ ", _module, " ", filepath, ":", line) - write(stream, take!(buf)) + b = take!(buf) + @lock logger.lock begin + if !(isopen(stream)::Bool) + stream = stderr + end + write(stream, b) + end nothing end diff --git a/base/math.jl b/base/math.jl index b415d60e9bb27..d6efadf56eccd 100644 --- a/base/math.jl +++ b/base/math.jl @@ -228,6 +228,27 @@ end return hi, lo end +# generic, but involves double rounding +function _180_over_pi(z::AbstractFloat) + 180 / oftype(z, pi) +end +function _pi_over_180(z::AbstractFloat) + oftype(z, pi) / 180 +end + +# rounded to closest representable number where necessary +function _180_over_pi(z::Union{Float16, Float32}) + if z isa Float16 + r = Float16(57.28) + elseif z isa Float32 + r = 57.29578f0 + end + r +end +function _pi_over_180(::Float16) + Float16(0.01746) +end + """ rad2deg(x) @@ -241,7 +262,7 @@ julia> rad2deg(pi) 180.0 ``` """ -rad2deg(z::AbstractFloat) = z * (180 / oftype(z, pi)) +rad2deg(z::AbstractFloat) = z * _180_over_pi(z) """ deg2rad(x) @@ -256,7 +277,7 @@ julia> deg2rad(90) 1.5707963267948966 ``` """ -deg2rad(z::AbstractFloat) = z * (oftype(z, pi) / 180) +deg2rad(z::AbstractFloat) = z * _pi_over_180(z) rad2deg(z::Real) = rad2deg(float(z)) deg2rad(z::Real) = deg2rad(float(z)) rad2deg(z::Number) = (z/pi)*180 @@ -482,7 +503,7 @@ asin(x::Number) """ acos(x::T) where {T <: Number} -> float(T) -Compute the inverse cosine of `x`, where the output is in radians +Compute the inverse cosine of `x`, where the output is in radians. Return a `T(NaN)` if `isnan(x)`. """ diff --git a/base/meta.jl b/base/meta.jl index 4807b910c494a..5d880a7442b3e 100644 --- a/base/meta.jl +++ b/base/meta.jl @@ -160,7 +160,7 @@ Takes the expression `x` and returns an equivalent expression in lowered form for executing in module `m`. See also [`code_lowered`](@ref). """ -lower(m::Module, @nospecialize(x)) = ccall(:jl_expand, Any, (Any, Any), x, m) +lower(m::Module, @nospecialize(x)) = Core._lower(x, m, "none", 0, typemax(Csize_t), false)[1] """ @lower [m] x diff --git a/base/methodshow.jl b/base/methodshow.jl index 7fdefc9b7311f..1470303a01bbc 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -78,10 +78,10 @@ end # NOTE: second argument is deprecated and is no longer used function kwarg_decl(m::Method, kwtype = nothing) - if m.sig !== Tuple # OpaqueClosure or Builtin + if !(m.sig === Tuple || m.sig <: Tuple{Core.Builtin, Vararg}) # OpaqueClosure or Builtin kwtype = typeof(Core.kwcall) sig = rewrap_unionall(Tuple{kwtype, NamedTuple, (unwrap_unionall(m.sig)::DataType).parameters...}, m.sig) - kwli = ccall(:jl_methtable_lookup, Any, (Any, Any, UInt), kwtype.name.mt, sig, get_world_counter()) + kwli = ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, get_world_counter()) if kwli !== nothing kwli = kwli::Method slotnames = ccall(:jl_uncompress_argnames, Vector{Symbol}, (Any,), kwli.slot_syms) @@ -181,6 +181,7 @@ end Return a tuple `(filename,line)` giving the location of a generic `Function` definition. """ functionloc(@nospecialize(f), @nospecialize(types)) = functionloc(which(f,types)) +functionloc(@nospecialize(argtypes::Union{Tuple, Type{<:Tuple}})) = functionloc(which(argtypes)) function functionloc(@nospecialize(f)) mt = methods(f) @@ -214,10 +215,11 @@ show(io::IO, m::Method; kwargs...) = show_method(IOContext(io, :compact=>true), show(io::IO, ::MIME"text/plain", m::Method; kwargs...) = show_method(io, m; kwargs...) -function show_method(io::IO, m::Method; modulecolor = :light_black, digit_align_width = 1) +function show_method(io::IO, m::Method; + modulecolor = :light_black, digit_align_width = 1, + print_signature_only::Bool = get(io, :print_method_signature_only, false)::Bool) tv, decls, file, line = arg_decl_parts(m) - sig = unwrap_unionall(m.sig) - if sig === Tuple + if m.sig <: Tuple{Core.Builtin, Vararg} # Builtin print(io, m.name, "(...)") file = "none" @@ -250,19 +252,21 @@ function show_method(io::IO, m::Method; modulecolor = :light_black, digit_align_ show_method_params(io, tv) end - if !(get(io, :compact, false)::Bool) # single-line mode - println(io) - digit_align_width += 4 + if !print_signature_only + if !(get(io, :compact, false)::Bool) # single-line mode + println(io) + digit_align_width += 4 + end + # module & file, re-using function from errorshow.jl + print_module_path_file(io, parentmodule(m), string(file), line; modulecolor, digit_align_width) end - # module & file, re-using function from errorshow.jl - print_module_path_file(io, parentmodule(m), string(file), line; modulecolor, digit_align_width) end function show_method_list_header(io::IO, ms::MethodList, namefmt::Function) - mt = ms.mt - name = mt.name - hasname = isdefined(mt.module, name) && - typeof(getfield(mt.module, name)) <: Function + tn = ms.tn + name = tn.singletonname + hasname = isdefined(tn.module, name) && + typeof(getfield(tn.module, name)) <: Function n = length(ms) m = n==1 ? "method" : "methods" print(io, "# $n $m") @@ -271,18 +275,18 @@ function show_method_list_header(io::IO, ms::MethodList, namefmt::Function) if hasname what = (startswith(sname, '@') ? "macro" - : mt.module === Core && mt.defs isa Core.TypeMapEntry && (mt.defs.func::Method).sig === Tuple ? + : tn.module === Core && tn.wrapper <: Core.Builtin ? "builtin function" : # else "generic function") print(io, " for ", what, " ", namedisplay, " from ") - col = get!(() -> popfirst!(STACKTRACE_MODULECOLORS), STACKTRACE_FIXEDCOLORS, parentmodule_before_main(ms.mt.module)) + col = get!(() -> popfirst!(STACKTRACE_MODULECOLORS), STACKTRACE_FIXEDCOLORS, parentmodule_before_main(tn.module)) - printstyled(io, ms.mt.module, color=col) + printstyled(io, tn.module, color=col) elseif '#' in sname print(io, " for anonymous function ", namedisplay) - elseif mt === _TYPE_NAME.mt + elseif tn === _TYPE_NAME || iskindtype(tn.wrapper) print(io, " for type constructor") else print(io, " for callable object") @@ -293,6 +297,8 @@ end # Determine the `modulecolor` value to pass to `show_method` function _modulecolor(method::Method) mmt = get_methodtable(method) + # TODO: this looks like a buggy bit of internal hacking, so disable for now + return nothing if mmt === nothing || mmt.module === parentmodule(method) return nothing end @@ -314,10 +320,10 @@ function _modulecolor(method::Method) end function show_method_table(io::IO, ms::MethodList, max::Int=-1, header::Bool=true) - mt = ms.mt - name = mt.name - hasname = isdefined(mt.module, name) && - typeof(getfield(mt.module, name)) <: Function + tn = ms.tn + name = tn.singletonname + hasname = isdefined(tn.module, name) && + typeof(getfield(tn.module, name)) <: Function if header show_method_list_header(io, ms, str -> "\""*str*"\"") end @@ -363,7 +369,7 @@ end show(io::IO, ms::MethodList) = show_method_table(io, ms) show(io::IO, ::MIME"text/plain", ms::MethodList) = show_method_table(io, ms) -show(io::IO, mt::Core.MethodTable) = show_method_table(io, MethodList(mt)) +show(io::IO, mt::Core.MethodTable) = print(io, mt.module, ".", mt.name, " is a Core.MethodTable with ", length(mt), " methods.") function inbase(m::Module) if m == Base @@ -418,8 +424,7 @@ end function show(io::IO, ::MIME"text/html", m::Method) tv, decls, file, line = arg_decl_parts(m, true) sig = unwrap_unionall(m.sig) - if sig === Tuple - # Builtin + if sig <: Tuple{Core.Builtin, Vararg} print(io, m.name, "(...) in ", parentmodule(m)) return end @@ -458,7 +463,6 @@ function show(io::IO, ::MIME"text/html", m::Method) end function show(io::IO, mime::MIME"text/html", ms::MethodList) - mt = ms.mt show_method_list_header(io, ms, str -> ""*str*"") print(io, "
    ") for meth in ms @@ -469,8 +473,6 @@ function show(io::IO, mime::MIME"text/html", ms::MethodList) print(io, "
") end -show(io::IO, mime::MIME"text/html", mt::Core.MethodTable) = show(io, mime, MethodList(mt)) - # pretty-printing of AbstractVector{Method} function show(io::IO, mime::MIME"text/plain", mt::AbstractVector{Method}) last_shown_line_infos = get(io, :last_shown_line_infos, nothing) diff --git a/base/missing.jl b/base/missing.jl index 6a8c09dc02aff..5fc0e6ec9e620 100644 --- a/base/missing.jl +++ b/base/missing.jl @@ -86,6 +86,8 @@ isequal(::Any, ::Missing) = false isless(::Missing, ::Missing) = false isless(::Missing, ::Any) = false isless(::Any, ::Missing) = true +ispositive(::Missing) = missing +isnegative(::Missing) = missing isapprox(::Missing, ::Missing; kwargs...) = missing isapprox(::Missing, ::Any; kwargs...) = missing isapprox(::Any, ::Missing; kwargs...) = missing diff --git a/base/module.jl b/base/module.jl new file mode 100644 index 0000000000000..c21414179446e --- /dev/null +++ b/base/module.jl @@ -0,0 +1,143 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Full-featured versions of _eval_import and _eval_using + +for m in methods(_eval_import) + delete_method(m) +end +for m in methods(_eval_using) + delete_method(m) +end + +function eval_import_path(at::Module, from::Union{Module, Nothing}, path::Expr, keyword::String) + isempty(path.args) && error("malformed import statement") + + i::Int = 1 + function next!() + local v + i <= length(path.args) || error("invalid module path") + v = path.args[i] + i += 1 + v isa Symbol || throw(TypeError(Symbol(keyword), "", Symbol, v)) + v + end + v = next!() + m = nothing + + if from !== nothing + m = from + elseif v !== :. + # `A.B`: call the loader to obtain the root A in the current environment. + if v === :Core + m = Core + elseif v === :Base + m = Base + else + m = require(at, v) + m isa Module || error("failed to load module $v") + end + i > lastindex(path.args) && return m, nothing + v = next!() + else + # `.A.B.C`: strip off leading dots by following parent links + m = at + while (v = next!()) === :. + m = parentmodule(m) + end + end + + while true + v === :. && error("invalid $keyword path: \".\" in identifier path") + i > lastindex(path.args) && break + m = getglobal(m, v) + m isa Module || error("invalid $keyword path: \"$v\" does not name a module") + v = next!() + end + m, v +end + +function eval_import_path_all(at::Module, path::Expr, keyword::String) + m, v = eval_import_path(at, nothing, path, keyword) + if v !== nothing + m = getglobal(m, v) + m isa Module || error("invalid $keyword path: \"$v\" does not name a module") + end + m +end + +function check_macro_rename(from::Symbol, to::Symbol, keyword::String) + c1(sym) = bitcast(Char, UInt32(unsafe_load(unsafe_convert(Ptr{UInt8}, sym))) << 24) + from_c, to_c = c1(from), c1(to) + if from_c == '@' && to_c != '@' + error("cannot rename macro \"$from\" to non-macro \"$to\" in \"$keyword\"") + end + if from_c != '@' && to_c == '@' + error("cannot rename non-macro \"$from\" to macro \"$to\" in \"$keyword\"") + end +end + +""" + _eval_import(imported::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...) + +Evaluate the import paths, calling `Core._import` for each name to be imported. +`imported` imports are created with `import`, `using A: x` sets this to false. +The `from` is the part of the import path before the `:`. This is the lowered +form of `import`, `import ...:`, and `using ...:`. + +``` +import A => _eval_import(true, Main, nothing, Expr(:., :A)) +import A.b => _eval_import(true, Main, nothing, Expr(:., :A, :b)) +import A.b as c => _eval_import(true, Main, nothing, Expr(:as, Expr(:., :A, :b), :c)) +import A.B: C.d, e => _eval_import(true, Main, Expr(:., :A, :B), Expr(:., :C, :d), Expr(:., :e)) +import A.B: C.d as e => _eval_import(true, Main, Expr(:., :A, :B), Expr(:as, Expr(:., :C, :d), :e)) +using A.B: C.d, e => _eval_import(false, Main, Expr(:., :A, :B), Expr(:., :C, :d), Expr(:., :e)) + +See also [`_import`](@ref Core._import). +``` +""" +function _eval_import(imported::Bool, to::Module, from::Union{Expr, Nothing}, paths::Expr...) + keyword = imported ? "import" : "using" + fail() = error("malformed \"$keyword\" statement") + from = from !== nothing ? eval_import_path_all(to, from, keyword) : nothing + + for path in paths + path isa Expr || fail() + asname = nothing + if path.head === :as && length(path.args) == 2 + path, asname = path.args + elseif path.head !== :. + fail() + end + m, name = eval_import_path(to, from, path, keyword) + + if name !== nothing + asname = asname === nothing ? name : asname + check_macro_rename(name, asname, keyword) + Core._import(to, m, asname, name, imported) + else + Core._import(to, m, asname === nothing ? nameof(m) : asname) + end + end +end + +""" + _eval_using(to::Module, path::Expr) + +Evaluate the import path to a module and call [`Core._using`](@ref) on it, +making its exports available to the `to` module; this is the lowered form of +`using A`. + +``` +using A.B => _module_using(Main, Expr(:., :A, :B)) +``` + +See also [`_using`](@ref Core._using). +""" +function _eval_using(to::Module, path::Expr) + from = eval_import_path_all(to, path, "using") + Core._using(to, from) + is_package = length(path.args) == 1 && path.args[1] !== :. + if to == Main && is_package + Core._import(to, from, nameof(from)) + end +end diff --git a/base/mpfr.jl b/base/mpfr.jl index bcdce70a64635..cb51d21aa5bc5 100644 --- a/base/mpfr.jl +++ b/base/mpfr.jl @@ -18,7 +18,8 @@ import setrounding, maxintfloat, widen, significand, frexp, tryparse, iszero, isone, big, _string_n, decompose, minmax, _precision_with_base_2, sinpi, cospi, sincospi, tanpi, sind, cosd, tand, asind, acosd, atand, - uinttype, exponent_max, exponent_min, ieee754_representation, significand_mask + uinttype, exponent_max, exponent_min, ieee754_representation, significand_mask, + ispositive, isnegative import .Core: AbstractFloat import .Base: Rational, Float16, Float32, Float64, Bool @@ -391,12 +392,18 @@ BigFloat(x::Union{Float16,Float32}, r::MPFRRoundingMode=rounding_raw(BigFloat); BigFloat(Float64(x), r; precision=precision) function BigFloat(x::Rational, r::MPFRRoundingMode=rounding_raw(BigFloat); precision::Integer=_precision_with_base_2(BigFloat)) + r_den = _opposite_round(r) setprecision(BigFloat, precision) do setrounding_raw(BigFloat, r) do - BigFloat(numerator(x))::BigFloat / BigFloat(denominator(x))::BigFloat + BigFloat(numerator(x))::BigFloat / BigFloat(denominator(x), r_den)::BigFloat end end end +function _opposite_round(r::MPFRRoundingMode) + r == MPFRRoundUp && return MPFRRoundDown + r == MPFRRoundDown && return MPFRRoundUp + return r +end function tryparse(::Type{BigFloat}, s::AbstractString; base::Integer=0, precision::Integer=_precision_with_base_2(BigFloat), rounding::MPFRRoundingMode=rounding_raw(BigFloat)) !isempty(s) && isspace(s[end]) && return tryparse(BigFloat, rstrip(s), base = base) @@ -1135,6 +1142,10 @@ isfinite(x::BigFloat) = !isinf(x) && !isnan(x) iszero(x::BigFloat) = x.exp == mpfr_special_exponent_zero isone(x::BigFloat) = x == Clong(1) +# In theory, `!iszero(x) && !isnan(x)` should be the same as `x.exp > mpfr_special_exponent_nan`, but this is safer. +ispositive(x::BigFloat) = !signbit(x) && !iszero(x) && !isnan(x) +isnegative(x::BigFloat) = signbit(x) && !iszero(x) && !isnan(x) + @eval typemax(::Type{BigFloat}) = $(BigFloat(Inf)) @eval typemin(::Type{BigFloat}) = $(BigFloat(-Inf)) @@ -1178,9 +1189,29 @@ Often used as `setprecision(T, precision) do ... end` Note: `nextfloat()`, `prevfloat()` do not use the precision mentioned by `setprecision`. +!!! warning + There is a fallback implementation of this method that calls `precision` + and `setprecision`, but it should no longer be relied on. Instead, you + should define the 3-argument form directly in a way that uses `ScopedValue`, + or recommend that callers use `ScopedValue` and `@with` themselves. + !!! compat "Julia 1.8" The `base` keyword requires at least Julia 1.8. """ +function setprecision(f::Function, ::Type{T}, prec::Integer; kws...) where T + depwarn(""" + The fallback `setprecision(::Function, ...)` method is deprecated. Packages overloading this method should + implement their own specialization using `ScopedValue` instead. + """, :setprecision) + old_prec = precision(T) + setprecision(T, prec; kws...) + try + return f() + finally + setprecision(T, old_prec) + end +end + function setprecision(f::Function, ::Type{BigFloat}, prec::Integer; base::Integer=2) Base.ScopedValues.@with(CURRENT_PRECISION => _convert_precision_from_base(prec, base), f()) end @@ -1229,7 +1260,7 @@ function _prettify_bigfloat(s::String)::String string(neg ? '-' : "", '0', '.', '0'^(-expo-1), int, frac == "0" ? "" : frac) end else - string(mantissa, 'e', exponent) + string(mantissa, 'e', expo) end end diff --git a/base/multidimensional.jl b/base/multidimensional.jl index f15c72a746698..83a03fc1b45bf 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -5,7 +5,7 @@ module IteratorsMD import .Base: eltype, length, size, first, last, in, getindex, setindex!, min, max, zero, oneunit, isless, eachindex, convert, show, iterate, promote_rule, to_indices, copy, - isassigned + isassigned, lastindex, firstindex import .Base: +, -, *, (:) import .Base: simd_outer_range, simd_inner_length, simd_index, setindex @@ -103,6 +103,8 @@ module IteratorsMD # indexing getindex(index::CartesianIndex, i::Integer) = index.I[i] + firstindex(index::CartesianIndex) = firstindex(index.I) + lastindex(index::CartesianIndex) = lastindex(index.I) Base.get(A::AbstractArray, I::CartesianIndex, default) = get(A, I.I, default) eltype(::Type{T}) where {T<:CartesianIndex} = eltype(fieldtype(T, :I)) @@ -146,7 +148,7 @@ module IteratorsMD # hashing const cartindexhash_seed = UInt == UInt64 ? 0xd60ca92f8284b8b0 : 0xf2ea7c2e function Base.hash(ci::CartesianIndex, h::UInt) - h += cartindexhash_seed + h ⊻= cartindexhash_seed for i in ci.I h = hash(i, h) end @@ -187,6 +189,24 @@ module IteratorsMD step(r), ", ", length(r), ")") end + Base.in(x::CartesianIndex, r::AbstractRange{<:CartesianIndex}) = false + function Base.in(x::CartesianIndex{N}, r::AbstractRange{CartesianIndex{N}}) where {N} + isempty(r) && return false + f, st, l = first(r), step(r), last(r) + # The n-th element of the range is a CartesianIndex + # whose elements are the n-th along each dimension + # Find the first dimension along which the index is changing, + # so that n may be uniquely determined + for i in 1:N + iszero(st[i]) && continue + n = findfirst(==(x[i]), f[i]:st[i]:l[i]) + isnothing(n) && return false + return r[n] == x + end + # if the step is zero, the elements are identical, so compare with the first + return x == f + end + # Iteration const OrdinalRangeInt = OrdinalRange{Int, Int} """ @@ -414,7 +434,9 @@ module IteratorsMD @inline function eachindex(::IndexCartesian, A::AbstractArray, B::AbstractArray...) axsA = axes(A) - Base._all_match_first(axes, axsA, B...) || Base.throw_eachindex_mismatch_indices(IndexCartesian(), axes(A), axes.(B)...) + axsBs = map(axes, B) + all(==(axsA), axsBs) || + Base.throw_eachindex_mismatch_indices("axes", axsA, axsBs...) CartesianIndices(axsA) end @@ -450,12 +472,12 @@ module IteratorsMD end @inline function __inc(state::Tuple{Int,Int,Vararg{Int}}, indices::Tuple{OrdinalRangeInt,OrdinalRangeInt,Vararg{OrdinalRangeInt}}) rng = indices[1] - I = state[1] + step(rng) if state[1] != last(rng) + I = state[1] + step(rng) return true, (I, tail(state)...) end - valid, I = __inc(tail(state), tail(indices)) - return valid, (first(rng), I...) + valid, Itail = __inc(tail(state), tail(indices)) + return valid, (first(rng), Itail...) end # 0-d cartesian ranges are special-cased to iterate once and only once @@ -1995,3 +2017,105 @@ end getindex(b::Ref, ::CartesianIndex{0}) = getindex(b) setindex!(b::Ref, x, ::CartesianIndex{0}) = setindex!(b, x) + +## hashing AbstractArray ## can't be put in abstractarray.jl due to bootstrapping problems with the use of @nexpr + +function _hash_fib(A, h::UInt) + # Goal: Hash approximately log(N) entries with a higher density of hashed elements + # weighted towards the end and special consideration for repeated values. Colliding + # hashes will often subsequently be compared by equality -- and equality between arrays + # works elementwise forwards and is short-circuiting. This means that a collision + # between arrays that differ by elements at the beginning is cheaper than one where the + # difference is towards the end. Furthermore, choosing `log(N)` arbitrary entries from a + # sparse array will likely only choose the same element repeatedly (zero in this case). + + # To achieve this, we work backwards, starting by hashing the last element of the + # array. After hashing each element, we skip `fibskip` elements, where `fibskip` + # is pulled from the Fibonacci sequence -- Fibonacci was chosen as a simple + # ~O(log(N)) algorithm that ensures we don't hit a common divisor of a dimension + # and only end up hashing one slice of the array (as might happen with powers of + # two). Finally, we find the next distinct value from the one we just hashed. + + # This is a little tricky since skipping an integer number of values inherently works + # with linear indices, but `findprev` uses `keys`. Hoist out the conversion "maps": + ks = keys(A) + key_to_linear = LinearIndices(ks) # Index into this map to compute the linear index + linear_to_key = vec(ks) # And vice-versa + + # Start at the last index + keyidx = last(ks) + linidx = key_to_linear[keyidx] + fibskip = prevfibskip = oneunit(linidx) + first_linear = first(LinearIndices(linear_to_key)) + @nexprs 4 i -> p_i = h + + n = 0 + while true + n += 1 + # Hash the element + elt = A[keyidx] + + stream_idx = mod1(n, 4) + @nexprs 4 i -> stream_idx == i && (p_i = hash_mix_linear(hash(keyidx, p_i), hash(elt, p_i))) + + # Skip backwards a Fibonacci number of indices -- this is a linear index operation + linidx = key_to_linear[keyidx] + linidx < fibskip + first_linear && break + linidx -= fibskip + keyidx = linear_to_key[linidx] + + # Only increase the Fibonacci skip once every N iterations. This was chosen + # to be big enough that all elements of small arrays get hashed while + # obscenely large arrays are still tractable. With a choice of N=4096, an + # entirely-distinct 8000-element array will have ~75% of its elements hashed, + # with every other element hashed in the first half of the array. At the same + # time, hashing a `typemax(Int64)`-length Float64 range takes about a second. + if rem(n, 4096) == 0 + fibskip, prevfibskip = fibskip + prevfibskip, fibskip + end + + # Find a key index with a value distinct from `elt` -- might be `keyidx` itself + keyidx = findprev(!isequal(elt), A, keyidx) + keyidx === nothing && break + end + + @nexprs 4 i -> h = hash_mix_linear(p_i, h) + return hash_uint(h) +end + +function hash_shaped(A, h::UInt) + # Axes are themselves AbstractArrays, so hashing them directly would stack overflow + # Instead hash the tuple of firsts and lasts along each dimension + h = hash(map(first, axes(A)), h) + h = hash(map(last, axes(A)), h) + len = length(A) + + if len < 8 + # for the shortest arrays we chain directly + for elt in A + h = hash(elt, h) + end + return h + elseif len < 32768 + # separate accumulator streams, unrolled + @nexprs 8 i -> p_i = h + n = 1 + limit = len - 7 + while n <= limit + @nexprs 8 i -> p_i = hash(A[n + i - 1], p_i) + n += 8 + end + while n <= len + p_1 = hash(A[n], p_1) + n += 1 + end + # fold all streams back together + @nexprs 8 i -> h = hash_mix_linear(p_i, h) + return hash_uint(h) + else + return _hash_fib(A, h) + end +end + +const hash_abstractarray_seed = UInt === UInt64 ? 0x7e2d6fb6448beb77 : 0xd4514ce5 +hash(A::AbstractArray, h::UInt) = hash_shaped(A, h ⊻ hash_abstractarray_seed) diff --git a/base/number.jl b/base/number.jl index 7c9ea1db19cac..8314c546147c7 100644 --- a/base/number.jl +++ b/base/number.jl @@ -136,6 +136,50 @@ true """ signbit(x::Real) = x < 0 +""" + ispositive(x) + +Test whether `x > 0`. See also [`isnegative`](@ref). + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. + +# Examples +```jldoctest +julia> ispositive(-4.0) +false + +julia> ispositive(99) +true + +julia> ispositive(0.0) +false +``` +""" +ispositive(x::Real) = x > 0 + +""" + isnegative(x) + +Test whether `x < 0`. See also [`ispositive`](@ref). + +!!! compat "Julia 1.13" + This function requires at least Julia 1.13. + +# Examples +```jldoctest +julia> isnegative(-4.0) +true + +julia> isnegative(99) +false + +julia> isnegative(-0.0) +false +``` +""" +isnegative(x::Real) = x < 0 + """ sign(x) diff --git a/base/operators.jl b/base/operators.jl index d87e55498bd75..51729b852070d 100644 --- a/base/operators.jl +++ b/base/operators.jl @@ -633,7 +633,7 @@ function afoldl(op, a, bs...) end return y end -setfield!(typeof(afoldl).name.mt, :max_args, 34, :monotonic) +setfield!(typeof(afoldl).name, :max_args, Int32(34), :monotonic) for op in (:+, :*, :&, :|, :xor, :min, :max, :kron) @eval begin diff --git a/base/options.jl b/base/options.jl index 2e1751e775ccb..203a91ca8f641 100644 --- a/base/options.jl +++ b/base/options.jl @@ -9,7 +9,7 @@ struct JLOptions commands::Ptr{Ptr{UInt8}} # (e)eval, (E)print, (L)load image_file::Ptr{UInt8} cpu_target::Ptr{UInt8} - nthreadpools::Int16 + nthreadpools::Int8 nthreads::Int16 nmarkthreads::Int16 nsweepthreads::Int8 @@ -17,6 +17,7 @@ struct JLOptions nprocs::Int32 machine_file::Ptr{UInt8} project::Ptr{UInt8} + program_file::Ptr{UInt8} isinteractive::Int8 color::Int8 historyfile::Int8 @@ -65,6 +66,7 @@ struct JLOptions trim::Int8 task_metrics::Int8 timeout_for_safepoint_straggler_s::Int16 + gc_sweep_always_full::Int8 end # This runs early in the sysimage != is not defined yet diff --git a/base/ordering.jl b/base/ordering.jl index 19e8a1cf18109..f2ddd20ab09f0 100644 --- a/base/ordering.jl +++ b/base/ordering.jl @@ -3,7 +3,7 @@ module Order -import ..@__MODULE__, ..parentmodule +import Base: @__MODULE__, parentmodule const Base = parentmodule(@__MODULE__) import .Base: AbstractVector, @propagate_inbounds, isless, identity, getindex, reverse, diff --git a/base/parse.jl b/base/parse.jl index 2530e5a46146a..7e6f35cf6b35a 100644 --- a/base/parse.jl +++ b/base/parse.jl @@ -261,12 +261,12 @@ tryparse(::Type{Union{}}, slurp...; kwargs...) = error("cannot parse a value as function tryparse(::Type{Float64}, s::String) hasvalue, val = ccall(:jl_try_substrtod, Tuple{Bool, Float64}, - (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s)) + (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s) % UInt) hasvalue ? val : nothing end function tryparse(::Type{Float64}, s::SubString{String}) hasvalue, val = ccall(:jl_try_substrtod, Tuple{Bool, Float64}, - (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits) + (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits % UInt) hasvalue ? val : nothing end function tryparse_internal(::Type{Float64}, s::String, startpos::Int, endpos::Int) @@ -281,12 +281,12 @@ function tryparse_internal(::Type{Float64}, s::SubString{String}, startpos::Int, end function tryparse(::Type{Float32}, s::String) hasvalue, val = ccall(:jl_try_substrtof, Tuple{Bool, Float32}, - (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s)) + (Ptr{UInt8},Csize_t,Csize_t), s, 0, sizeof(s) % UInt) hasvalue ? val : nothing end function tryparse(::Type{Float32}, s::SubString{String}) hasvalue, val = ccall(:jl_try_substrtof, Tuple{Bool, Float32}, - (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits) + (Ptr{UInt8},Csize_t,Csize_t), s.string, s.offset, s.ncodeunits % UInt) hasvalue ? val : nothing end function tryparse_internal(::Type{Float32}, s::String, startpos::Int, endpos::Int) diff --git a/base/partr.jl b/base/partr.jl index 6053a584af5ba..d488330f0c87e 100644 --- a/base/partr.jl +++ b/base/partr.jl @@ -107,7 +107,6 @@ function multiq_sift_down(heap::taskheap, idx::Int32) end end - function multiq_size(tpid::Int8) nt = UInt32(Threads._nthreads_in_pool(tpid)) tp = tpid + 1 @@ -138,7 +137,6 @@ function multiq_size(tpid::Int8) return heap_p end - function multiq_insert(task::Task, priority::UInt16) tpid = ccall(:jl_get_task_threadpoolid, Int8, (Any,), task) @assert tpid > -1 @@ -171,10 +169,8 @@ function multiq_insert(task::Task, priority::UInt16) return true end - function multiq_deletemin() - local rn1, rn2 - local prio1, prio2 + local rn1::UInt32 tid = Threads.threadid() tp = ccall(:jl_threadpoolid, Int8, (Int16,), tid-1) + 1 @@ -208,6 +204,8 @@ function multiq_deletemin() end end + @assert @isdefined(rn1) "Assertion to tell the compiler about the definedness of this variable" + heap = tpheaps[rn1] task = heap.tasks[1] if ccall(:jl_set_task_tid, Cint, (Any, Cint), task, tid-1) == 0 diff --git a/base/path.jl b/base/path.jl index 0e67dc4e95db6..a1221e0bdc844 100644 --- a/base/path.jl +++ b/base/path.jl @@ -633,7 +633,7 @@ the [Freedesktop File URI spec](https://www.freedesktop.org/wiki/Specifications/ julia> uripath("/home/user/example file.jl") # On a unix machine "file:///home/user/example%20file.jl" -juila> uripath("C:\\Users\\user\\example file.jl") # On a windows machine +julia> uripath("C:\\Users\\user\\example file.jl") # On a windows machine "file:///C:/Users/user/example%20file.jl" ``` """ diff --git a/base/pkgid.jl b/base/pkgid.jl index 8c776d79a69cb..7ef7c58eee4cc 100644 --- a/base/pkgid.jl +++ b/base/pkgid.jl @@ -17,7 +17,7 @@ end ==(a::PkgId, b::PkgId) = a.uuid == b.uuid && a.name == b.name function hash(pkg::PkgId, h::UInt) - h += 0xc9f248583a0ca36c % UInt + h ⊻= 0xc9f248583a0ca36c % UInt h = hash(pkg.uuid, h) h = hash(pkg.name, h) return h @@ -32,7 +32,7 @@ function binpack(pkg::PkgId) uuid = pkg.uuid write(io, uuid === nothing ? UInt128(0) : UInt128(uuid)) write(io, pkg.name) - return String(take!(io)) + return unsafe_takestring!(io) end function binunpack(s::String) diff --git a/base/pointer.jl b/base/pointer.jl index fdbcc7bb427b9..72c567eaf2a85 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -59,8 +59,6 @@ cconvert(::Type{Ptr{UInt8}}, s::AbstractString) = String(s) cconvert(::Type{Ptr{Int8}}, s::AbstractString) = String(s) unsafe_convert(::Type{Ptr{UInt8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{UInt8}, (Any,), x) unsafe_convert(::Type{Ptr{Int8}}, x::Symbol) = ccall(:jl_symbol_name, Ptr{Int8}, (Any,), x) -unsafe_convert(::Type{Ptr{UInt8}}, s::String) = ccall(:jl_string_ptr, Ptr{UInt8}, (Any,), s) -unsafe_convert(::Type{Ptr{Int8}}, s::String) = ccall(:jl_string_ptr, Ptr{Int8}, (Any,), s) cconvert(::Type{<:Ptr}, a::Array) = getfield(a, :ref) unsafe_convert(::Type{Ptr{S}}, a::AbstractArray{T}) where {S,T} = convert(Ptr{S}, unsafe_convert(Ptr{T}, a)) diff --git a/base/precompilation.jl b/base/precompilation.jl index 5392e119d25a2..1b609f3e2015a 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -26,9 +26,24 @@ struct ExplicitEnv #local_prefs::Union{Nothing, Dict{String, Any}} end -function ExplicitEnv(envpath::String=Base.active_project()) +ExplicitEnv() = ExplicitEnv(Base.active_project()) +function ExplicitEnv(::Nothing, envpath::String="") + ExplicitEnv(envpath, + Dict{String, UUID}(), # project_deps + Dict{String, UUID}(), # project_weakdeps + Dict{String, UUID}(), # project_extras + Dict{String, Vector{UUID}}(), # project_extensions + Dict{UUID, Vector{UUID}}(), # deps + Dict{UUID, Vector{UUID}}(), # weakdeps + Dict{UUID, Dict{String, Vector{UUID}}}(), # extensions + Dict{UUID, String}(), # names + Dict{UUID, Union{SHA1, String, Nothing, Missing}}()) +end +function ExplicitEnv(envpath::String) + # Handle missing project file by creating an empty environment if !isfile(envpath) - error("expected a project file at $(repr(envpath))") + envpath = abspath(envpath) + return ExplicitEnv(nothing, envpath) end envpath = abspath(envpath) project_d = parsed_toml(envpath) @@ -301,7 +316,7 @@ function show_progress(io::IO, p::MiniProgressBar; termwidth=nothing, carriagere else string(p.current, "/", p.max) end - termwidth = @something termwidth displaysize(io)[2] + termwidth = @something termwidth (displaysize(io)::Tuple{Int,Int})[2] max_progress_width = max(0, min(termwidth - textwidth(p.header) - textwidth(progress_text) - 10 , p.width)) n_filled = floor(Int, max_progress_width * perc / 100) partial_filled = (max_progress_width * perc / 100) - n_filled @@ -355,7 +370,7 @@ Base.show(io::IO, err::PkgPrecompileError) = print(io, "PkgPrecompileError: ", e import Base: StaleCacheKey -can_fancyprint(io::IO) = io isa Base.TTY && (get(ENV, "CI", nothing) != "true") +can_fancyprint(io::IO) = @something(get(io, :force_fancyprint, nothing), (io isa Base.TTY && (get(ENV, "CI", nothing) != "true"))) function printpkgstyle(io, header, msg; color=:green) printstyled(io, header; color, bold=true) @@ -409,6 +424,53 @@ function excluded_circular_deps_explanation(io::IOContext{IO}, ext_to_parent::Di return msg end + +function scan_pkg!(stack, could_be_cycle, cycles, pkg, dmap) + if haskey(could_be_cycle, pkg) + return could_be_cycle[pkg] + else + return scan_deps!(stack, could_be_cycle, cycles, pkg, dmap) + end +end +function scan_deps!(stack, could_be_cycle, cycles, pkg, dmap) + push!(stack, pkg) + cycle = nothing + for dep in dmap[pkg] + if dep in stack + # Created fresh cycle + cycle′ = stack[findlast(==(dep), stack):end] + if cycle === nothing || length(cycle′) < length(cycle) + cycle = cycle′ # try to report smallest cycle possible + end + elseif scan_pkg!(stack, could_be_cycle, cycles, dep, dmap) + # Reaches an existing cycle + could_be_cycle[pkg] = true + pop!(stack) + return true + end + end + pop!(stack) + if cycle !== nothing + push!(cycles, cycle) + could_be_cycle[pkg] = true + return true + end + could_be_cycle[pkg] = false + return false +end + +# restrict to dependencies of given packages +function collect_all_deps(direct_deps, dep, alldeps=Set{Base.PkgId}()) + for _dep in direct_deps[dep] + if !(_dep in alldeps) + push!(alldeps, _dep) + collect_all_deps(direct_deps, _dep, alldeps) + end + end + return alldeps +end + + function precompilepkgs(pkgs::Vector{String}=String[]; internal_call::Bool=false, strict::Bool = false, @@ -421,6 +483,7 @@ function precompilepkgs(pkgs::Vector{String}=String[]; fancyprint::Bool = can_fancyprint(io) && !timing, manifest::Bool=false, ignore_loaded::Bool=true) + @debug "precompilepkgs called with" pkgs internal_call strict warn_loaded timing _from_loading configs fancyprint manifest ignore_loaded # monomorphize this to avoid latency problems _precompilepkgs(pkgs, internal_call, strict, warn_loaded, timing, _from_loading, configs isa Vector{Config} ? configs : [configs], @@ -434,7 +497,7 @@ function _precompilepkgs(pkgs::Vector{String}, timing::Bool, _from_loading::Bool, configs::Vector{Config}, - io::IOContext{IO}, + _io::IOContext{IO}, fancyprint::Bool, manifest::Bool, ignore_loaded::Bool) @@ -452,10 +515,9 @@ function _precompilepkgs(pkgs::Vector{String}, num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks))) parallel_limiter = Base.Semaphore(num_tasks) - if _from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false) - # suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests - io = IOContext{IO}(devnull) - end + # suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests + io = (_from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false)) ? IOContext{IO}(devnull) : _io + nconfigs = length(configs) hascolor = get(io, :color, false)::Bool @@ -472,9 +534,12 @@ function _precompilepkgs(pkgs::Vector{String}, # inverse map of `parent_to_ext` above (ext → parent) ext_to_parent = Dict{Base.PkgId, Base.PkgId}() - function describe_pkg(pkg::PkgId, is_project_dep::Bool, flags::Cmd, cacheflags::Base.CacheFlags) + function describe_pkg(pkg::PkgId, is_project_dep::Bool, is_serial_dep::Bool, flags::Cmd, cacheflags::Base.CacheFlags) name = full_name(ext_to_parent, pkg) name = is_project_dep ? name : color_string(name, :light_black) + if is_serial_dep + name *= color_string(" (serial)", :light_black) + end if nconfigs > 1 && !isempty(flags) config_str = join(flags, " ") name *= color_string(" `$config_str`", :light_black) @@ -552,7 +617,7 @@ function _precompilepkgs(pkgs::Vector{String}, end end - indirect_deps = Dict{Base.PkgId, Set{Base.PkgId}}() + local indirect_deps = Dict{Base.PkgId, Set{Base.PkgId}}() for package in keys(direct_deps) # Initialize a set to keep track of all dependencies for 'package' all_deps = Set{Base.PkgId}() @@ -584,15 +649,28 @@ function _precompilepkgs(pkgs::Vector{String}, end @debug "precompile: extensions collected" + serial_deps = Base.PkgId[] # packages that are being precompiled in serial + + if _from_loading + # if called from loading precompilation it may be a package from another environment stack + # where we don't have access to the dep graph, so just add as a single package and do serial + # precompilation of its deps within the job. + for pkg in requested_pkgs # In case loading asks for multiple packages + pkgid = Base.identify_package(pkg) + pkgid === nothing && continue + if !haskey(direct_deps, pkgid) + @debug "precompile: package `$(pkgid)` is outside of the environment, so adding as single package serial job" + direct_deps[pkgid] = Base.PkgId[] # no deps, do them in serial in the job + push!(project_deps, pkgid) # add to project_deps so it doesn't show up in gray + push!(serial_deps, pkgid) + end + end + end + # return early if no deps if isempty(direct_deps) if isempty(pkgs) return - elseif _from_loading - # if called from loading precompilation it may be a package from another environment stack so - # don't error and allow serial precompilation to try - # TODO: actually handle packages from other envs in the stack - return else error("No direct dependencies outside of the sysimage found matching $(pkgs)") end @@ -619,46 +697,14 @@ function _precompilepkgs(pkgs::Vector{String}, could_be_cycle = Dict{Base.PkgId, Bool}() # temporary stack for the SCC-like algorithm below stack = Base.PkgId[] - function scan_pkg!(pkg, dmap) - if haskey(could_be_cycle, pkg) - return could_be_cycle[pkg] - else - return scan_deps!(pkg, dmap) - end - end - function scan_deps!(pkg, dmap) - push!(stack, pkg) - cycle = nothing - for dep in dmap[pkg] - if dep in stack - # Created fresh cycle - cycle′ = stack[findlast(==(dep), stack):end] - if cycle === nothing || length(cycle′) < length(cycle) - cycle = cycle′ # try to report smallest cycle possible - end - elseif scan_pkg!(dep, dmap) - # Reaches an existing cycle - could_be_cycle[pkg] = true - pop!(stack) - return true - end - end - pop!(stack) - if cycle !== nothing - push!(cycles, cycle) - could_be_cycle[pkg] = true - return true - end - could_be_cycle[pkg] = false - return false - end + # set of packages that depend on a cycle (either because they are # a part of a cycle themselves or because they transitively depend # on a package in some cycle) circular_deps = Base.PkgId[] for pkg in keys(direct_deps) @assert isempty(stack) - if scan_pkg!(pkg, direct_deps) + if scan_pkg!(stack, could_be_cycle, cycles, pkg, direct_deps) push!(circular_deps, pkg) for pkg_config in keys(was_processed) # notify all to allow skipping @@ -675,16 +721,6 @@ function _precompilepkgs(pkgs::Vector{String}, if isempty(pkgs) pkgs = [pkg.name for pkg in project_deps] end - # restrict to dependencies of given packages - function collect_all_deps(direct_deps, dep, alldeps=Set{Base.PkgId}()) - for _dep in direct_deps[dep] - if !(_dep in alldeps) - push!(alldeps, _dep) - collect_all_deps(direct_deps, _dep, alldeps) - end - end - return alldeps - end keep = Set{Base.PkgId}() for dep in direct_deps dep_pkgid = first(dep) @@ -711,13 +747,13 @@ function _precompilepkgs(pkgs::Vector{String}, end end - target = nothing + target = Ref{Union{Nothing, String}}(nothing) if nconfigs == 1 if !isempty(only(configs)[1]) - target = "for configuration $(join(only(configs)[1], " "))" + target[] = "for configuration $(join(only(configs)[1], " "))" end else - target = "for $nconfigs compilation configurations..." + target[] = "for $nconfigs compilation configurations..." end @debug "precompile: packages filtered" @@ -727,7 +763,7 @@ function _precompilepkgs(pkgs::Vector{String}, print_lock = io.io isa Base.LibuvStream ? io.io.lock::ReentrantLock : ReentrantLock() first_started = Base.Event() - printloop_should_exit::Bool = !fancyprint # exit print loop immediately if not fancy printing + printloop_should_exit = Ref{Bool}(!fancyprint) # exit print loop immediately if not fancy printing interrupted_or_done = Base.Event() ansi_moveup(n::Int) = string("\e[", n, "A") @@ -736,19 +772,19 @@ function _precompilepkgs(pkgs::Vector{String}, ansi_cleartoendofline = "\e[0K" ansi_enablecursor = "\e[?25h" ansi_disablecursor = "\e[?25l" - n_done::Int = 0 - n_already_precomp::Int = 0 - n_loaded::Int = 0 - interrupted = false + n_done = Ref(0) + n_already_precomp = Ref(0) + n_loaded = Ref(0) + interrupted = Ref(false) - function handle_interrupt(err, in_printloop = false) + function handle_interrupt(err, in_printloop::Bool) notify(interrupted_or_done) in_printloop || wait(t_print) # wait to let the print loop cease first if err isa InterruptException - lock(print_lock) do + @lock print_lock begin println(io, " Interrupted: Exiting precompilation...", ansi_cleartoendofline) end - interrupted = true + interrupted[] = true return true else return false @@ -757,34 +793,34 @@ function _precompilepkgs(pkgs::Vector{String}, std_outputs = Dict{PkgConfig,IOBuffer}() taskwaiting = Set{PkgConfig}() pkgspidlocked = Dict{PkgConfig,String}() - pkg_liveprinted = nothing + pkg_liveprinted = Ref{Union{Nothing, PkgId}}(nothing) function monitor_std(pkg_config, pipe; single_requested_pkg=false) pkg, config = pkg_config try liveprinting = false while !eof(pipe) - str = readline(pipe, keep=true) + local str = readline(pipe, keep=true) if single_requested_pkg && (liveprinting || !isempty(str)) - lock(print_lock) do + @lock print_lock begin if !liveprinting printpkgstyle(io, :Info, "Given $(pkg.name) was explicitly requested, output will be shown live $ansi_cleartoendofline", color = Base.info_color()) liveprinting = true - pkg_liveprinted = pkg + pkg_liveprinted[] = pkg end print(io, ansi_cleartoendofline, str) end end write(get!(IOBuffer, std_outputs, pkg_config), str) if !in(pkg_config, taskwaiting) && occursin("waiting for IO to finish", str) - !fancyprint && lock(print_lock) do + !fancyprint && @lock print_lock begin println(io, pkg.name, color_string(" Waiting for background task / IO / timer.", Base.warn_color())) end push!(taskwaiting, pkg_config) end if !fancyprint && in(pkg_config, taskwaiting) - lock(print_lock) do + @lock print_lock begin print(io, str) end end @@ -799,9 +835,9 @@ function _precompilepkgs(pkgs::Vector{String}, try wait(first_started) (isempty(pkg_queue) || interrupted_or_done.set) && return - lock(print_lock) do - if target !== nothing - printpkgstyle(io, :Precompiling, target) + @lock print_lock begin + if target[] !== nothing + printpkgstyle(io, :Precompiling, target[]) end if fancyprint print(io, ansi_disablecursor) @@ -813,12 +849,12 @@ function _precompilepkgs(pkgs::Vector{String}, last_length = 0 bar = MiniProgressBar(; indent=0, header = "Precompiling packages ", color = :green, percentage=false, always_reprint=true) n_total = length(direct_deps) * length(configs) - bar.max = n_total - n_already_precomp + bar.max = n_total - n_already_precomp[] final_loop = false n_print_rows = 0 - while !printloop_should_exit - lock(print_lock) do - term_size = displaysize(io) + while !printloop_should_exit[] + @lock print_lock begin + term_size = displaysize(io)::Tuple{Int, Int} num_deps_show = max(term_size[1] - 3, 2) # show at least 2 deps pkg_queue_show = if !interrupted_or_done.set && length(pkg_queue) > num_deps_show last(pkg_queue, num_deps_show) @@ -829,20 +865,20 @@ function _precompilepkgs(pkgs::Vector{String}, if i > 1 print(iostr, ansi_cleartoend) end - bar.current = n_done - n_already_precomp - bar.max = n_total - n_already_precomp + bar.current = n_done[] - n_already_precomp[] + bar.max = n_total - n_already_precomp[] # when sizing to the terminal width subtract a little to give some tolerance to resizing the # window between print cycles - termwidth = displaysize(io)[2] - 4 + termwidth = (displaysize(io)::Tuple{Int,Int})[2] - 4 if !final_loop - str = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) - print(iostr, Base._truncate_at_width_or_chars(true, str, termwidth), "\n") + s = sprint(io -> show_progress(io, bar; termwidth, carriagereturn=false); context=io) + print(iostr, Base._truncate_at_width_or_chars(true, s, termwidth), "\n") end for pkg_config in pkg_queue_show dep, config = pkg_config loaded = warn_loaded && haskey(Base.loaded_modules, dep) flags, cacheflags = config - name = describe_pkg(dep, dep in project_deps, flags, cacheflags) + name = describe_pkg(dep, dep in project_deps, dep in serial_deps, flags, cacheflags) line = if pkg_config in precomperr_deps string(color_string(" ? ", Base.warn_color()), name) elseif haskey(failed_deps, pkg_config) @@ -877,10 +913,10 @@ function _precompilepkgs(pkgs::Vector{String}, last_length = length(pkg_queue_show) n_print_rows = count("\n", str_) print(io, str_) - printloop_should_exit = interrupted_or_done.set && final_loop + printloop_should_exit[] = interrupted_or_done.set && final_loop final_loop = interrupted_or_done.set # ensures one more loop to tidy last task after finish i += 1 - printloop_should_exit || print(io, ansi_moveup(n_print_rows), ansi_movecol1) + printloop_should_exit[] || print(io, ansi_moveup(n_print_rows), ansi_movecol1) end wait(t) end @@ -916,7 +952,7 @@ function _precompilepkgs(pkgs::Vector{String}, flags, cacheflags = config task = @async begin try - loaded = haskey(Base.loaded_modules, pkg) + loaded = warn_loaded && haskey(Base.loaded_modules, pkg) for dep in deps # wait for deps to finish wait(was_processed[(dep,config)]) end @@ -925,15 +961,16 @@ function _precompilepkgs(pkgs::Vector{String}, if !circular && is_stale Base.acquire(parallel_limiter) is_project_dep = pkg in project_deps + is_serial_dep = pkg in serial_deps # std monitoring std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) t_monitor = @async monitor_std(pkg_config, std_pipe; single_requested_pkg) - name = describe_pkg(pkg, is_project_dep, flags, cacheflags) - lock(print_lock) do + name = describe_pkg(pkg, is_project_dep, is_serial_dep, flags, cacheflags) + @lock print_lock begin if !fancyprint && isempty(pkg_queue) - printpkgstyle(io, :Precompiling, something(target, "packages...")) + printpkgstyle(io, :Precompiling, something(target[], "packages...")) end end push!(pkg_queue, pkg_config) @@ -960,16 +997,16 @@ function _precompilepkgs(pkgs::Vector{String}, end if ret isa Base.PrecompilableError push!(precomperr_deps, pkg_config) - !fancyprint && lock(print_lock) do + !fancyprint && @lock print_lock begin println(io, _timing_string(t), color_string(" ? ", Base.warn_color()), name) end else - !fancyprint && lock(print_lock) do + !fancyprint && @lock print_lock begin println(io, _timing_string(t), color_string(" ✓ ", loaded ? Base.warn_color() : :green), name) end was_recompiled[pkg_config] = true end - loaded && (n_loaded += 1) + loaded && (n_loaded[] += 1) catch err # @show err close(std_pipe.in) # close pipe to end the std output monitor @@ -978,8 +1015,8 @@ function _precompilepkgs(pkgs::Vector{String}, errmsg = String(take!(get(IOBuffer, std_outputs, pkg_config))) delete!(std_outputs, pkg_config) # so it's not shown as warnings, given error report failed_deps[pkg_config] = (strict || is_project_dep) ? string(sprint(showerror, err), "\n", strip(errmsg)) : "" - !fancyprint && lock(print_lock) do - println(io, " "^9, color_string(" ✗ ", Base.error_color()), name) + !fancyprint && @lock print_lock begin + println(io, " "^12, color_string(" ✗ ", Base.error_color()), name) end else rethrow() @@ -990,15 +1027,15 @@ function _precompilepkgs(pkgs::Vector{String}, Base.release(parallel_limiter) end else - is_stale || (n_already_precomp += 1) + is_stale || (n_already_precomp[] += 1) end - n_done += 1 + n_done[] += 1 notify(was_processed[pkg_config]) catch err_outer # For debugging: # println("Task failed $err_outer") # Base.display_error(ErrorException(""), Base.catch_backtrace())# logging doesn't show here - handle_interrupt(err_outer) || rethrow() + handle_interrupt(err_outer, false) || rethrow() notify(was_processed[pkg_config]) finally filter!(!istaskdone, tasks) @@ -1013,13 +1050,13 @@ function _precompilepkgs(pkgs::Vector{String}, try wait(interrupted_or_done) catch err - handle_interrupt(err) || rethrow() + handle_interrupt(err, false) || rethrow() finally Base.LOADING_CACHE[] = nothing end notify(first_started) # in cases of no-op or !fancyprint fancyprint && wait(t_print) - quick_exit = !all(istaskdone, tasks) || interrupted # if some not finished internal error is likely + quick_exit = !all(istaskdone, tasks) || interrupted[] # if some not finished internal error is likely seconds_elapsed = round(Int, (time_ns() - time_start) / 1e9) ndeps = count(values(was_recompiled)) if ndeps > 0 || !isempty(failed_deps) || (quick_exit && !isempty(std_outputs)) @@ -1031,18 +1068,18 @@ function _precompilepkgs(pkgs::Vector{String}, end plural = length(configs) > 1 ? "dependency configurations" : ndeps == 1 ? "dependency" : "dependencies" print(iostr, " $(ndeps) $(plural) successfully precompiled in $(seconds_elapsed) seconds") - if n_already_precomp > 0 || !isempty(circular_deps) - n_already_precomp > 0 && (print(iostr, ". $n_already_precomp already precompiled")) + if n_already_precomp[] > 0 || !isempty(circular_deps) + n_already_precomp[] > 0 && (print(iostr, ". $(n_already_precomp[]) already precompiled")) !isempty(circular_deps) && (print(iostr, ". $(length(circular_deps)) skipped due to circular dependency")) print(iostr, ".") end - if n_loaded > 0 - plural1 = length(configs) > 1 ? "dependency configurations" : n_loaded == 1 ? "dependency" : "dependencies" - plural2 = n_loaded == 1 ? "a different version is" : "different versions are" - plural3 = n_loaded == 1 ? "" : "s" - plural4 = n_loaded == 1 ? "this package" : "these packages" + if n_loaded[] > 0 + local plural1 = length(configs) > 1 ? "dependency configurations" : n_loaded[] == 1 ? "dependency" : "dependencies" + local plural2 = n_loaded[] == 1 ? "a different version is" : "different versions are" + local plural3 = n_loaded[] == 1 ? "" : "s" + local plural4 = n_loaded[] == 1 ? "this package" : "these packages" print(iostr, "\n ", - color_string(string(n_loaded), Base.warn_color()), + color_string(string(n_loaded[]), Base.warn_color()), " $(plural1) precompiled but ", color_string("$(plural2) currently loaded", Base.warn_color()), ". Restart julia to access the new version$(plural3). \ @@ -1062,12 +1099,12 @@ function _precompilepkgs(pkgs::Vector{String}, let std_outputs = Tuple{PkgConfig,SubString{String}}[(pkg_config, strip(String(take!(io)))) for (pkg_config,io) in std_outputs] filter!(kv -> !isempty(last(kv)), std_outputs) if !isempty(std_outputs) - plural1 = length(std_outputs) == 1 ? "y" : "ies" - plural2 = length(std_outputs) == 1 ? "" : "s" + local plural1 = length(std_outputs) == 1 ? "y" : "ies" + local plural2 = length(std_outputs) == 1 ? "" : "s" print(iostr, "\n ", color_string("$(length(std_outputs))", Base.warn_color()), " dependenc$(plural1) had output during precompilation:") for (pkg_config, err) in std_outputs pkg, config = pkg_config - err = if pkg == pkg_liveprinted + err = if pkg == pkg_liveprinted[] "[Output was shown above]" else join(split(err, "\n"), color_string("\n│ ", Base.warn_color())) @@ -1079,7 +1116,7 @@ function _precompilepkgs(pkgs::Vector{String}, end end let str=str - lock(print_lock) do + @lock print_lock begin println(io, str) end end @@ -1157,7 +1194,7 @@ function precompile_pkgs_maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLo else "another machine (hostname: $hostname, pid: $pid, pidfile: $pidfile)" end - !fancyprint && lock(print_lock) do + !fancyprint && @lock print_lock begin println(io, " ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg_config])", Base.info_color(), hascolor)) end Base.release(parallel_limiter) # release so other work can be done while waiting diff --git a/base/promotion.jl b/base/promotion.jl index 719cd2dc32b61..f935c546915be 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -23,11 +23,16 @@ typejoin(@nospecialize(t), @nospecialize(s), @nospecialize(u)) = (@_foldable_met typejoin(@nospecialize(t), @nospecialize(s), @nospecialize(u), ts...) = (@_foldable_meta; @_nospecializeinfer_meta; afoldl(typejoin, typejoin(t, s, u), ts...)) function typejoin(@nospecialize(a), @nospecialize(b)) @_foldable_meta + @_nothrow_meta @_nospecializeinfer_meta if isa(a, TypeVar) return typejoin(a.ub, b) elseif isa(b, TypeVar) return typejoin(a, b.ub) + elseif a === b + return a + elseif !isa(a, Type) || !isa(b, Type) + return Any elseif a <: b return b elseif b <: a diff --git a/base/public.jl b/base/public.jl index 8d2a65f9a150c..bc3d76e86eadc 100644 --- a/base/public.jl +++ b/base/public.jl @@ -10,6 +10,7 @@ public # Types AbstractLock, + AbstractOneTo, AbstractPipe, AsyncCondition, CodeUnits, @@ -27,6 +28,7 @@ public # Semaphores Semaphore, acquire, + @acquire, release, # arrays @@ -52,6 +54,7 @@ public active_project, # Reflection and introspection + get_extension, isambiguous, isexpr, isidentifier, @@ -65,6 +68,11 @@ public ispublic, remove_linenums!, +# AST handling + IR, + isa_ast_node, + quoted, + # Operators operator_associativity, operator_precedence, @@ -108,6 +116,7 @@ public reseteof, link_pipe!, dup, + showarg, # filesystem operations rename, @@ -116,4 +125,5 @@ public notnothing, runtests, text_colors, - depwarn + depwarn, + donotdelete diff --git a/base/range.jl b/base/range.jl index 1f776879e7bac..e9d28daf3ba3b 100644 --- a/base/range.jl +++ b/base/range.jl @@ -451,6 +451,13 @@ if isdefined(Main, :Base) end end +""" + Base.AbstractOneTo + +Abstract type for ranges that start at 1 and have a step size of 1. +""" +abstract type AbstractOneTo{T} <: AbstractUnitRange{T} end + """ Base.OneTo(n) @@ -458,7 +465,7 @@ Define an `AbstractUnitRange` that behaves like `1:n`, with the added distinction that the lower limit is guaranteed (by the type system) to be 1. """ -struct OneTo{T<:Integer} <: AbstractUnitRange{T} +struct OneTo{T<:Integer} <: AbstractOneTo{T} stop::T # invariant: stop >= zero(stop) function OneTo{T}(stop) where {T<:Integer} throwbool(r) = (@noinline; throw(ArgumentError("invalid index: $r of type Bool"))) @@ -562,7 +569,7 @@ julia> collect(LinRange(-0.1, 0.3, 5)) 0.3 ``` -See also [`Logrange`](@ref Base.LogRange) for logarithmically spaced points. +See also [`Base.LogRange`](@ref Base.LogRange) for logarithmically spaced points. """ struct LinRange{T,L<:Integer} <: AbstractRange{T} start::T @@ -680,7 +687,7 @@ end ## interface implementations length(r::AbstractRange) = error("length implementation missing") # catch mistakes -size(r::AbstractRange) = (length(r),) +size(r::AbstractRange) = (@inline; (length(r),)) isempty(r::StepRange) = # steprange_last(r.start, r.step, r.stop) == r.stop @@ -798,17 +805,12 @@ let bigints = Union{Int, UInt, Int64, UInt64, Int128, UInt128}, s = step(r) diff = last(r) - first(r) isempty(r) && return zero(diff) - # if |s| > 1, diff might have overflowed, but unsigned(diff)÷s should - # therefore still be valid (if the result is representable at all) - # n.b. !(s isa T) - if s isa Unsigned || -1 <= s <= 1 || s == -s - a = div(diff, s) % typeof(diff) - elseif s < 0 - a = div(unsigned(-diff), -s) % typeof(diff) - else - a = div(unsigned(diff), s) % typeof(diff) - end - return a + oneunit(a) + # Compute `(diff ÷ s) + 1` in a manner robust to signed overflow + # by using the absolute values as unsigneds for non-empty ranges. + # Note that `s` may be a different type from T and diff; it may not + # even be a BitInteger that supports `unsigned`. Handle with care. + a = div(unsigned(flipsign(diff, s)), s) % typeof(diff) + return flipsign(a, s) + oneunit(a) end function checked_length(r::OrdinalRange{T}) where T<:bigints s = step(r) diff --git a/base/rational.jl b/base/rational.jl index 8c5244ea4bad3..e04f14760d8f4 100644 --- a/base/rational.jl +++ b/base/rational.jl @@ -572,13 +572,13 @@ float(::Type{Rational{T}}) where {T<:Integer} = float(T) function gcd(x::Rational, y::Rational) if isinf(x) != isinf(y) - throw(ArgumentError("lcm is not defined between infinite and finite numbers")) + throw(ArgumentError("gcd is not defined between infinite and finite numbers")) end unsafe_rational(gcd(x.num, y.num), lcm(x.den, y.den)) end function lcm(x::Rational, y::Rational) if isinf(x) != isinf(y) - throw(ArgumentError("lcm is not defined")) + throw(ArgumentError("lcm is not defined between infinite and finite numbers")) end return unsafe_rational(lcm(x.num, y.num), gcd(x.den, y.den)) end @@ -620,7 +620,7 @@ function hash(x::Rational{<:BitInteger64}, h::UInt) end end h = hash_integer(pow, h) - h = hash_integer(num, h) + h = hash_integer((pow > 0) ? (num << (pow % 64)) : num, h) return h end diff --git a/base/reduce.jl b/base/reduce.jl index c132ef385358d..743713360ed35 100644 --- a/base/reduce.jl +++ b/base/reduce.jl @@ -14,7 +14,7 @@ The reduction operator used in `sum`. The main difference from [`+`](@ref) is th integers are promoted to `Int`/`UInt`. """ add_sum(x, y) = x + y -add_sum(x::BitSignedSmall, y::BitSignedSmall) = Int(x) + Int(y) +add_sum(x::Union{Bool,BitIntegerSmall}, y::Union{Bool,BitIntegerSmall}) = Int(x) + Int(y) add_sum(x::BitUnsignedSmall, y::BitUnsignedSmall) = UInt(x) + UInt(y) add_sum(x::Real, y::Real)::Real = x + y @@ -29,6 +29,12 @@ mul_prod(x::BitSignedSmall, y::BitSignedSmall) = Int(x) * Int(y) mul_prod(x::BitUnsignedSmall, y::BitUnsignedSmall) = UInt(x) * UInt(y) mul_prod(x::Real, y::Real)::Real = x * y +and_all(x, y) = (x && y)::Bool +or_any(x, y) = (x || y)::Bool +# As a performance optimization, avoid runtime branches: +and_all(x::Bool, y::Bool) = (x & y)::Bool +or_any(x::Bool, y::Bool) = (x | y)::Bool + ## foldl && mapfoldl function mapfoldl_impl(f::F, op::OP, nt, itr) where {F,OP} @@ -338,6 +344,8 @@ reduce_empty(::typeof(*), ::Type{T}) where {T} = one(T) reduce_empty(::typeof(*), ::Type{<:AbstractChar}) = "" reduce_empty(::typeof(&), ::Type{Bool}) = true reduce_empty(::typeof(|), ::Type{Bool}) = false +reduce_empty(::typeof(and_all), ::Type{T}) where {T} = true +reduce_empty(::typeof(or_any), ::Type{T}) where {T} = false reduce_empty(::typeof(add_sum), ::Type{T}) where {T} = reduce_empty(+, T) reduce_empty(::typeof(add_sum), ::Type{T}) where {T<:BitSignedSmall} = zero(Int) @@ -354,7 +362,7 @@ reduce_empty(op::FlipArgs, ::Type{T}) where {T} = reduce_empty(op.f, T) """ Base.mapreduce_empty(f, op, T) -The value to be returned when calling [`mapreduce`](@ref), [`mapfoldl`](@ref`) or +The value to be returned when calling [`mapreduce`](@ref), [`mapfoldl`](@ref) or [`mapfoldr`](@ref) with map `f` and reduction `op` over an empty array with element type of `T`. See [`Base.reduce_empty`](@ref) for more information. """ @@ -380,7 +388,7 @@ reduce_empty_iter(op, itr, ::EltypeUnknown) = throw(ArgumentError(""" """ Base.reduce_first(op, x) -The value to be returned when calling [`reduce`](@ref), [`foldl`](@ref`) or +The value to be returned when calling [`reduce`](@ref), [`foldl`](@ref) or [`foldr`](@ref) with reduction `op` over an iterator which contains a single element `x`. This value may also be used to initialise the recursion, so that `reduce(op, [x, y])` may call `op(reduce_first(op, x), y)`. @@ -399,11 +407,13 @@ reduce_first(::typeof(add_sum), x::BitUnsignedSmall) = UInt(x) reduce_first(::typeof(mul_prod), x) = reduce_first(*, x) reduce_first(::typeof(mul_prod), x::BitSignedSmall) = Int(x) reduce_first(::typeof(mul_prod), x::BitUnsignedSmall) = UInt(x) +reduce_first(::typeof(vcat), x) = vcat(x) +reduce_first(::typeof(hcat), x) = hcat(x) """ Base.mapreduce_first(f, op, x) -The value to be returned when calling [`mapreduce`](@ref), [`mapfoldl`](@ref`) or +The value to be returned when calling [`mapreduce`](@ref), [`mapfoldl`](@ref) or [`mapfoldr`](@ref) with map `f` and reduction `op` over an iterator which contains a single element `x`. This value may also be used to initialise the recursion, so that `mapreduce(f, op, [x, y])` may call `op(mapreduce_first(f, op, x), f(y))`. @@ -609,63 +619,6 @@ julia> prod(1:5; init = 1.0) prod(a; kw...) = mapreduce(identity, mul_prod, a; kw...) ## maximum, minimum, & extrema -_fast(::typeof(min),x,y) = min(x,y) -_fast(::typeof(max),x,y) = max(x,y) -function _fast(::typeof(max), x::AbstractFloat, y::AbstractFloat) - ifelse(isnan(x), - x, - ifelse(x > y, x, y)) -end - -function _fast(::typeof(min),x::AbstractFloat, y::AbstractFloat) - ifelse(isnan(x), - x, - ifelse(x < y, x, y)) -end - -isbadzero(::typeof(max), x::AbstractFloat) = (x == zero(x)) & signbit(x) -isbadzero(::typeof(min), x::AbstractFloat) = (x == zero(x)) & !signbit(x) -isbadzero(op, x) = false -isgoodzero(::typeof(max), x) = isbadzero(min, x) -isgoodzero(::typeof(min), x) = isbadzero(max, x) - -function mapreduce_impl(f, op::Union{typeof(max), typeof(min)}, - A::AbstractArrayOrBroadcasted, first::Int, last::Int) - a1 = @inbounds A[first] - v1 = mapreduce_first(f, op, a1) - v2 = v3 = v4 = v1 - chunk_len = 256 - start = first + 1 - simdstop = start + chunk_len - 4 - while simdstop <= last - 3 - for i in start:4:simdstop - v1 = _fast(op, v1, f(@inbounds(A[i+0]))) - v2 = _fast(op, v2, f(@inbounds(A[i+1]))) - v3 = _fast(op, v3, f(@inbounds(A[i+2]))) - v4 = _fast(op, v4, f(@inbounds(A[i+3]))) - end - checkbounds(A, simdstop+3) - start += chunk_len - simdstop += chunk_len - end - v = op(op(v1,v2),op(v3,v4)) - for i in start:last - @inbounds ai = A[i] - v = op(v, f(ai)) - end - - # enforce correct order of 0.0 and -0.0 - # e.g. maximum([0.0, -0.0]) === 0.0 - # should hold - if isbadzero(op, v) - for i in first:last - x = @inbounds A[i] - isgoodzero(op,x) && return x - end - end - return v -end - """ maximum(f, itr; [init]) @@ -852,14 +805,6 @@ ExtremaMap(::Type{T}) where {T} = ExtremaMap{Type{T}}(T) @inline (f::ExtremaMap)(x) = (y = f.f(x); (y, y)) @inline _extrema_rf((min1, max1), (min2, max2)) = (min(min1, min2), max(max1, max2)) -# optimization for IEEEFloat -function _extrema_rf(x::NTuple{2,T}, y::NTuple{2,T}) where {T<:IEEEFloat} - (x1, x2), (y1, y2) = x, y - anynan = isnan(x1)|isnan(y1) - z1 = ifelse(anynan, x1-y1, ifelse(signbit(x1-y1), x1, y1)) - z2 = ifelse(anynan, x1-y1, ifelse(signbit(x2-y2), y2, x2)) - z1, z2 -end ## findmax, findmin, argmax & argmin @@ -1130,8 +1075,8 @@ julia> count(i->(4<=i<=6), [2,3,4,5,6]) julia> count([true, false, true, true]) 3 -julia> count(>(3), 1:7, init=0x03) -0x07 +julia> count(>(3), 1:7, init=UInt(0)) +0x0000000000000004 ``` """ count(itr; init=0) = count(identity, itr; init) @@ -1140,8 +1085,10 @@ count(f, itr; init=0) = _simple_count(f, itr, init) _simple_count(pred, itr, init) = sum(_bool(pred), itr; init) -function _simple_count(::typeof(identity), x::Array{Bool}, init::T=0) where {T} - n::T = init +function _simple_count(::typeof(identity), x::Array{Bool}, init=0) + v0 = Base.add_sum(init, false) + T = typeof(v0) + n::T = v0 chunks = length(x) ÷ sizeof(UInt) mask = 0x0101010101010101 % UInt GC.@preserve x begin diff --git a/base/reducedim.jl b/base/reducedim.jl index 0478afe1a46b6..ba3434494bc0b 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -45,7 +45,7 @@ end initarray!(a::AbstractArray{T}, f, ::Union{typeof(min),typeof(max),typeof(_extrema_rf)}, init::Bool, src::AbstractArray) where {T} = (init && mapfirst!(f, a, src); a) -for (Op, initval) in ((:(typeof(&)), true), (:(typeof(|)), false)) +for (Op, initval) in ((:(typeof(and_all)), true), (:(typeof(or_any)), false)) @eval initarray!(a::AbstractArray, ::Any, ::$(Op), init::Bool, src::AbstractArray) = (init && fill!(a, $initval); a) end @@ -173,6 +173,10 @@ end reducedim_init(f::Union{typeof(abs),typeof(abs2)}, op::typeof(max), A::AbstractArray{T}, region) where {T} = reducedim_initarray(A, region, zero(f(zero(T))), _realtype(f, T)) +reducedim_init(f, op::typeof(and_all), A::AbstractArrayOrBroadcasted, region) = reducedim_initarray(A, region, true) +reducedim_init(f, op::typeof(or_any), A::AbstractArrayOrBroadcasted, region) = reducedim_initarray(A, region, false) + +# These definitions are wrong in general; Cf. JuliaLang/julia#45562 reducedim_init(f, op::typeof(&), A::AbstractArrayOrBroadcasted, region) = reducedim_initarray(A, region, true) reducedim_init(f, op::typeof(|), A::AbstractArrayOrBroadcasted, region) = reducedim_initarray(A, region, false) @@ -883,13 +887,13 @@ julia> A = [true false; true false] 1 0 1 0 -julia> all!([1; 1], A) -2-element Vector{Int64}: +julia> all!(Bool[1; 1], A) +2-element Vector{Bool}: 0 0 -julia> all!([1 1], A) -1×2 Matrix{Int64}: +julia> all!(Bool[1 1], A) +1×2 Matrix{Bool}: 1 0 ``` """ @@ -958,13 +962,13 @@ julia> A = [true false; true false] 1 0 1 0 -julia> any!([1; 1], A) -2-element Vector{Int64}: +julia> any!(Bool[1; 1], A) +2-element Vector{Bool}: 1 1 -julia> any!([1 1], A) -1×2 Matrix{Int64}: +julia> any!(Bool[1 1], A) +1×2 Matrix{Bool}: 1 0 ``` """ @@ -994,7 +998,7 @@ _all(a, ::Colon) = _all(identity, a, :) for (fname, op) in [(:sum, :add_sum), (:prod, :mul_prod), (:maximum, :max), (:minimum, :min), - (:all, :&), (:any, :|), + (:all, :and_all), (:any, :or_any), (:extrema, :_extrema_rf)] fname! = Symbol(fname, '!') _fname = Symbol('_', fname) diff --git a/base/reflection.jl b/base/reflection.jl index 3b0d522bfd0c2..5ad05b615ddfd 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -16,7 +16,7 @@ The keyword `debuginfo` controls the amount of code metadata present in the outp Note that an error will be thrown if `types` are not concrete types when `generated` is `true` and any of the corresponding methods are an `@generated` method. """ -function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool=true, debuginfo::Symbol=:default) +function code_lowered(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); generated::Bool=true, debuginfo::Symbol=:default) if @isdefined(IRShow) debuginfo = IRShow.debuginfo(debuginfo) elseif debuginfo === :default @@ -28,7 +28,7 @@ function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool= world = get_world_counter() world == typemax(UInt) && error("code reflection cannot be used from generated functions") ret = CodeInfo[] - for m in method_instances(f, t, world) + for m in method_instances(argtypes, world) if generated && hasgenerator(m) if may_invoke_generator(m) code = ccall(:jl_code_for_staged, Ref{CodeInfo}, (Any, UInt, Ptr{Cvoid}), m, world, C_NULL) @@ -46,12 +46,17 @@ function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool= return ret end +function code_lowered(@nospecialize(f), @nospecialize(t=Tuple); generated::Bool=true, debuginfo::Symbol=:default) + tt = signature_type(f, t) + return code_lowered(tt; generated, debuginfo) +end + # for backwards compat const uncompressed_ast = uncompressed_ir const _uncompressed_ast = _uncompressed_ir -function method_instances(@nospecialize(f), @nospecialize(t), world::UInt) - tt = signature_type(f, t) +function method_instances(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}), world::UInt) + tt = to_tuple_type(argtypes) results = Core.MethodInstance[] # this make a better error message than the typeassert that follows world == typemax(UInt) && error("code reflection cannot be used from generated functions") @@ -62,15 +67,26 @@ function method_instances(@nospecialize(f), @nospecialize(t), world::UInt) return results end -function method_instance(@nospecialize(f), @nospecialize(t); - world=Base.get_world_counter(), method_table=nothing) +function method_instances(@nospecialize(f), @nospecialize(t), world::UInt) tt = signature_type(f, t) + return method_instances(tt, world) +end + +function method_instance(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); + world=Base.get_world_counter(), method_table=nothing) + tt = to_tuple_type(argtypes) mi = ccall(:jl_method_lookup_by_tt, Any, (Any, Csize_t, Any), tt, world, method_table) return mi::Union{Nothing, MethodInstance} end +function method_instance(@nospecialize(f), @nospecialize(t); + world=Base.get_world_counter(), method_table=nothing) + tt = signature_type(f, t) + return method_instance(tt; world, method_table) +end + default_debug_info_kind() = unsafe_load(cglobal(:jl_default_debug_info_kind, Cint)) # this type mirrors jl_cgparams_t (documented in julia.h) @@ -211,8 +227,31 @@ julia> code_typed(+, (Float64, Float64)) 1 ─ %1 = Base.add_float(x, y)::Float64 └── return %1 ) => Float64 + +julia> code_typed((typeof(-), Float64, Float64)) +1-element Vector{Any}: + CodeInfo( +1 ─ %1 = Base.sub_float(x, y)::Float64 +└── return %1 +) => Float64 + +julia> code_typed((Type{Int}, UInt8)) +1-element Vector{Any}: + CodeInfo( +1 ─ %1 = Core.zext_int(Core.Int64, x)::Int64 +└── return %1 +) => Int64 + +julia> code_typed((Returns{Int64},)) +1-element Vector{Any}: + CodeInfo( +1 ─ %1 = builtin Base.getfield(obj, :value)::Int64 +└── return %1 +) => Int64 ``` """ +function code_typed end + function code_typed(@nospecialize(f), @nospecialize(types=default_tt(f)); kwargs...) if isa(f, Core.OpaqueClosure) return code_typed_opaque_closure(f, types; kwargs...) @@ -221,6 +260,12 @@ function code_typed(@nospecialize(f), @nospecialize(types=default_tt(f)); kwargs return code_typed_by_type(tt; kwargs...) end +# support 'functor'-like queries, such as `(::Foo)(::Int, ::Int)` via `code_typed((Foo, Int, Int))` +function code_typed(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) + tt = to_tuple_type(argtypes) + return code_typed_by_type(tt; kwargs...) +end + # returns argument tuple type which is supposed to be used for `code_typed` and its family; # if there is a single method this functions returns the method argument signature, # otherwise returns `Tuple` that doesn't match with any signature @@ -402,6 +447,11 @@ function code_ircode(@nospecialize(f), @nospecialize(types = default_tt(f)); kwa return code_ircode_by_type(tt; kwargs...) end +function code_ircode(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) + tt = to_tuple_type(argtypes) + return code_ircode_by_type(tt; kwargs...) +end + """ code_ircode_by_type(types::Type{<:Tuple}; ...) @@ -910,6 +960,7 @@ Returns the method that would be called by the given type signature (as a tuple function which(@nospecialize(tt#=::Type=#)) return _which(tt).method end +which(@nospecialize(argtypes::Tuple)) = which(to_tuple_type(argtypes)) """ which(module, symbol) @@ -933,13 +984,7 @@ this is a compiler-generated name. For explicitly-declared subtypes of `Function`, it is the name of the function's type. """ function nameof(f::Function) - t = typeof(f) - mt = t.name.mt - if mt === Symbol.name.mt - # uses shared method table, so name is not unique to this function type - return nameof(t) - end - return mt.name + return typeof(f).name.singletonname end function nameof(f::Core.IntrinsicFunction) @@ -1035,11 +1080,11 @@ function hasmethod(f, t, kwnames::Tuple{Vararg{Symbol}}; world::UInt=get_world_c match = ccall(:jl_gf_invoke_lookup, Any, (Any, Any, UInt), tt, nothing, world) match === nothing && return false kws = ccall(:jl_uncompress_argnames, Array{Symbol,1}, (Any,), (match::Method).slot_syms) + kws = kws[((match::Method).nargs + 1):end] # remove positional arguments isempty(kws) && return true # some kwfuncs simply forward everything directly for kw in kws endswith(String(kw), "...") && return true end - kwnames = collect(kwnames) return issubset(kwnames, kws) end @@ -1275,16 +1320,17 @@ macro invoke(ex) return esc(out) end -apply_gr(gr::GlobalRef, @nospecialize args...) = getglobal(gr.mod, gr.name)(args...) -apply_gr_kw(@nospecialize(kwargs::NamedTuple), gr::GlobalRef, @nospecialize args...) = Core.kwcall(kwargs, getglobal(gr.mod, gr.name), args...) +getglobalref(gr::GlobalRef, world::UInt) = ccall(:jl_eval_globalref, Any, (Any, UInt), gr, world) -function invokelatest_gr(gr::GlobalRef, @nospecialize args...; kwargs...) +function invokelatest_gr(gr::GlobalRef, args...; kwargs...) @inline kwargs = merge(NamedTuple(), kwargs) + world = get_world_counter() + f = getglobalref(gr, world) if isempty(kwargs) - return invokelatest(apply_gr, gr, args...) + return invoke_in_world(world, f, args...) end - return invokelatest(apply_gr_kw, kwargs, gr, args...) + return invoke_in_world(world, Core.kwcall, kwargs, f, args...) end """ diff --git a/base/regex.jl b/base/regex.jl index baef3f9fdd197..691dbc94c5563 100644 --- a/base/regex.jl +++ b/base/regex.jl @@ -802,7 +802,7 @@ end ## hash ## const hashre_seed = UInt === UInt64 ? 0x67e195eb8555e72d : 0xe32373e4 function hash(r::Regex, h::UInt) - h += hashre_seed + h ⊻= hashre_seed h = hash(r.pattern, h) h = hash(r.compile_options, h) h = hash(r.match_options, h) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 4feb14539f9dc..dc993515ed0cd 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -271,7 +271,8 @@ SCartesianIndices2{K}(indices2::AbstractUnitRange{Int}) where {K} = (@assert K:: eachindex(::IndexSCartesian2{K}, A::ReshapedReinterpretArray) where {K} = SCartesianIndices2{K}(eachindex(IndexLinear(), parent(A))) @inline function eachindex(style::IndexSCartesian2{K}, A::AbstractArray, B::AbstractArray...) where {K} iter = eachindex(style, A) - _all_match_first(C->eachindex(style, C), iter, B...) || throw_eachindex_mismatch_indices(IndexSCartesian2{K}(), axes(A), axes.(B)...) + itersBs = map(C->eachindex(style, C), B) + all(==(iter), itersBs) || throw_eachindex_mismatch_indices("axes", axes(A), map(axes, B)...) return iter end @@ -310,16 +311,17 @@ SimdLoop.simd_inner_length(::SCartesianIndices2{K}, ::Any) where K = K SCartesianIndex2{K}(I1+1, Ilast) end +_maybe_reshape(::IndexSCartesian2, A::AbstractArray, I...) = _maybe_reshape(IndexCartesian(), A, I...) _maybe_reshape(::IndexSCartesian2, A::ReshapedReinterpretArray, I...) = A # fallbacks -function _getindex(::IndexSCartesian2, A::AbstractArray{T,N}, I::Vararg{Int, N}) where {T,N} +function _getindex(::IndexSCartesian2, A::AbstractArray, I::Vararg{Int, N}) where {N} @_propagate_inbounds_meta - getindex(A, I...) + _getindex(IndexCartesian(), A, I...) end -function _setindex!(::IndexSCartesian2, A::AbstractArray{T,N}, v, I::Vararg{Int, N}) where {T,N} +function _setindex!(::IndexSCartesian2, A::AbstractArray, v, I::Vararg{Int, N}) where {N} @_propagate_inbounds_meta - setindex!(A, v, I...) + _setindex!(IndexCartesian(), A, v, I...) end # fallbacks for array types that use "pass-through" indexing (e.g., `IndexStyle(A) = IndexStyle(parent(A))`) # but which don't handle SCartesianIndex2 @@ -328,11 +330,25 @@ function _getindex(::IndexSCartesian2, A::AbstractArray{T,N}, ind::SCartesianInd J = _ind2sub(tail(axes(A)), ind.j) getindex(A, ind.i, J...) end + +function _getindex(::IndexSCartesian2{2}, A::AbstractArray{T,2}, ind::SCartesianIndex2) where {T} + @_propagate_inbounds_meta + J = first(axes(A, 2)) + ind.j - 1 + getindex(A, ind.i, J) +end + function _setindex!(::IndexSCartesian2, A::AbstractArray{T,N}, v, ind::SCartesianIndex2) where {T,N} @_propagate_inbounds_meta J = _ind2sub(tail(axes(A)), ind.j) setindex!(A, v, ind.i, J...) end + +function _setindex!(::IndexSCartesian2{2}, A::AbstractArray{T,2}, v, ind::SCartesianIndex2) where {T} + @_propagate_inbounds_meta + J = first(axes(A, 2)) + ind.j - 1 + setindex!(A, v, ind.i, J) +end + eachindex(style::IndexSCartesian2, A::AbstractArray) = eachindex(style, parent(A)) ## AbstractArray interface @@ -390,7 +406,7 @@ check_ptr_indexable(a::Array, sz) = sizeof(eltype(a)) !== sz check_ptr_indexable(a::Memory, sz) = true check_ptr_indexable(a::AbstractArray, sz) = false -@propagate_inbounds getindex(a::ReinterpretArray) = a[firstindex(a)] +@propagate_inbounds getindex(a::ReshapedReinterpretArray{T,0}) where {T} = a[firstindex(a)] @propagate_inbounds isassigned(a::ReinterpretArray, inds::Integer...) = checkbounds(Bool, a, inds...) && (check_ptr_indexable(a) || _isassigned_ra(a, inds...)) @propagate_inbounds isassigned(a::ReinterpretArray, inds::SCartesianIndex2) = isassigned(a.parent, inds.j) @@ -411,7 +427,7 @@ end # Convert to full indices here, to avoid needing multiple conversions in # the loop in _getindex_ra inds = _to_subscript_indices(a, i) - isempty(inds) ? _getindex_ra(a, 1, ()) : _getindex_ra(a, inds[1], tail(inds)) + isempty(inds) ? _getindex_ra(a, firstindex(a), ()) : _getindex_ra(a, inds[1], tail(inds)) end @propagate_inbounds function getindex(a::ReshapedReinterpretArray{T,N,S}, ind::SCartesianIndex2) where {T,N,S} @@ -534,13 +550,13 @@ end @propagate_inbounds function setindex!(a::NonReshapedReinterpretArray{T,0,S}, v) where {T,S} if isprimitivetype(S) && isprimitivetype(T) - a.parent[] = reinterpret(S, v) + a.parent[] = reinterpret(S, convert(T, v)::T) return a end setindex!(a, v, firstindex(a)) end -@propagate_inbounds setindex!(a::ReinterpretArray, v) = setindex!(a, v, firstindex(a)) +@propagate_inbounds setindex!(a::ReshapedReinterpretArray{T,0}, v) where {T} = setindex!(a, v, firstindex(a)) @propagate_inbounds function setindex!(a::ReinterpretArray{T,N,S}, v, inds::Vararg{Int, N}) where {T,N,S} check_writable(a) @@ -555,7 +571,7 @@ end return _setindex_ra!(a, v, i, ()) end inds = _to_subscript_indices(a, i) - _setindex_ra!(a, v, inds[1], tail(inds)) + isempty(inds) ? _setindex_ra!(a, v, firstindex(a), ()) : _setindex_ra!(a, v, inds[1], tail(inds)) end @propagate_inbounds function setindex!(a::ReshapedReinterpretArray{T,N,S}, v, ind::SCartesianIndex2) where {T,N,S} diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index d5292a73052eb..5bfa64d756a97 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -36,7 +36,7 @@ length(R::ReshapedArrayIterator) = length(R.iter) eltype(::Type{<:ReshapedArrayIterator{I}}) where {I} = @isdefined(I) ? ReshapedIndex{eltype(I)} : Any @noinline throw_dmrsa(dims, len) = - throw(DimensionMismatch("new dimensions $(dims) must be consistent with array length $len")) + throw(DimensionMismatch(LazyString("new dimensions ", dims, " must be consistent with array length ", len))) ## reshape(::Array, ::Dims) returns a new Array (to avoid conditionally aliasing the structure, only the data) # reshaping to same # of dimensions @@ -120,6 +120,9 @@ julia> reshape(1:6, 2, 3) reshape reshape(parent::AbstractArray, dims::IntOrInd...) = reshape(parent, dims) +reshape(parent::AbstractArray, shp::Tuple{Union{Integer,AbstractOneTo}, Vararg{Union{Integer,AbstractOneTo}}}) = reshape(parent, to_shape(shp)) +# legacy method for packages that specialize reshape(parent::AbstractArray, shp::Tuple{Union{Integer,OneTo,CustomAxis}, Vararg{Union{Integer,OneTo,CustomAxis}}}) +# leaving this method in ensures that Base owns the more specific method reshape(parent::AbstractArray, shp::Tuple{Union{Integer,OneTo}, Vararg{Union{Integer,OneTo}}}) = reshape(parent, to_shape(shp)) reshape(parent::AbstractArray, dims::Tuple{Integer, Vararg{Integer}}) = reshape(parent, map(Int, dims)) reshape(parent::AbstractArray, dims::Dims) = _reshape(parent, dims) @@ -222,7 +225,7 @@ function _reshape(parent::AbstractArray, dims::Dims) end @noinline function _throw_dmrs(n, str, dims) - throw(DimensionMismatch("parent has $n elements, which is incompatible with $str $dims")) + throw(DimensionMismatch("parent has $n elements, which is incompatible with $str $dims ($(prod(dims)) elements)")) end # Reshaping a ReshapedArray diff --git a/base/rounding.jl b/base/rounding.jl index 0d8f576ded398..88966c82fb3a6 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -337,7 +337,7 @@ Without keyword arguments, `x` is rounded to an integer value, returning a value thrown if the value is not representable by `T`, similar to [`convert`](@ref). If the `digits` keyword argument is provided, it rounds to the specified number of digits -after the decimal place (or before if negative), in base `base`. +after the decimal place (or before if `digits` is negative), in base `base`. If the `sigdigits` keyword argument is provided, it rounds to the specified number of significant digits, in base `base`. diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 449b08580ae1a..6fc4f139f4a89 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -173,6 +173,43 @@ false """ ispublic(m::Module, s::Symbol) = ccall(:jl_module_public_p, Cint, (Any, Any), m, s) != 0 +""" + @__FUNCTION__ + +Get the innermost enclosing function object. + +!!! note + `@__FUNCTION__` has the same scoping behavior as `return`: when used + inside a closure, it refers to the closure and not the outer function. + Some macros, including [`@spawn`](@ref Threads.@spawn), [`@async`](@ref), etc., + wrap their input in closures. When `@__FUNCTION__` is used within such code, + it will refer to the closure created by the macro rather than the enclosing function. + +# Examples + +`@__FUNCTION__` enables recursive anonymous functions: + +```jldoctest +julia> factorial = (n -> n <= 1 ? 1 : n * (@__FUNCTION__)(n - 1)); + +julia> factorial(5) +120 +``` + +`@__FUNCTION__` can be combined with `nameof` to identify a function's +name from within its body: + +```jldoctest +julia> bar() = nameof(@__FUNCTION__); + +julia> bar() +:bar +``` +""" +macro __FUNCTION__() + Expr(:thisfunction) +end + # TODO: this is vaguely broken because it only works for explicit calls to # `Base.deprecate`, not the @deprecated macro: isdeprecated(m::Module, s::Symbol) = ccall(:jl_is_binding_deprecated, Cint, (Any, Any), m, s) != 0 @@ -530,6 +567,10 @@ struct DataTypeLayout # fielddesc_type : 2; # arrayelem_isboxed : 1; # arrayelem_isunion : 1; + # arrayelem_isatomic : 1; + # arrayelem_islocked : 1; + # isbitsegal : 1; + # padding : 8; end """ @@ -602,7 +643,7 @@ function datatype_isbitsegal(dt::DataType) @_foldable_meta dt.layout == C_NULL && throw(UndefRefError()) flags = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).flags - return (flags & (1<<5)) != 0 + return (flags & (1<<7)) != 0 end """ @@ -853,10 +894,45 @@ end """ isdispatchtuple(T) -Determine whether type `T` is a tuple of concrete types, -meaning it could appear as a type signature in dispatch -and has no subtypes (or supertypes) which could appear in a call. +Determine whether type `T` is a [`Tuple`](@ref) that could appear as a type +signature in dispatch. For this to be true, every element of the tuple type +must be either: +- [concrete](@ref isconcretetype) but not a [kind type](@ref Base.iskindtype) +- a [`Type{U}`](@ref Type) with no free type variables in `U` + +!!! note + A dispatch tuple is relevant for method dispatch because it has no inhabited + subtypes. + + For example, `Tuple{Int, DataType}` is concrete, but is not a dispatch tuple + because `Tuple{Int, Type{Bool}}` is an inhabited subtype. + + `Tuple{Tuple{DataType}}` *is* a dispatch tuple because `Tuple{DataType}` is + concrete and not a kind; the subtype `Tuple{Tuple{Type{Int}}}` is not + inhabited. + If `T` is not a type, then return `false`. + +# Examples +```jldoctest +julia> isdispatchtuple(Int) +false + +julia> isdispatchtuple(Tuple{Int}) +true + +julia> isdispatchtuple(Tuple{Number}) +false + +julia> isdispatchtuple(Tuple{DataType}) +false + +julia> isdispatchtuple(Tuple{Type{Int}}) +true + +julia> isdispatchtuple(Tuple{Type}) +false +``` """ isdispatchtuple(@nospecialize(t)) = (@_total_meta; isa(t, DataType) && (t.flags & 0x0004) == 0x0004) @@ -902,7 +978,40 @@ function isidentityfree(@nospecialize(t)) return false end +""" + Base.iskindtype(T) + +Determine whether `T` is a kind, that is, the type of a Julia type: +a [`DataType`](@ref), [`Union`](@ref), [`UnionAll`](@ref), +or [`Core.TypeofBottom`](@ref). + +All kinds are [concrete](@ref isconcretetype) because types are Julia values. +""" iskindtype(@nospecialize t) = (t === DataType || t === UnionAll || t === Union || t === typeof(Bottom)) + +""" + Base.isconcretedispatch(T) + +Returns true if `T` is a [concrete type](@ref isconcretetype) that could appear +as an element of a [dispatch tuple](@ref isdispatchtuple). + +See also: [`isdispatchtuple`](@ref). + +# Examples +```jldoctest +julia> Base.isconcretedispatch(Int) +true + +julia> Base.isconcretedispatch(Number) +false + +julia> Base.isconcretedispatch(DataType) +false + +julia> Base.isconcretedispatch(Type{Int}) +false +``` +""" isconcretedispatch(@nospecialize t) = isconcretetype(t) && !iskindtype(t) using Core: has_free_typevars @@ -925,6 +1034,16 @@ Determine whether type `T` is a concrete type, meaning it could have direct inst Note that this is not the negation of `isabstracttype(T)`. If `T` is not a type, then return `false`. +!!! note + While concrete types are not [abstract](@ref isabstracttype) and + vice versa, types can be neither concrete nor abstract (for example, + `Vector` (a [`UnionAll`](@ref))). + +!!! note + `T` must be the exact type that would be returned from `typeof`. It is + possible for a type `U` to exist such that `T == U`, `isconcretetype(T)`, + but `!isconcretetype(U)`. + See also: [`isbits`](@ref), [`isabstracttype`](@ref), [`issingletontype`](@ref). # Examples @@ -935,6 +1054,9 @@ false julia> isconcretetype(Complex{Float32}) true +julia> isconcretetype(Vector) +false + julia> isconcretetype(Vector{Complex}) true @@ -946,6 +1068,9 @@ false julia> isconcretetype(Union{Int,String}) false + +julia> isconcretetype(Tuple{T} where T<:Int) +false ``` """ isconcretetype(@nospecialize(t)) = (@_total_meta; isa(t, DataType) && (t.flags & 0x0002) == 0x0002) @@ -955,9 +1080,15 @@ isconcretetype(@nospecialize(t)) = (@_total_meta; isa(t, DataType) && (t.flags & Determine whether type `T` was declared as an abstract type (i.e. using the `abstract type` syntax). -Note that this is not the negation of `isconcretetype(T)`. If `T` is not a type, then return `false`. +!!! note + While abstract types are not [concrete](@ref isconcretetype) and + vice versa, types can be neither concrete nor abstract (for example, + `Vector` (a [`UnionAll`](@ref))). + +See also: [`isconcretetype`](@ref). + # Examples ```jldoctest julia> isabstracttype(AbstractArray) @@ -1009,10 +1140,25 @@ morespecific(@nospecialize(a), @nospecialize(b)) = (@_total_meta; ccall(:jl_type morespecific(a::Method, b::Method) = ccall(:jl_method_morespecific, Cint, (Any, Any), a, b) != 0 """ - fieldoffset(type, i) + fieldoffset(type, name::Symbol | i::Integer) + +The byte offset of a field (specified by name or index) of a type relative to its start. + +# Examples +```jldoctest +julia> struct Foo + x::Int64 + y::String + end + +julia> fieldoffset(Foo, 2) +0x0000000000000008 -The byte offset of field `i` of a type relative to the data start. For example, we could -use it in the following manner to summarize information about a struct: +julia> fieldoffset(Foo, :x) +0x0000000000000000 +``` + +We can use it to summarize information about a struct: ```jldoctest julia> structinfo(T) = [(fieldoffset(T,i), fieldname(T,i), fieldtype(T,i)) for i = 1:fieldcount(T)]; @@ -1034,8 +1180,12 @@ julia> structinfo(Base.Filesystem.StatStruct) (0x0000000000000060, :ctime, Float64) (0x0000000000000068, :ioerrno, Int32) ``` + +!!! compat "Julia 1.13" + Specifying the field by name rather than index requires Julia 1.13 or later. """ fieldoffset(x::DataType, idx::Integer) = (@_foldable_meta; ccall(:jl_get_field_offset, Csize_t, (Any, Cint), x, idx)) +fieldoffset(x::DataType, name::Symbol) = fieldoffset(x, fieldindex(x, name)) """ fieldtype(T, name::Symbol | index::Int) @@ -1059,7 +1209,7 @@ String fieldtype """ - Base.fieldindex(T, name::Symbol, err:Bool=true) + fieldindex(T, name::Symbol, err:Bool=true) Get the index of a named field, throwing an error if the field does not exist (when err==true) or returning 0 (when err==false). @@ -1071,14 +1221,20 @@ julia> struct Foo y::String end -julia> Base.fieldindex(Foo, :z) +julia> fieldindex(Foo, :y) +2 + +julia> fieldindex(Foo, :z) ERROR: FieldError: type Foo has no field `z`, available fields: `x`, `y` Stacktrace: [...] -julia> Base.fieldindex(Foo, :z, false) +julia> fieldindex(Foo, :z, false) 0 ``` + +!!! compat "Julia 1.13" + This function is exported as of Julia 1.13. """ function fieldindex(T::DataType, name::Symbol, err::Bool=true) return err ? _fieldindex_maythrow(T, name) : _fieldindex_nothrow(T, name) @@ -1118,7 +1274,7 @@ function datatype_fieldcount(t::DataType) return length(names) end if types isa DataType && types <: Tuple - return fieldcount(types) + return datatype_fieldcount(types) end return nothing elseif isabstracttype(t) @@ -1283,8 +1439,14 @@ max_world(m::Core.CodeInfo) = m.max_world """ get_world_counter() -Returns the current maximum world-age counter. This counter is global and monotonically +Returns the current maximum world-age counter. This counter is monotonically increasing. + +!!! warning + This counter is global and may change at any time between invocations. + In general, most reflection functions operate on the current task's world + age, rather than the global maximum world age. See [`tls_world_age`](@ref) + as well as the [manual chapter of world age](@ref man-world-age). """ get_world_counter() = ccall(:jl_get_world_counter, UInt, ()) @@ -1295,6 +1457,8 @@ Returns the world the [current_task()](@ref) is executing within. """ tls_world_age() = ccall(:jl_get_tls_world_age, UInt, ()) +get_require_world() = unsafe_load(cglobal(:jl_require_world, UInt)) + """ propertynames(x, private=false) @@ -1333,14 +1497,14 @@ hasproperty(x, s::Symbol) = s in propertynames(x) Make method `m` uncallable and force recompilation of any methods that use(d) it. """ function delete_method(m::Method) - ccall(:jl_method_table_disable, Cvoid, (Any, Any), get_methodtable(m), m) + ccall(:jl_method_table_disable, Cvoid, (Any,), m) end # type for reflecting and pretty-printing a subset of methods mutable struct MethodList <: AbstractArray{Method,1} ms::Array{Method,1} - mt::Core.MethodTable + tn::Core.TypeName # contains module.singletonname globalref for altering some aspects of printing end size(m::MethodList) = size(m.ms) @@ -1351,25 +1515,17 @@ function MethodList(mt::Core.MethodTable) visit(mt) do m push!(ms, m) end - return MethodList(ms, mt) + return MethodList(ms, Any.name) end -function matches_to_methods(ms::Array{Any,1}, mt::Core.MethodTable, mod) +function matches_to_methods(ms::Array{Any,1}, tn::Core.TypeName, mod) # Lack of specialization => a comprehension triggers too many invalidations via _collect, so collect the methods manually ms = Method[(ms[i]::Core.MethodMatch).method for i in 1:length(ms)] - # Remove shadowed methods with identical type signatures - prev = nothing - filter!(ms) do m - l = prev - repeated = (l isa Method && m.sig == l.sig) - prev = m - return !repeated - end - # Remove methods not part of module (after removing shadowed methods) + # Remove methods not part of module mod === nothing || filter!(ms) do m return parentmodule(m) ∈ mod end - return MethodList(ms, mt) + return MethodList(ms, tn) end """ @@ -1391,7 +1547,7 @@ function methods(@nospecialize(f), @nospecialize(t), world = get_world_counter() world == typemax(UInt) && error("code reflection cannot be used from generated functions") ms = _methods(f, t, -1, world)::Vector{Any} - return matches_to_methods(ms, typeof(f).name.mt, mod) + return matches_to_methods(ms, typeof(f).name, mod) end methods(@nospecialize(f), @nospecialize(t), mod::Module) = methods(f, t, (mod,)) @@ -1402,7 +1558,7 @@ function methods_including_ambiguous(@nospecialize(f), @nospecialize(t)) min = RefValue{UInt}(typemin(UInt)) max = RefValue{UInt}(typemax(UInt)) ms = _methods_by_ftype(tt, nothing, -1, world, true, min, max, Ptr{Int32}(C_NULL))::Vector{Any} - return matches_to_methods(ms, typeof(f).name.mt, nothing) + return matches_to_methods(ms, typeof(f).name, nothing) end function methods(@nospecialize(f), @@ -1455,6 +1611,15 @@ end _uncompressed_ir(codeinst::CodeInstance, s::String) = ccall(:jl_uncompress_ir, Ref{CodeInfo}, (Any, Any, Any), codeinst.def.def::Method, codeinst, s) +function get_ci_mi(codeinst::CodeInstance) + def = codeinst.def + if def isa Core.ABIOverride + return def.def + else + return def::MethodInstance + end +end + """ Base.generating_output([incremental::Bool])::Bool @@ -1591,20 +1756,24 @@ end function get_nospecializeinfer_sig(method::Method, @nospecialize(atype), sparams::SimpleVector) isa(atype, DataType) || return method.sig - mt = ccall(:jl_method_get_table, Any, (Any,), method) - mt === nothing && return method.sig - return ccall(:jl_normalize_to_compilable_sig, Any, (Any, Any, Any, Any, Cint), - mt, atype, sparams, method, #=int return_if_compileable=#0) + return ccall(:jl_normalize_to_compilable_sig, Any, (Any, Any, Any, Cint), + atype, sparams, method, #=int return_if_compileable=#0) end is_nospecialized(method::Method) = method.nospecialize ≠ 0 is_nospecializeinfer(method::Method) = method.nospecializeinfer && is_nospecialized(method) + +""" +Return MethodInstance corresponding to `atype` and `sparams`. + +No widening / narrowing / compileable-normalization of `atype` is performed. +""" function specialize_method(method::Method, @nospecialize(atype), sparams::SimpleVector; preexisting::Bool=false) @inline if isa(atype, UnionAll) atype, sparams = normalize_typevars(method, atype, sparams) end - if is_nospecializeinfer(method) + if is_nospecializeinfer(method) # TODO: this shouldn't be here atype = get_nospecializeinfer_sig(method, atype, sparams) end if preexisting diff --git a/base/ryu/shortest.jl b/base/ryu/shortest.jl index c1ec648bfacdd..b4ba8255e0b8a 100644 --- a/base/ryu/shortest.jl +++ b/base/ryu/shortest.jl @@ -196,6 +196,7 @@ integer. If a `maxsignif` argument is provided, then `b < maxsignif`. e10 = 0 if maxsignif !== nothing && b > maxsignif + roundup = false b_allzero = true # reduce to max significant digits while true @@ -250,8 +251,10 @@ function writeshortest(buf::AbstractVector{UInt8}, pos, x::T, pos += 1 end if precision == -1 - @inbounds buf[pos] = UInt8('0') - pos += 1 + if hash + @inbounds buf[pos] = UInt8('0') + pos += 1 + end if typed && x isa Float32 @inbounds buf[pos] = UInt8('f') @inbounds buf[pos + 1] = UInt8('0') diff --git a/base/set.jl b/base/set.jl index 0427550c6da54..8b8f3d44603c8 100644 --- a/base/set.jl +++ b/base/set.jl @@ -169,7 +169,7 @@ copymutable(s::Set{T}) where {T} = Set{T}(s) # Set is the default mutable fall-back copymutable(s::AbstractSet{T}) where {T} = Set{T}(s) -sizehint!(s::Set, newsz; shrink::Bool=true) = (sizehint!(s.dict, newsz; shrink); s) +sizehint!(s::Set, newsz::Integer; shrink::Bool=true) = (sizehint!(s.dict, newsz; shrink); s) empty!(s::Set) = (empty!(s.dict); s) rehash!(s::Set) = (rehash!(s.dict); s) @@ -259,7 +259,7 @@ _unique_from(itr, out, seen, i) = unique_from(itr, out, seen, i) return out end -unique(r::AbstractRange) = allunique(r) ? r : oftype(r, r[begin:begin]) +unique(r::AbstractRange) = allunique(r) ? r : oftype(r, r[begin]:r[begin]) """ unique(f, itr) @@ -717,7 +717,7 @@ If `count` is specified, then replace at most `count` occurrences in total. See also [`replace`](@ref replace(A, old_new::Pair...)). # Examples -```jldoctest +```jldoctest; filter = r"^\\s+\\d\$"m julia> replace!([1, 2, 1, 3], 1=>0, 2=>4, count=2) 4-element Vector{Int64}: 0 diff --git a/base/shell.jl b/base/shell.jl index e07fff128acfe..68925cbd5d5af 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -344,7 +344,7 @@ function shell_escape_csh(io::IO, args::AbstractString...) end shell_escape_csh(args::AbstractString...) = sprint(shell_escape_csh, args...; - sizehint = sum(sizeof.(args)) + length(args) * 3) + sizehint = sum(sizeof, args) + length(args) * 3) """ shell_escape_wincmd(s::AbstractString) @@ -494,4 +494,4 @@ function escape_microsoft_c_args(io::IO, args::AbstractString...) end escape_microsoft_c_args(args::AbstractString...) = sprint(escape_microsoft_c_args, args...; - sizehint = (sum(sizeof.(args)) + 3*length(args))) + sizehint = (sum(sizeof, args) + 3*length(args))) diff --git a/base/show.jl b/base/show.jl index 7dfe36fb696c5..870b2f47fe4a6 100644 --- a/base/show.jl +++ b/base/show.jl @@ -39,16 +39,16 @@ end function _isself(ft::DataType) ftname = ft.name - isdefined(ftname, :mt) || return false - name = ftname.mt.name - mod = parentmodule(ft) # NOTE: not necessarily the same as ft.name.mt.module - return invokelatest(isdefinedglobal, mod, name) && ft == typeof(invokelatest(getglobal, mod, name)) + name = ftname.singletonname + ftname.name === name && return false + mod = parentmodule(ft) + return isdefinedglobal(mod, name) && ft === typeof(getglobal(mod, name)) end function show(io::IO, ::MIME"text/plain", f::Function) get(io, :compact, false)::Bool && return show(io, f) ft = typeof(f) - name = ft.name.mt.name + name = ft.name.singletonname if isa(f, Core.IntrinsicFunction) print(io, f) id = Core.Intrinsics.bitcast(Int32, f) @@ -390,12 +390,12 @@ julia> io = IOBuffer(); julia> printstyled(IOContext(io, :color => true), "string", color=:red) -julia> String(take!(io)) +julia> takestring!(io) "\\e[31mstring\\e[39m" julia> printstyled(io, "string", color=:red) -julia> String(take!(io)) +julia> takestring!(io) "string" ``` @@ -442,7 +442,7 @@ get(io::IO, key, default) = default keys(io::IOContext) = keys(io.dict) keys(io::IO) = keys(ImmutableDict{Symbol,Any}()) -displaysize(io::IOContext) = haskey(io, :displaysize) ? io[:displaysize]::Tuple{Int,Int} : displaysize(io.io) +displaysize(io::IOContext) = haskey(io, :displaysize) ? io[:displaysize]::Tuple{Int,Int} : displaysize(io.io)::Tuple{Int,Int} show_circular(io::IO, @nospecialize(x)) = false function show_circular(io::IOContext, @nospecialize(x)) @@ -538,26 +538,24 @@ function active_module() return invokelatest(active_module, active_repl)::Module end -module UsesCoreAndBaseOnly +module UsesBaseOnly end function show_function(io::IO, f::Function, compact::Bool, fallback::Function) - ft = typeof(f) - mt = ft.name.mt - if mt === Symbol.name.mt - # uses shared method table + fname = typeof(f).name + if fname.name === fname.singletonname fallback(io, f) elseif compact - print(io, mt.name) - elseif isdefined(mt, :module) && isdefinedglobal(mt.module, mt.name) && - getglobal(mt.module, mt.name) === f + print(io, fname.singletonname) + elseif isdefined(fname, :module) && isdefinedglobal(fname.module, fname.singletonname) && isconst(fname.module, fname.singletonname) && + getglobal(fname.module, fname.singletonname) === f # this used to call the removed internal function `is_exported_from_stdlib`, which effectively - # just checked for exports from Core and Base. - mod = get(io, :module, UsesCoreAndBaseOnly) - if !(isvisible(mt.name, mt.module, mod) || mt.module === mod) - print(io, mt.module, ".") + # just checked for exports from Base. + mod = get(io, :module, UsesBaseOnly) + if !(isvisible(fname.singletonname, fname.module, mod) || fname.module === mod) + print(io, fname.module, ".") end - show_sym(io, mt.name) + show_sym(io, fname.singletonname) else fallback(io, f) end @@ -634,8 +632,8 @@ function make_typealias(@nospecialize(x::Type)) x isa UnionAll && push!(xenv, x) for mod in mods for name in unsorted_names(mod) - if isdefined(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) - alias = getfield(mod, name) + if isdefinedglobal(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) + alias = getglobal(mod, name) if alias isa Type && !has_free_typevars(alias) && !print_without_params(alias) && x <: alias if alias isa UnionAll (ti, env) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), x, alias)::SimpleVector @@ -827,7 +825,7 @@ function make_typealiases(@nospecialize(x::Type)) Any === x && return aliases, Union{} x <: Tuple && return aliases, Union{} mods = modulesof!(Set{Module}(), x) - Core in mods && push!(mods, Base) + replace!(mods, Core=>Base) vars = Dict{Symbol,TypeVar}() xenv = UnionAll[] each = Any[] @@ -838,14 +836,14 @@ function make_typealiases(@nospecialize(x::Type)) x isa UnionAll && push!(xenv, x) for mod in mods for name in unsorted_names(mod) - if isdefined(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) - alias = getfield(mod, name) + if isdefinedglobal(mod, name) && !isdeprecated(mod, name) && isconst(mod, name) + alias = getglobal(mod, name) if alias isa Type && !has_free_typevars(alias) && !print_without_params(alias) && !(alias <: Tuple) (ti, env) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), x, alias)::SimpleVector ti === Union{} && continue # make sure this alias wasn't from an unrelated part of the Union mod2 = modulesof!(Set{Module}(), alias) - mod in mod2 || (mod === Base && Core in mods) || continue + mod in mod2 || (mod === Base && Core in mod2) || continue env = env::SimpleVector applied = alias if !isempty(env) @@ -965,7 +963,7 @@ function show(io::IO, ::MIME"text/plain", @nospecialize(x::Type)) # give a helpful hint for function types if x isa DataType && x !== UnionAll && !(get(io, :compact, false)::Bool) tn = x.name::Core.TypeName - globname = isdefined(tn, :mt) ? tn.mt.name : nothing + globname = tn.singletonname if is_global_function(tn, globname) print(io, " (singleton type of function ") show_sym(io, globname) @@ -1048,11 +1046,11 @@ function isvisible(sym::Symbol, parent::Module, from::Module) end function is_global_function(tn::Core.TypeName, globname::Union{Symbol,Nothing}) - if globname !== nothing + if globname !== nothing && isconcretetype(tn.wrapper) && tn !== DataType.name # ignore that typeof(DataType)===DataType, since it is valid but not useful globname_str = string(globname::Symbol) - if ('#' ∉ globname_str && '@' ∉ globname_str && isdefined(tn, :module) && - isdefinedglobal(tn.module, globname) && - isconcretetype(tn.wrapper) && isa(getglobal(tn.module, globname), tn.wrapper)) + if '#' ∉ globname_str && '@' ∉ globname_str && isdefined(tn, :module) && + isdefinedglobal(tn.module, globname) && isconst(tn.module, globname) && + isa(getglobal(tn.module, globname), tn.wrapper) return true end end @@ -1083,7 +1081,7 @@ function show_type_name(io::IO, tn::Core.TypeName) # intercept this case and print `UnionAll` instead. return print(io, "UnionAll") end - globname = isdefined(tn, :mt) ? tn.mt.name : nothing + globname = tn.singletonname globfunc = is_global_function(tn, globname) sym = (globfunc ? globname : tn.name)::Symbol globfunc && print(io, "typeof(") @@ -1782,8 +1780,8 @@ end const keyword_syms = Set([ :baremodule, :begin, :break, :catch, :const, :continue, :do, :else, :elseif, - :end, :export, :false, :finally, :for, :function, :global, :if, :import, - :let, :local, :macro, :module, :public, :quote, :return, :struct, :true, + :end, :export, :var"false", :finally, :for, :function, :global, :if, :import, + :let, :local, :macro, :module, :public, :quote, :return, :struct, :var"true", :try, :using, :while ]) function is_valid_identifier(sym) @@ -2567,10 +2565,10 @@ function show_signature_function(io::IO, @nospecialize(ft), demangle=false, farg uw = unwrap_unionall(ft) if ft <: Function && isa(uw, DataType) && isempty(uw.parameters) && _isself(uw) uwmod = parentmodule(uw) - if qualified && !isexported(uwmod, uw.name.mt.name) && uwmod !== Main + if qualified && !isexported(uwmod, uw.name.singletonname) && uwmod !== Main print_within_stacktrace(io, uwmod, '.', bold=true) end - s = sprint(show_sym, (demangle ? demangle_function_name : identity)(uw.name.mt.name), context=io) + s = sprint(show_sym, (demangle ? demangle_function_name : identity)(uw.name.singletonname), context=io) print_within_stacktrace(io, s, bold=true) elseif isType(ft) && (f = ft.parameters[1]; !isa(f, TypeVar)) uwf = unwrap_unionall(f) @@ -2649,7 +2647,7 @@ function show_tuple_as_call(out::IO, name::Symbol, sig::Type; end print_within_stacktrace(io, ")", bold=true) show_method_params(io, tv) - str = String(take!(buf)) + str = takestring!(buf) str = type_limited_string_from_context(out, str) print(out, str) nothing @@ -2758,7 +2756,7 @@ function type_depth_limit(str::String, n::Int; maxdepth = nothing) end prev = di end - return String(take!(output)) + return unsafe_takestring!(output) end function print_type_bicolor(io, type; kwargs...) @@ -3193,7 +3191,7 @@ summary(io::IO, x) = print(io, typeof(x)) function summary(x) io = IOBuffer() summary(io, x) - String(take!(io)) + takestring!(io) end ## `summary` for AbstractArrays @@ -3414,6 +3412,9 @@ function print_partition(io::IO, partition::Core.BindingPartition) elseif kind == PARTITION_KIND_CONST print(io, "constant binding to ") print(io, partition_restriction(partition)) + elseif kind == PARTITION_KIND_CONST_IMPORT + print(io, "constant binding (declared with `import`) to ") + print(io, partition_restriction(partition)) elseif kind == PARTITION_KIND_UNDEF_CONST print(io, "undefined const binding") elseif kind == PARTITION_KIND_GUARD diff --git a/base/slicearray.jl b/base/slicearray.jl index 7318181c1e826..1928020a1155a 100644 --- a/base/slicearray.jl +++ b/base/slicearray.jl @@ -25,7 +25,7 @@ struct Slices{P,SM,AX,S,N} <: AbstractSlices{S,N} """ parent::P """ - A tuple of length `ndims(parent)`, denoting how each dimension should be handled: + A tuple of length at least `ndims(parent)`, denoting how each dimension should be handled: - an integer `i`: this is the `i`th dimension of the outer `Slices` object. - `:`: an "inner" dimension """ @@ -39,34 +39,39 @@ end unitaxis(::AbstractArray) = Base.OneTo(1) function Slices(A::P, slicemap::SM, ax::AX) where {P,SM,AX} + length(slicemap) >= ndims(A) || + throw(ArgumentError("Slices cannot be constructed with a slicemap of fewer elements than the parent has dimensions")) N = length(ax) - argT = map((a,l) -> l === (:) ? Colon : eltype(a), axes(A), slicemap) + parent_axes = ntuple(d -> axes(A, d), length(slicemap)) + argT = map((a,l) -> l === (:) ? Colon : eltype(a), parent_axes, slicemap) S = Base.promote_op(view, P, argT...) Slices{P,SM,AX,S,N}(A, slicemap, ax) end _slice_check_dims(N) = nothing function _slice_check_dims(N, dim, dims...) - 1 <= dim <= N || throw(DimensionMismatch("Invalid dimension $dim")) + 1 <= dim || throw(DimensionMismatch("Invalid dimension $dim")) dim in dims && throw(DimensionMismatch("Dimensions $dims are not unique")) _slice_check_dims(N,dims...) end @constprop :aggressive function _eachslice(A::AbstractArray{T,N}, dims::NTuple{M,Integer}, drop::Bool) where {T,N,M} _slice_check_dims(N,dims...) + N_ = foldl(max, dims; init=N) + if drop # if N = 4, dims = (3,1) then # axes = (axes(A,3), axes(A,1)) # slicemap = (2, :, 1, :) ax = map(dim -> axes(A,dim), dims) - slicemap = ntuple(dim -> something(findfirst(isequal(dim), dims), (:)), N) + slicemap = ntuple(dim -> something(findfirst(isequal(dim), dims), (:)), N_) return Slices(A, slicemap, ax) else # if N = 4, dims = (3,1) then # axes = (axes(A,1), OneTo(1), axes(A,3), OneTo(1)) # slicemap = (1, :, 3, :) - ax = ntuple(dim -> dim in dims ? axes(A,dim) : unitaxis(A), N) - slicemap = ntuple(dim -> dim in dims ? dim : (:), N) + ax = ntuple(dim -> dim in dims ? axes(A,dim) : unitaxis(A), N_) + slicemap = ntuple(dim -> dim in dims ? dim : (:), N_) return Slices(A, slicemap, ax) end end @@ -225,7 +230,6 @@ constructed by [`eachcol`](@ref). const ColumnSlices{P<:AbstractMatrix,AX,S<:AbstractVector} = Slices{P,Tuple{Colon,Int},AX,S,1} -IteratorSize(::Type{Slices{P,SM,AX,S,N}}) where {P,SM,AX,S,N} = HasShape{N}() axes(s::Slices) = s.axes size(s::Slices) = map(length, s.axes) diff --git a/base/sort.jl b/base/sort.jl index f5709a368256c..bfa65d2459680 100644 --- a/base/sort.jl +++ b/base/sort.jl @@ -96,7 +96,7 @@ function issorted(itr; end end -function partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}, o::Ordering) +function partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}, o::Ordering; scratch::Union{Nothing, Vector} = nothing) # TODO move k from `alg` to `kw` # Don't perform InitialOptimizations before Bracketing. The optimizations take O(n) # time and so does the whole sort. But do perform them before recursive calls because @@ -105,7 +105,7 @@ function partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}, o::Orde _sort!(v, BoolOptimization( Small{12}( # Very small inputs should go straight to insertion sort BracketedSort(k))), - o, (;)) + o, (; scratch)) maybeview(v, k) end @@ -115,7 +115,7 @@ maybeview(v, k::Integer) = v[k] """ partialsort!(v, k; by=identity, lt=isless, rev=false) -Partially sort the vector `v` in place so that the value at index `k` (or +Mutate the vector `v` so that the value at index `k` (or range of adjacent values if `k` is a range) occurs at the position where it would appear if the array were fully sorted. If `k` is a single index, that value is returned; if `k` is a range, an array of values at those indices is @@ -166,8 +166,8 @@ julia> a ``` """ partialsort!(v::AbstractVector, k::Union{Integer,OrdinalRange}; - lt=isless, by=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) = - partialsort!(v, k, ord(lt,by,rev,order)) + lt=isless, by=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward, kws...) = + partialsort!(v, k, ord(lt,by,rev,order); kws...) """ partialsort(v, k, by=identity, lt=isless, rev=false) @@ -1625,10 +1625,11 @@ defalg(v) = DEFAULT_STABLE """ sort!(v; alg::Base.Sort.Algorithm=Base.Sort.defalg(v), lt=isless, by=identity, rev::Bool=false, order::Base.Order.Ordering=Base.Order.Forward) -Sort the vector `v` in place. A stable algorithm is used by default: the -ordering of elements that compare equal is preserved. A specific algorithm can -be selected via the `alg` keyword (see [Sorting Algorithms](@ref) for available -algorithms). +Mutate the vector `v` so that it is sorted. + +A stable algorithm is used by default: the ordering of elements that +compare equal is preserved. A specific algorithm can be selected via the +`alg` keyword (see [Sorting Algorithms](@ref) for available algorithms). Elements are first transformed with the function `by` and then compared according to either the function `lt` or the ordering `order`. Finally, the @@ -1745,7 +1746,7 @@ end Variant of [`sort!`](@ref) that returns a sorted copy of `v` leaving `v` itself unmodified. When calling `sort` on the [`keys`](@ref) or [`values](@ref) of a dictionary, `v` is -collected and then sorted in place. +collected and then sorted. !!! compat "Julia 1.12" Sorting `NTuple`s requires Julia 1.12 or later. @@ -2289,7 +2290,7 @@ UIntMappable(T::Type, order::ReverseOrdering) = UIntMappable(T, order.fwd) ### Vectors -# Convert v to unsigned integers in place, maintaining sort order. +# Convert v to unsigned integers in-place, maintaining sort order. function uint_map!(v::AbstractVector, lo::Integer, hi::Integer, order::Ordering) u = reinterpret(UIntMappable(eltype(v), order), v) @inbounds for i in lo:hi diff --git a/base/special/exp.jl b/base/special/exp.jl index 38d7509807aed..f423551b02f29 100644 --- a/base/special/exp.jl +++ b/base/special/exp.jl @@ -224,7 +224,7 @@ end twopk = (k + UInt64(53)) << 52 return reinterpret(T, twopk + reinterpret(UInt64, small_part))*0x1p-53 end - #k == 1024 && return (small_part * 2.0) * 2.0^1023 + #k == 1024 && return (small_part * 2.0) * 0x1p1023 end twopk = Int64(k) << 52 return reinterpret(T, twopk + reinterpret(Int64, small_part)) @@ -252,7 +252,7 @@ end twopk = (k + UInt64(53)) << 52 return reinterpret(T, twopk + reinterpret(UInt64, small_part))*0x1p-53 end - k == 1024 && return (small_part * 2.0) * 2.0^1023 + k == 1024 && return (small_part * 2.0) * 0x1p1023 end twopk = Int64(k) << 52 return reinterpret(T, twopk + reinterpret(Int64, small_part)) @@ -491,7 +491,7 @@ end expm1(x) Accurately compute ``e^x-1``. It avoids the loss of precision involved in the direct -evaluation of exp(x)-1 for small values of x. +evaluation of exp(x) - 1 for small values of x. # Examples ```jldoctest julia> expm1(1e-16) diff --git a/base/special/trig.jl b/base/special/trig.jl index 66e4b46d7d489..96a2da5f5b968 100644 --- a/base/special/trig.jl +++ b/base/special/trig.jl @@ -824,7 +824,7 @@ Compute ``\\cos(\\pi x)`` more accurately than `cos(pi*x)`, especially for large Throw a [`DomainError`](@ref) if `isinf(x)`, return a `T(NaN)` if `isnan(x)`. -See also: [`cispi`](@ref), [`sincosd`](@ref), [`cospi`](@ref). +See also: [`cispi`](@ref), [`sincosd`](@ref), [`sinpi`](@ref). """ function cospi(x::T) where T<:IEEEFloat x = abs(x) diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 7ba23f6e715dc..8f2866af170f7 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -24,7 +24,7 @@ Stack information representing execution context, with the following fields: - `linfo::Union{Method, Core.MethodInstance, Core.CodeInstance, Core.CodeInfo, Nothing}` - The Method, MethodInstance, CodeInstance, or CodeInfo containing the execution context (if it could be found), \ + The Method, MethodInstance, CodeInstance, or CodeInfo containing the execution context (if it could be found), or nothing (for example, if the inlining was a result of macro expansion). - `file::Symbol` @@ -90,7 +90,7 @@ function ==(a::StackFrame, b::StackFrame) end function hash(frame::StackFrame, h::UInt) - h += 0xf4fbda67fe20ce88 % UInt + h ⊻= 0xf4fbda67fe20ce88 % UInt h = hash(frame.line, h) h = hash(frame.file, h) h = hash(frame.func, h) @@ -273,8 +273,12 @@ function show_spec_linfo(io::IO, frame::StackFrame) if linfo isa Union{MethodInstance, CodeInstance} def = frame_method_or_module(frame) if def isa Module - Base.show_mi(io, linfo, #=from_stackframe=#true) + Base.show_mi(io, linfo::MethodInstance, #=from_stackframe=#true) + elseif linfo isa CodeInstance && linfo.owner !== nothing + show_custom_spec_sig(io, linfo.owner, linfo, frame) else + # Equivalent to the default implementation of `show_custom_spec_sig` + # for `linfo isa CodeInstance`, but saves an extra dynamic dispatch. show_spec_sig(io, def, frame_mi(frame).specTypes) end else @@ -284,6 +288,12 @@ function show_spec_linfo(io::IO, frame::StackFrame) end end +# Can be extended by compiler packages to customize backtrace display of custom code instance frames +function show_custom_spec_sig(io::IO, @nospecialize(owner), linfo::CodeInstance, frame::StackFrame) + mi = Base.get_ci_mi(linfo) + return show_spec_sig(io, mi.def, mi.specTypes) +end + function show_spec_sig(io::IO, m::Method, @nospecialize(sig::Type)) if get(io, :limit, :false)::Bool if !haskey(io, :displaysize) diff --git a/base/stat.jl b/base/stat.jl index 36496db4df415..fbab5126d39bc 100644 --- a/base/stat.jl +++ b/base/stat.jl @@ -183,7 +183,8 @@ show(io::IO, ::MIME"text/plain", st::StatStruct) = show_statstruct(io, st, false # stat & lstat functions -checkstat(s::StatStruct) = Int(s.ioerrno) in (0, Base.UV_ENOENT, Base.UV_ENOTDIR, Base.UV_EINVAL) ? s : uv_error(string("stat(", repr(s.desc), ")"), s.ioerrno) +checkstat(s::StatStruct) = Int(s.ioerrno) in (0, Base.UV_ENOENT, Base.UV_ENOTDIR, Base.UV_EINVAL) ? s : + _uv_error(string("stat(", repr(s.desc), ")"), s.ioerrno) macro stat_call(sym, arg1type, arg) return quote @@ -319,7 +320,7 @@ function filemode_string(mode) end complete && write(str, "-") end - return String(take!(str)) + return unsafe_takestring!(str) end """ diff --git a/base/staticdata.jl b/base/staticdata.jl index 94526cc4f7bd3..a9ba58f3d82aa 100644 --- a/base/staticdata.jl +++ b/base/staticdata.jl @@ -1,24 +1,68 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -module StaticData - -using Core: CodeInstance, MethodInstance -using Base: get_world_counter +using .Core: CodeInstance, MethodInstance +using .Base: JLOptions, Compiler, get_world_counter, _methods_by_ftype, get_methodtable, get_ci_mi, morespecific const WORLD_AGE_REVALIDATION_SENTINEL::UInt = 1 const _jl_debug_method_invalidation = Ref{Union{Nothing,Vector{Any}}}(nothing) debug_method_invalidation(onoff::Bool) = _jl_debug_method_invalidation[] = onoff ? Any[] : nothing -function get_ci_mi(codeinst::CodeInstance) - def = codeinst.def - if def isa Core.ABIOverride - return def.def - else - return def::MethodInstance +# Immutable structs for different categories of state data +struct VerifyMethodInitialState + codeinst::CodeInstance + mi::MethodInstance + def::Method + callees::Core.SimpleVector +end + +struct VerifyMethodWorkState + depth::Int + cause::CodeInstance + recursive_index::Int + stage::Symbol +end + +struct VerifyMethodResultState + child_cycle::Int + result_minworld::UInt + result_maxworld::UInt +end + +# Container for all the work arrays +struct VerifyMethodWorkspace + # Arrays of different state categories + initial_states::Vector{VerifyMethodInitialState} + work_states::Vector{VerifyMethodWorkState} + result_states::Vector{VerifyMethodResultState} + + # Tarjan's algorithm working data + stack::Vector{CodeInstance} + visiting::IdDict{CodeInstance,Int} + + function VerifyMethodWorkspace() + new(VerifyMethodInitialState[], VerifyMethodWorkState[], VerifyMethodResultState[], + CodeInstance[], IdDict{CodeInstance,Int}()) end end +# Helper functions to create default states +function VerifyMethodInitialState(codeinst::CodeInstance) + mi = get_ci_mi(codeinst) + def = mi.def::Method + callees = codeinst.edges + VerifyMethodInitialState(codeinst, mi, def, callees) +end + +function VerifyMethodWorkState(dummy_cause::CodeInstance) + VerifyMethodWorkState(0, dummy_cause, 1, :init_and_process_callees) +end + +function VerifyMethodResultState() + VerifyMethodResultState(0, 0, 0) +end + + # Restore backedges to external targets # `edges` = [caller1, ...], the list of worklist-owned code instances internally # `ext_ci_list` = [caller1, ...], the list of worklist-owned code instances externally @@ -26,27 +70,26 @@ function insert_backedges(edges::Vector{Any}, ext_ci_list::Union{Nothing,Vector{ # determine which CodeInstance objects are still valid in our image # to enable any applicable new codes backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt) - methods_with_invalidated_source = Base.scan_new_methods(extext_methods, internal_methods, backedges_only) - stack = CodeInstance[] - visiting = IdDict{CodeInstance,Int}() - _insert_backedges(edges, stack, visiting, methods_with_invalidated_source) + Base.scan_new_methods!(extext_methods, internal_methods, backedges_only) + workspace = VerifyMethodWorkspace() + _insert_backedges(edges, workspace) if ext_ci_list !== nothing - _insert_backedges(ext_ci_list, stack, visiting, methods_with_invalidated_source, #=external=#true) + _insert_backedges(ext_ci_list, workspace, #=external=#true) end end -function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, external::Bool=false) +function _insert_backedges(edges::Vector{Any}, workspace::VerifyMethodWorkspace, external::Bool=false) for i = 1:length(edges) codeinst = edges[i]::CodeInstance validation_world = get_world_counter() - verify_method_graph(codeinst, stack, visiting, mwis, validation_world) + verify_method_graph(codeinst, validation_world, workspace) # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining # validity. @ccall jl_promote_ci_to_current(codeinst::Any, validation_world::UInt)::Cvoid minvalid = codeinst.min_world maxvalid = codeinst.max_world - # Finally, if this CI is still valid in some world age and and belongs to an external method(specialization), + # Finally, if this CI is still valid in some world age and belongs to an external method(specialization), # poke it that mi's cache if maxvalid ≥ minvalid && external caller = get_ci_mi(codeinst) @@ -63,189 +106,465 @@ function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visi end end -function verify_method_graph(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, validation_world::UInt) - @assert isempty(stack); @assert isempty(visiting); - child_cycle, minworld, maxworld = verify_method(codeinst, stack, visiting, mwis, validation_world) +function verify_method_graph(codeinst::CodeInstance, validation_world::UInt, workspace::VerifyMethodWorkspace) + @assert isempty(workspace.stack); @assert isempty(workspace.visiting); + @assert isempty(workspace.initial_states); @assert isempty(workspace.work_states); @assert isempty(workspace.result_states) + child_cycle, minworld, maxworld = verify_method(codeinst, validation_world, workspace) @assert child_cycle == 0 - @assert isempty(stack); @assert isempty(visiting); + @assert isempty(workspace.stack); @assert isempty(workspace.visiting); + @assert isempty(workspace.initial_states); @assert isempty(workspace.work_states); @assert isempty(workspace.result_states) nothing end -get_require_world() = unsafe_load(cglobal(:jl_require_world, UInt)) +function gen_staged_sig(def::Method, mi::MethodInstance) + isdefined(def, :generator) || return nothing + isdispatchtuple(mi.specTypes) || return nothing + gen = Core.Typeof(def.generator) + return Tuple{gen, UInt, Method, Vararg} + ## more precise method lookup, but more costly and likely not actually better? + #tts = (mi.specTypes::DataType).parameters + #sps = Any[Core.Typeof(mi.sparam_vals[i]) for i in 1:length(mi.sparam_vals)] + #if def.isva + # return Tuple{gen, UInt, Method, sps..., tts[1:def.nargs - 1]..., Tuple{tts[def.nargs - 1:end]...}} + #else + # return Tuple{gen, UInt, Method, sps..., tts...} + #end +end + +function needs_instrumentation(codeinst::CodeInstance, mi::MethodInstance, def::Method, validation_world::UInt) + if JLOptions().code_coverage != 0 || JLOptions().malloc_log != 0 + # test if the code needs to run with instrumentation, in which case we cannot use existing generated code + if isdefined(def, :debuginfo) ? # generated_only functions do not have debuginfo, so fall back to considering their codeinst debuginfo though this may be slower and less reliable + Compiler.should_instrument(def.module, def.debuginfo) : + isdefined(codeinst, :debuginfo) && Compiler.should_instrument(def.module, codeinst.debuginfo) + return true + end + gensig = gen_staged_sig(def, mi) + if gensig !== nothing + # if this is defined by a generator, try to consider forcing re-running the generators too, to add coverage for them + minworld = Ref{UInt}(1) + maxworld = Ref{UInt}(typemax(UInt)) + has_ambig = Ref{Int32}(0) + result = _methods_by_ftype(gensig, nothing, -1, validation_world, #=ambig=#false, minworld, maxworld, has_ambig) + if result !== nothing + for k = 1:length(result) + match = result[k]::Core.MethodMatch + genmethod = match.method + # no, I refuse to refuse to recurse into your cursed generated function generators and will only test one level deep here + if isdefined(genmethod, :debuginfo) && Compiler.should_instrument(genmethod.module, genmethod.debuginfo) + return true + end + end + end + end + end + return false +end # Test all edges relevant to a method: # - Visit the entire call graph, starting from edges[idx] to determine if that method is valid # - Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable # and slightly modified with an early termination option once the computation reaches its minimum -function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, validation_world::UInt) - world = codeinst.min_world - let max_valid2 = codeinst.max_world - if max_valid2 ≠ WORLD_AGE_REVALIDATION_SENTINEL - return 0, world, max_valid2 - end - end - # Implicitly referenced bindings in the current module do not get explicit edges. - # If they were invalidated, they'll be in `mwis`. If they weren't, they imply a minworld - # of `get_require_world`. In principle, this is only required for methods that do reference - # an implicit globalref. However, we already don't perform this validation for methods that - # don't have any (implicit or explicit) edges at all. The remaining corner case (some explicit, - # but no implicit edges) is rare and there would be little benefit to lower the minworld for it - # in any case, so we just always use `get_require_world` here. - local minworld::UInt, maxworld::UInt = get_require_world(), validation_world - def = get_ci_mi(codeinst).def - @assert def isa Method - if haskey(visiting, codeinst) - return visiting[codeinst], minworld, maxworld - end - push!(stack, codeinst) - depth = length(stack) - visiting[codeinst] = depth - # TODO JL_TIMING(VERIFY_IMAGE, VERIFY_Methods) - callees = codeinst.edges - # Check for invalidation of the implicit edges from GlobalRef in the Method source - if def in mwis - maxworld = 0 - invalidations = _jl_debug_method_invalidation[] - if invalidations !== nothing - push!(invalidations, def, "method_globalref", codeinst, nothing) - end - end - # verify current edges - if isempty(callees) - # quick return: no edges to verify (though we probably shouldn't have gotten here from WORLD_AGE_REVALIDATION_SENTINEL) - elseif maxworld == get_require_world() - # if no new worlds were allocated since serializing the base module, then no new validation is worth doing right now either - else - j = 1 - while j ≤ length(callees) - local min_valid2::UInt, max_valid2::UInt - edge = callees[j] - @assert !(edge isa Method) # `Method`-edge isn't allowed for the optimized one-edge format - if edge isa CodeInstance - edge = get_ci_mi(edge) - end - if edge isa MethodInstance - sig = typeintersect((edge.def::Method).sig, edge.specTypes) # TODO?? - min_valid2, max_valid2, matches = verify_call(sig, callees, j, 1, world) - j += 1 - elseif edge isa Int - sig = callees[j+1] - min_valid2, max_valid2, matches = verify_call(sig, callees, j+2, edge, world) - j += 2 + edge - edge = sig - elseif edge isa Core.Binding - j += 1 - min_valid2 = minworld - max_valid2 = maxworld - if !Base.binding_was_invalidated(edge) - if isdefined(edge, :partitions) - min_valid2 = edge.partitions.min_world - max_valid2 = edge.partitions.max_world - end - else - # Binding was previously invalidated - min_valid2 = 1 - max_valid2 = 0 - end - matches = nothing - else - callee = callees[j+1] - if callee isa Core.MethodTable # skip the legacy edge (missing backedge) - j += 2 +function verify_method(codeinst::CodeInstance, validation_world::UInt, workspace::VerifyMethodWorkspace) + # Initialize root state + push!(workspace.initial_states, VerifyMethodInitialState(codeinst)) + push!(workspace.work_states, VerifyMethodWorkState(codeinst)) + push!(workspace.result_states, VerifyMethodResultState()) + + current_depth = 1 # == length(workspace._states) == end + while true + # Get current state indices + initial = workspace.initial_states[current_depth] + work = workspace.work_states[current_depth] + + if work.stage == :init_and_process_callees + # Initialize state and handle early returns + world = initial.codeinst.min_world + let max_valid2 = initial.codeinst.max_world + if max_valid2 ≠ WORLD_AGE_REVALIDATION_SENTINEL + workspace.result_states[current_depth] = VerifyMethodResultState(0, world, max_valid2) + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, work.recursive_index, :return_to_parent) continue end - if callee isa CodeInstance - callee = get_ci_mi(callee) - end - if callee isa MethodInstance - meth = callee.def::Method - else - meth = callee::Method + end + + if needs_instrumentation(initial.codeinst, initial.mi, initial.def, validation_world) + workspace.result_states[current_depth] = VerifyMethodResultState(0, world, UInt(0)) + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, work.recursive_index, :return_to_parent) + continue + end + + minworld, maxworld = Base.get_require_world(), validation_world + + if haskey(workspace.visiting, initial.codeinst) + workspace.result_states[current_depth] = VerifyMethodResultState(workspace.visiting[initial.codeinst], minworld, maxworld) + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, work.recursive_index, :return_to_parent) + continue + end + + push!(workspace.stack, initial.codeinst) + depth = length(workspace.stack) + workspace.visiting[initial.codeinst] = depth + + # Check for invalidation of GlobalRef edges + if (initial.def.did_scan_source & 0x1) == 0x0 + backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt) + Base.scan_new_method!(initial.def, backedges_only) + end + if (initial.def.did_scan_source & 0x4) != 0x0 + maxworld = 0 + invalidations = _jl_debug_method_invalidation[] + if invalidations !== nothing + push!(invalidations, initial.def, "method_globalref", initial.codeinst, nothing) end - min_valid2, max_valid2 = verify_invokesig(edge, meth, world) - matches = nothing - j += 2 end - if minworld < min_valid2 - minworld = min_valid2 + + # Process all non-CodeInstance edges + if !isempty(initial.callees) && maxworld != Base.get_require_world() + matches = [] + j = 1 + while j <= length(initial.callees) + local min_valid2::UInt, max_valid2::UInt + edge = initial.callees[j] + @assert !(edge isa Method) + + if edge isa CodeInstance + # Convert CodeInstance to MethodInstance for validation (like original) + edge = get_ci_mi(edge) + end + + if edge isa MethodInstance + sig = edge.specTypes + min_valid2, max_valid2 = verify_call(sig, initial.callees, j, 1, world, true, matches) + j += 1 + elseif edge isa Int + sig = initial.callees[j+1] + nmatches = abs(edge) + fully_covers = edge > 0 + min_valid2, max_valid2 = verify_call(sig, initial.callees, j+2, nmatches, world, fully_covers, matches) + j += 2 + nmatches + edge = sig + elseif edge isa Core.Binding + j += 1 + min_valid2 = minworld + max_valid2 = maxworld + if !Base.binding_was_invalidated(edge) + if isdefined(edge, :partitions) + min_valid2 = edge.partitions.min_world + max_valid2 = edge.partitions.max_world + end + else + min_valid2 = 1 + max_valid2 = 0 + end + else + callee = initial.callees[j+1] + if callee isa Core.MethodTable + j += 2 + continue + end + if callee isa CodeInstance + callee = get_ci_mi(callee) + end + if callee isa MethodInstance + meth = callee.def::Method + else + meth = callee::Method + end + min_valid2, max_valid2 = verify_invokesig(edge, meth, world, matches) + j += 2 + end + + if minworld < min_valid2 + minworld = min_valid2 + end + if maxworld > max_valid2 + maxworld = max_valid2 + end + invalidations = _jl_debug_method_invalidation[] + if max_valid2 ≠ typemax(UInt) && invalidations !== nothing + push!(invalidations, edge, "insert_backedges_callee", initial.codeinst, copy(matches)) + end + if max_valid2 == 0 && invalidations === nothing + break + end + end end - if maxworld > max_valid2 - maxworld = max_valid2 + + # Store computed minworld/maxworld in result state and transition to recursive phase + workspace.result_states[current_depth] = VerifyMethodResultState(depth, minworld, maxworld) + workspace.work_states[current_depth] = VerifyMethodWorkState(depth, work.cause, 1, :recursive_phase) + + elseif work.stage == :recursive_phase + # Find next CodeInstance edge that needs processing + recursive_index = work.recursive_index + found_child = false + while recursive_index ≤ length(initial.callees) + edge = initial.callees[recursive_index] + recursive_index += 1 + + if edge isa CodeInstance + # Create child state and add to stack + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, recursive_index, :recursive_phase) + push!(workspace.initial_states, VerifyMethodInitialState(edge)) + push!(workspace.work_states, VerifyMethodWorkState(edge)) + push!(workspace.result_states, VerifyMethodResultState()) + current_depth += 1 + found_child = true + break + end end - invalidations = _jl_debug_method_invalidation[] - if max_valid2 ≠ typemax(UInt) && invalidations !== nothing - push!(invalidations, edge, "insert_backedges_callee", codeinst, matches) + + if !found_child + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, recursive_index, :cleanup) end - if max_valid2 == 0 && invalidations === nothing - break + + elseif work.stage == :cleanup + # If we are the top of the current cycle, now mark all other parts of + # our cycle with what we found. + # Or if we found a failed edge, also mark all of the other parts of the + # cycle as also having a failed edge. + result = workspace.result_states[current_depth] + if result.result_maxworld == 0 || result.child_cycle == work.depth + while length(workspace.stack) ≥ work.depth + child = pop!(workspace.stack) + if result.result_maxworld ≠ 0 + @atomic :monotonic child.min_world = result.result_minworld + end + @atomic :monotonic child.max_world = result.result_maxworld + if result.result_maxworld == validation_world && validation_world == get_world_counter() + Compiler.store_backedges(child, child.edges) + end + @assert workspace.visiting[child] == length(workspace.stack) + 1 + delete!(workspace.visiting, child) + invalidations = _jl_debug_method_invalidation[] + if invalidations !== nothing && result.result_maxworld < validation_world + push!(invalidations, child, "verify_methods", work.cause) + end + end + + workspace.result_states[current_depth] = VerifyMethodResultState(0, result.result_minworld, result.result_maxworld) end - end - end - # verify recursive edges (if valid, or debugging) - cycle = depth - cause = codeinst - if maxworld ≠ 0 || _jl_debug_method_invalidation[] !== nothing - for j = 1:length(callees) - edge = callees[j] - if !(edge isa CodeInstance) - continue + + workspace.work_states[current_depth] = VerifyMethodWorkState(work.depth, work.cause, work.recursive_index, :return_to_parent) + + elseif work.stage == :return_to_parent + # Pass results to parent and process them + pop!(workspace.initial_states) + pop!(workspace.work_states) + result = pop!(workspace.result_states) + current_depth -= 1 + if current_depth == 0 # Return results from the root call + return (result.child_cycle, result.result_minworld, result.result_maxworld) end - callee = edge - local min_valid2::UInt, max_valid2::UInt - child_cycle, min_valid2, max_valid2 = verify_method(callee, stack, visiting, mwis, validation_world) - if minworld < min_valid2 - minworld = min_valid2 + # Propagate results to parent + parent_work = workspace.work_states[current_depth] + parent_result = workspace.result_states[current_depth] + callee = initial.codeinst + child_cycle, min_valid2, max_valid2 = result.child_cycle, result.result_minworld, result.result_maxworld + parent_cycle = parent_result.child_cycle + parent_minworld = parent_result.result_minworld + parent_maxworld = parent_result.result_maxworld + parent_cause = parent_work.cause + parent_stage = parent_work.stage + if parent_minworld < min_valid2 + parent_minworld = min_valid2 end - if minworld > max_valid2 + if parent_minworld > max_valid2 max_valid2 = 0 end - if maxworld > max_valid2 - cause = callee - maxworld = max_valid2 + if parent_maxworld > max_valid2 + parent_cause = callee + parent_maxworld = max_valid2 end if max_valid2 == 0 # found what we were looking for, so terminate early - break - elseif child_cycle ≠ 0 && child_cycle < cycle + # The parent should break out of its loop in :recursive_phase + parent_stage = :cleanup + elseif child_cycle ≠ 0 && child_cycle < parent_cycle # record the cycle will resolve at depth "cycle" - cycle = child_cycle + parent_cycle = child_cycle end + workspace.work_states[current_depth] = VerifyMethodWorkState(parent_work.depth, parent_cause, parent_work.recursive_index, parent_stage) + workspace.result_states[current_depth] = VerifyMethodResultState(parent_cycle, parent_minworld, parent_maxworld) end end - if maxworld ≠ 0 && cycle ≠ depth - return cycle, minworld, maxworld +end + +function get_method_from_edge(@nospecialize t) + if t isa Method + return t + else + if t isa CodeInstance + t = get_ci_mi(t)::MethodInstance + else + t = t::MethodInstance + end + return t.def::Method end - # If we are the top of the current cycle, now mark all other parts of - # our cycle with what we found. - # Or if we found a failed edge, also mark all of the other parts of the - # cycle as also having a failed edge. - while length(stack) ≥ depth - child = pop!(stack) - if maxworld ≠ 0 - @atomic :monotonic child.min_world = minworld +end + +# Check if method2 is in method1's interferences set +# Returns true if method2 is found (meaning !morespecific(method1, method2)) +function method_in_interferences(method2::Method, method1::Method) + interferences = method1.interferences + for k = 1:length(interferences) + isassigned(interferences, k) || break + interference_method = interferences[k]::Method + if interference_method === method2 + return true end - @atomic :monotonic child.max_world = maxworld - if maxworld == validation_world && validation_world == get_world_counter() - Base.Compiler.store_backedges(child, child.edges) + end + return false +end + +# Check if method1 is more specific than method2 via the interference graph +function method_morespecific_via_interferences(method1::Method, method2::Method) + if method1 === method2 + return false + end + ms = method_in_interferences_recursive(method1, method2, IdSet{Method}()) + # slow check: @assert ms === morespecific(method1, method2) || typeintersect(method1.sig, method2.sig) === Union{} || typeintersect(method2.sig, method1.sig) === Union{} + return ms +end + +# Returns true if method1 is in method2's interferences (meaning !morespecific(method2, method1)) +function method_in_interferences_recursive(method1::Method, method2::Method, visited::IdSet{Method}) + if method_in_interferences(method2, method1) + return false + end + if method_in_interferences(method1, method2) + return true + end + + # Recursively check through interference graph + method2 in visited && return false + push!(visited, method2) + interferences = method2.interferences + for k = 1:length(interferences) + isassigned(interferences, k) || break + method3 = interferences[k]::Method + if method_in_interferences(method2, method3) + continue # only follow edges to morespecific methods in search of the morespecific target (skip ambiguities) end - @assert visiting[child] == length(stack) + 1 - delete!(visiting, child) - invalidations = _jl_debug_method_invalidation[] - if invalidations !== nothing && maxworld < validation_world - push!(invalidations, child, "verify_methods", cause) + if method_in_interferences_recursive(method1, method3, visited) + return true # found method1 in the interference graph end end - return 0, minworld, maxworld + + return false end -function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt) +function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt, fully_covers::Bool, matches::Vector{Any}) # verify that these edges intersect with the same methods as before + mi = nothing + expected_deleted = false + for j = 1:n + t = expecteds[i+j-1] + meth = get_method_from_edge(t) + if iszero(meth.dispatch_status & METHOD_SIG_LATEST_WHICH) + expected_deleted = true + break + end + end + if expected_deleted + if _jl_debug_method_invalidation[] === nothing && world == get_world_counter() + return UInt(1), UInt(0) + end + elseif n == 1 + # first, fast-path a check if the expected method simply dominates its sig anyways + # so the result of ml_matches is already simply known + let t = expecteds[i], meth, minworld, maxworld + meth = get_method_from_edge(t) + if !(t isa Method) + if t isa CodeInstance + mi = get_ci_mi(t)::MethodInstance + else + mi = t::MethodInstance + end + # Fast path is legal when fully_covers=true + if fully_covers && !iszero(mi.dispatch_status & METHOD_SIG_LATEST_ONLY) + minworld = meth.primary_world + @assert minworld ≤ world + maxworld = typemax(UInt) + return minworld, maxworld + end + end + # Fast path is legal when fully_covers=true + if fully_covers && !iszero(meth.dispatch_status & METHOD_SIG_LATEST_ONLY) + minworld = meth.primary_world + @assert minworld ≤ world + maxworld = typemax(UInt) + return minworld, maxworld + end + end + elseif n > 1 + # Try the interference set fast path: check if all interference sets are covered by expecteds + interference_fast_path_success = fully_covers + # If it didn't fail yet, then check that all interference methods are either expected, or not applicable. + if interference_fast_path_success + local interference_minworld::UInt = 1 + for j = 1:n + meth = get_method_from_edge(expecteds[i+j-1]) + if interference_minworld < meth.primary_world + interference_minworld = meth.primary_world + end + interferences = meth.interferences + for k = 1:length(interferences) + isassigned(interferences, k) || break # no more entries + interference_method = interferences[k]::Method + if iszero(interference_method.dispatch_status & METHOD_SIG_LATEST_WHICH) + # detected a deleted interference_method, so need the full lookup to compute minworld + interference_fast_path_success = false + break + end + world < interference_method.primary_world && break # this and later entries are for a future world + local found_in_expecteds = false + for j = 1:n + if interference_method === get_method_from_edge(expecteds[i+j-1]) + found_in_expecteds = true + break + end + end + if !found_in_expecteds + ti = typeintersect(sig, interference_method.sig) + if !(ti === Union{}) + # try looking for a different expected method that fully covers this interference_method anyways over their intersection + for j = 1:n + meth2 = get_method_from_edge(expecteds[i+j-1]) + if method_morespecific_via_interferences(meth2, interference_method) && ti <: meth2.sig + found_in_expecteds = true + break + end + end + if !found_in_expecteds + meth2 = get_method_from_edge(expecteds[i]) + interference_fast_path_success = false + break + end + end + end + end + if !interference_fast_path_success + break + end + end + if interference_fast_path_success + # All interference sets are covered by expecteds, can return success + @assert interference_minworld ≤ world + maxworld = typemax(UInt) + return interference_minworld, maxworld + end + end + end + # next, compare the current result of ml_matches to the old result lim = _jl_debug_method_invalidation[] !== nothing ? Int(typemax(Int32)) : n minworld = Ref{UInt}(1) maxworld = Ref{UInt}(typemax(UInt)) has_ambig = Ref{Int32}(0) - result = Base._methods_by_ftype(sig, nothing, lim, world, #=ambig=#false, minworld, maxworld, has_ambig) + result = _methods_by_ftype(sig, nothing, lim, world, #=ambig=#false, minworld, maxworld, has_ambig) if result === nothing + empty!(matches) maxworld[] = 0 else # setdiff!(result, expected) @@ -258,17 +577,7 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n local found = false for j = 1:n t = expecteds[i+j-1] - if t isa Method - meth = t - else - if t isa CodeInstance - t = get_ci_mi(t) - else - t = t::MethodInstance - end - meth = t.def::Method - end - if match.method == meth + if match.method == get_method_from_edge(t) found = true break end @@ -287,39 +596,48 @@ function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n end if maxworld[] ≠ typemax(UInt) && _jl_debug_method_invalidation[] !== nothing resize!(result, ins) + copy!(matches, result) end end - return minworld[], maxworld[], result + if maxworld[] == typemax(UInt) && mi isa MethodInstance + ccall(:jl_promote_mi_to_current, Cvoid, (Any, UInt, UInt), mi, minworld[], world) + end + return minworld[], maxworld[] end -function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UInt) +# fast-path dispatch_status bit definitions (false indicates unknown) +# true indicates this method would be returned as the result from `which` when invoking `method.sig` in the current latest world +const METHOD_SIG_LATEST_WHICH = 0x1 +# true indicates this method would be returned as the only result from `methods` when calling `method.sig` in the current latest world +const METHOD_SIG_LATEST_ONLY = 0x2 + +function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UInt, matches::Vector{Any}) @assert invokesig isa Type local minworld::UInt, maxworld::UInt - if invokesig === expected.sig - # the invoke match is `expected` for `expected->sig`, unless `expected` is invalid + empty!(matches) + if invokesig === expected.sig && !iszero(expected.dispatch_status & METHOD_SIG_LATEST_WHICH) + # the invoke match is `expected` for `expected->sig`, unless `expected` is replaced minworld = expected.primary_world - maxworld = expected.deleted_world @assert minworld ≤ world - if maxworld < world - maxworld = 0 - end - else - minworld = 1 maxworld = typemax(UInt) - mt = Base.get_methodtable(expected) + else + mt = get_methodtable(expected) if mt === nothing + minworld = 1 maxworld = 0 else - matched, valid_worlds = Base.Compiler._findsup(invokesig, mt, world) + matched, valid_worlds = Compiler._findsup(invokesig, mt, world) minworld, maxworld = valid_worlds.min_world, valid_worlds.max_world if matched === nothing maxworld = 0 - elseif matched.method != expected - maxworld = 0 + else + matched = matched.method + push!(matches, matched) + if matched !== expected + maxworld = 0 + end end end end return minworld, maxworld end - -end # module StaticData diff --git a/base/stream.jl b/base/stream.jl index 67af2bff3682c..d88f164f3ccca 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -318,7 +318,7 @@ end """ open(fd::OS_HANDLE)::IO -Take a raw file descriptor wrap it in a Julia-aware IO type, +Take a raw file descriptor and wrap it in a Julia-aware IO type, and take ownership of the fd handle. Call `open(Libc.dup(fd))` to avoid the ownership capture of the original handle. @@ -615,9 +615,9 @@ end ## BUFFER ## ## Allocate space in buffer (for immediate use) function alloc_request(buffer::IOBuffer, recommended_size::UInt) - ensureroom(buffer, Int(recommended_size)) + ensureroom(buffer, recommended_size) ptr = buffer.append ? buffer.size + 1 : buffer.ptr - nb = min(length(buffer.data)-buffer.offset, buffer.maxsize) + buffer.offset - ptr + 1 + nb = min(length(buffer.data), buffer.maxsize + get_offset(buffer)) - ptr + 1 return (Ptr{Cvoid}(pointer(buffer.data, ptr)), nb) end @@ -943,8 +943,7 @@ function readbytes!(s::LibuvStream, a::Vector{UInt8}, nb::Int) nread = readbytes!(sbuf, a, nb) else initsize = length(a) - newbuf = PipeBuffer(a, maxsize=nb) - newbuf.size = newbuf.offset # reset the write pointer to the beginning + newbuf = _truncated_pipebuffer(a; maxsize=nb) nread = try s.buffer = newbuf write(newbuf, sbuf) @@ -991,8 +990,7 @@ function unsafe_read(s::LibuvStream, p::Ptr{UInt8}, nb::UInt) if bytesavailable(sbuf) >= nb unsafe_read(sbuf, p, nb) else - newbuf = PipeBuffer(unsafe_wrap(Array, p, nb), maxsize=Int(nb)) - newbuf.size = newbuf.offset # reset the write pointer to the beginning + newbuf = _truncated_pipebuffer(unsafe_wrap(Array, p, nb); maxsize=Int(nb)) try s.buffer = newbuf write(newbuf, sbuf) @@ -1600,8 +1598,7 @@ function readbytes!(s::BufferStream, a::Vector{UInt8}, nb::Int) nread = readbytes!(sbuf, a, nb) else initsize = length(a) - newbuf = PipeBuffer(a, maxsize=nb) - newbuf.size = newbuf.offset # reset the write pointer to the beginning + newbuf = _truncated_pipebuffer(a; maxsize=nb) nread = try s.buffer = newbuf write(newbuf, sbuf) diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index 5142c32e1b70d..cb2d09f5b7054 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -272,7 +272,7 @@ function annotatedstring(xs...) print(s, x) end end - str = String(take!(buf)) + str = takestring!(buf) AnnotatedString(str, annotations) end @@ -457,204 +457,112 @@ function annotated_chartransform(f::Function, str::AnnotatedString, state=nothin stop_offset = last(offsets[findlast(<=(stop) ∘ first, offsets)::Int]) push!(annots, setindex(annot, (start + start_offset):(stop + stop_offset), :region)) end - AnnotatedString(String(take!(outstr)), annots) + AnnotatedString(takestring!(outstr), annots) end -## AnnotatedIOBuffer - -struct AnnotatedIOBuffer <: AbstractPipe - io::IOBuffer - annotations::Vector{RegionAnnotation} -end - -AnnotatedIOBuffer(io::IOBuffer) = AnnotatedIOBuffer(io, Vector{RegionAnnotation}()) -AnnotatedIOBuffer() = AnnotatedIOBuffer(IOBuffer()) - -function show(io::IO, aio::AnnotatedIOBuffer) - show(io, AnnotatedIOBuffer) - size = filesize(aio.io) - print(io, '(', size, " byte", ifelse(size == 1, "", "s"), ", ", - length(aio.annotations), " annotation", ifelse(length(aio.annotations) == 1, "", "s"), ")") +struct RegionIterator{S <: AbstractString} + str::S + regions::Vector{UnitRange{Int}} + annotations::Vector{Vector{Annotation}} end -pipe_reader(io::AnnotatedIOBuffer) = io.io -pipe_writer(io::AnnotatedIOBuffer) = io.io - -# Useful `IOBuffer` methods that we don't get from `AbstractPipe` -position(io::AnnotatedIOBuffer) = position(io.io) -seek(io::AnnotatedIOBuffer, n::Integer) = (seek(io.io, n); io) -seekend(io::AnnotatedIOBuffer) = (seekend(io.io); io) -skip(io::AnnotatedIOBuffer, n::Integer) = (skip(io.io, n); io) -copy(io::AnnotatedIOBuffer) = AnnotatedIOBuffer(copy(io.io), copy(io.annotations)) - -annotations(io::AnnotatedIOBuffer) = io.annotations - -annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = - (_annotate!(io.annotations, range, label, val); io) - -function write(io::AnnotatedIOBuffer, astr::Union{AnnotatedString, SubString{<:AnnotatedString}}) - astr = AnnotatedString(astr) - offset = position(io.io) - eof(io) || _clear_annotations_in_region!(io.annotations, offset+1:offset+ncodeunits(astr)) - _insert_annotations!(io, astr.annotations) - write(io.io, String(astr)) -end +Base.length(si::RegionIterator) = length(si.regions) -write(io::AnnotatedIOBuffer, c::AnnotatedChar) = - write(io, AnnotatedString(string(c), [(region=1:ncodeunits(c), a...) for a in c.annotations])) -write(io::AnnotatedIOBuffer, x::AbstractString) = write(io.io, x) -write(io::AnnotatedIOBuffer, s::Union{SubString{String}, String}) = write(io.io, s) -write(io::AnnotatedIOBuffer, b::UInt8) = write(io.io, b) - -function write(dest::AnnotatedIOBuffer, src::AnnotatedIOBuffer) - destpos = position(dest) - isappending = eof(dest) - srcpos = position(src) - nb = write(dest.io, src.io) - isappending || _clear_annotations_in_region!(dest.annotations, destpos:destpos+nb) - srcannots = [setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region) - for annot in src.annotations if first(annot.region) >= srcpos] - _insert_annotations!(dest, srcannots, destpos - srcpos) - nb +Base.@propagate_inbounds function Base.iterate(si::RegionIterator, i::Integer=1) + if i <= length(si.regions) + @inbounds ((SubString(si.str, si.regions[i]), si.annotations[i]), i+1) + end end -# So that read/writes with `IOContext` (and any similar `AbstractPipe` wrappers) -# work as expected. -function write(io::AbstractPipe, s::Union{AnnotatedString, SubString{<:AnnotatedString}}) - if pipe_writer(io) isa AnnotatedIOBuffer - write(pipe_writer(io), s) - else - invoke(write, Tuple{IO, typeof(s)}, io, s) - end::Int -end -# Can't be part of the `Union` above because it introduces method ambiguities -function write(io::AbstractPipe, c::AnnotatedChar) - if pipe_writer(io) isa AnnotatedIOBuffer - write(pipe_writer(io), c) - else - invoke(write, Tuple{IO, typeof(c)}, io, c) - end::Int -end +Base.eltype(::RegionIterator{S}) where { S <: AbstractString} = + Tuple{SubString{S}, Vector{Annotation}} """ - _clear_annotations_in_region!(annotations::Vector{$RegionAnnotation}, span::UnitRange{Int}) + eachregion(s::AnnotatedString{S}) + eachregion(s::SubString{AnnotatedString{S}}) -Erase the presence of `annotations` within a certain `span`. +Identify the contiguous substrings of `s` with a constant annotations, and return +an iterator which provides each substring and the applicable annotations as a +`Tuple{SubString{S}, Vector{$Annotation}}`. -This operates by removing all elements of `annotations` that are entirely -contained in `span`, truncating ranges that partially overlap, and splitting -annotations that subsume `span` to just exist either side of `span`. +# Examples + +```jldoctest; setup=:(using Base: AnnotatedString, eachregion) +julia> collect(eachregion(AnnotatedString( + "hey there", [(1:3, :face, :bold), + (5:9, :face, :italic)]))) +3-element Vector{Tuple{SubString{String}, Vector{$Annotation}}}: + ("hey", [$Annotation((:face, :bold))]) + (" ", []) + ("there", [$Annotation((:face, :italic))]) +``` """ -function _clear_annotations_in_region!(annotations::Vector{RegionAnnotation}, span::UnitRange{Int}) - # Clear out any overlapping pre-existing annotations. - filter!(ann -> first(ann.region) < first(span) || last(ann.region) > last(span), annotations) - extras = Tuple{Int, RegionAnnotation}[] - for i in eachindex(annotations) - annot = annotations[i] - region = annot.region - # Test for partial overlap - if first(region) <= first(span) <= last(region) || first(region) <= last(span) <= last(region) - annotations[i] = - setindex(annot, - if first(region) < first(span) - first(region):first(span)-1 - else - last(span)+1:last(region) - end, - :region) - # If `span` fits exactly within `region`, then we've only copied over - # the beginning overhang, but also need to conserve the end overhang. - if first(region) < first(span) && last(span) < last(region) - push!(extras, (i, setindex(annot, last(span)+1:last(region), :region))) - end +function eachregion(s::AnnotatedString, subregion::UnitRange{Int}=firstindex(s):lastindex(s)) + isempty(s) || isempty(subregion) && + return RegionIterator(s.string, UnitRange{Int}[], Vector{Annotation}[]) + events = annotation_events(s, subregion) + isempty(events) && return RegionIterator(s.string, [subregion], [Annotation[]]) + annotvals = Annotation[ + (; label, value) for (; label, value) in annotations(s)] + regions = Vector{UnitRange{Int}}() + annots = Vector{Vector{Annotation}}() + pos = first(events).pos + if pos > first(subregion) + push!(regions, thisind(s, first(subregion)):prevind(s, pos)) + push!(annots, []) + end + activelist = Int[] + for event in events + if event.pos != pos + push!(regions, pos:prevind(s, event.pos)) + push!(annots, annotvals[activelist]) + pos = event.pos + end + if event.active + insert!(activelist, searchsortedfirst(activelist, event.index), event.index) + else + deleteat!(activelist, searchsortedfirst(activelist, event.index)) end end - # Insert any extra entries in the appropriate position - for (offset, (i, entry)) in enumerate(extras) - insert!(annotations, i + offset, entry) + if last(events).pos < nextind(s, last(subregion)) + push!(regions, last(events).pos:thisind(s, last(subregion))) + push!(annots, []) end - annotations + RegionIterator(s.string, regions, annots) end -""" - _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{$RegionAnnotation}, offset::Int = position(io)) +function eachregion(s::SubString{<:AnnotatedString}, pos::UnitRange{Int}=firstindex(s):lastindex(s)) + if isempty(s) + RegionIterator(s.string, Vector{UnitRange{Int}}(), Vector{Vector{Annotation}}()) + else + eachregion(s.string, first(pos)+s.offset:last(pos)+s.offset) + end +end -Register new `annotations` in `io`, applying an `offset` to their regions. +""" + annotation_events(string::AbstractString, annots::Vector{$RegionAnnotation}, subregion::UnitRange{Int}) + annotation_events(string::AnnotatedString, subregion::UnitRange{Int}) -The largely consists of simply shifting the regions of `annotations` by `offset` -and pushing them onto `io`'s annotations. However, when it is possible to merge -the new annotations with recent annotations in accordance with the semantics -outlined in [`AnnotatedString`](@ref), we do so. More specifically, when there -is a run of the most recent annotations that are also present as the first -`annotations`, with the same value and adjacent regions, the new annotations are -merged into the existing recent annotations by simply extending their range. +Find all annotation "change events" that occur within a `subregion` of `annots`, +with respect to `string`. When `string` is styled, `annots` is inferred. -This is implemented so that one can say write an `AnnotatedString` to an -`AnnotatedIOBuffer` one character at a time without needlessly producing a -new annotation for each character. +Each change event is given in the form of a `@NamedTuple{pos::Int, active::Bool, +index::Int}` where `pos` is the position of the event, `active` is a boolean +indicating whether the annotation is being activated or deactivated, and `index` +is the index of the annotation in question. """ -function _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{RegionAnnotation}, offset::Int = position(io)) - run = 0 - if !isempty(io.annotations) && last(last(io.annotations).region) == offset - for i in reverse(axes(annotations, 1)) - annot = annotations[i] - first(annot.region) == 1 || continue - i <= length(io.annotations) || continue - if annot.label == last(io.annotations).label && annot.value == last(io.annotations).value - valid_run = true - for runlen in 1:i - new = annotations[begin+runlen-1] - old = io.annotations[end-i+runlen] - if last(old.region) != offset || first(new.region) != 1 || old.label != new.label || old.value != new.value - valid_run = false - break - end - end - if valid_run - run = i - break - end - end +function annotation_events(s::AbstractString, annots::Vector{RegionAnnotation}, subregion::UnitRange{Int}) + events = Vector{NamedTuple{(:pos, :active, :index), Tuple{Int, Bool, Int}}}() # Position, Active?, Annotation index + for (i, (; region)) in enumerate(annots) + if !isempty(intersect(subregion, region)) + start, stop = max(first(subregion), first(region)), min(last(subregion), last(region)) + start <= stop || continue # Currently can't handle empty regions + push!(events, (pos=thisind(s, start), active=true, index=i)) + push!(events, (pos=nextind(s, stop), active=false, index=i)) end end - for runindex in 0:run-1 - old_index = lastindex(io.annotations) - run + 1 + runindex - old = io.annotations[old_index] - new = annotations[begin+runindex] - io.annotations[old_index] = setindex(old, first(old.region):last(new.region)+offset, :region) - end - for index in run+1:lastindex(annotations) - annot = annotations[index] - start, stop = first(annot.region), last(annot.region) - push!(io.annotations, setindex(annotations[index], start+offset:stop+offset, :region)) - end + sort(events, by=e -> e.pos) end -function read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{T}}) where {T <: AbstractString} - if (start = position(io)) == 0 - AnnotatedString(read(io.io, T), copy(io.annotations)) - else - annots = [setindex(annot, UnitRange{Int}(max(1, first(annot.region) - start), last(annot.region)-start), :region) - for annot in io.annotations if last(annot.region) > start] - AnnotatedString(read(io.io, T), annots) - end -end -read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{AbstractString}}) = read(io, AnnotatedString{String}) -read(io::AnnotatedIOBuffer, ::Type{AnnotatedString}) = read(io, AnnotatedString{String}) - -function read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{T}}) where {T <: AbstractChar} - pos = position(io) - char = read(io.io, T) - annots = [NamedTuple{(:label, :value)}(annot) for annot in io.annotations if pos+1 in annot.region] - AnnotatedChar(char, annots) -end -read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{AbstractChar}}) = read(io, AnnotatedChar{Char}) -read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar}) = read(io, AnnotatedChar{Char}) - -function truncate(io::AnnotatedIOBuffer, size::Integer) - truncate(io.io, size) - filter!(ann -> first(ann.region) <= size, io.annotations) - map!(ann -> setindex(ann, first(ann.region):min(size, last(ann.region)), :region), - io.annotations, io.annotations) - io -end +annotation_events(s::AnnotatedString, subregion::UnitRange{Int}) = + annotation_events(s.string, annotations(s), subregion) diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl new file mode 100644 index 0000000000000..9698fd5909b68 --- /dev/null +++ b/base/strings/annotated_io.jl @@ -0,0 +1,276 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## AnnotatedIOBuffer + +struct AnnotatedIOBuffer <: AbstractPipe + io::IOBuffer + annotations::Vector{RegionAnnotation} +end + +AnnotatedIOBuffer(io::IOBuffer) = AnnotatedIOBuffer(io, Vector{RegionAnnotation}()) +AnnotatedIOBuffer() = AnnotatedIOBuffer(IOBuffer()) + +function show(io::IO, aio::AnnotatedIOBuffer) + show(io, AnnotatedIOBuffer) + size = filesize(aio.io) + print(io, '(', size, " byte", ifelse(size == 1, "", "s"), ", ", + length(aio.annotations), " annotation", ifelse(length(aio.annotations) == 1, "", "s"), ")") +end + +pipe_reader(io::AnnotatedIOBuffer) = io.io +pipe_writer(io::AnnotatedIOBuffer) = io.io + +# Useful `IOBuffer` methods that we don't get from `AbstractPipe` +position(io::AnnotatedIOBuffer) = position(io.io) +seek(io::AnnotatedIOBuffer, n::Integer) = (seek(io.io, n); io) +seekend(io::AnnotatedIOBuffer) = (seekend(io.io); io) +skip(io::AnnotatedIOBuffer, n::Integer) = (skip(io.io, n); io) +copy(io::AnnotatedIOBuffer) = AnnotatedIOBuffer(copy(io.io), copy(io.annotations)) + +annotations(io::AnnotatedIOBuffer) = io.annotations + +annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = + (_annotate!(io.annotations, range, label, val); io) + +function write(io::AnnotatedIOBuffer, astr::Union{AnnotatedString, SubString{<:AnnotatedString}}) + astr = AnnotatedString(astr) + offset = position(io.io) + eof(io) || _clear_annotations_in_region!(io.annotations, offset+1:offset+ncodeunits(astr)) + _insert_annotations!(io, astr.annotations) + write(io.io, String(astr)) +end + +write(io::AnnotatedIOBuffer, c::AnnotatedChar) = + write(io, AnnotatedString(string(c), [(region=1:ncodeunits(c), a...) for a in c.annotations])) +write(io::AnnotatedIOBuffer, x::AbstractString) = write(io.io, x) +write(io::AnnotatedIOBuffer, s::Union{SubString{String}, String}) = write(io.io, s) +write(io::AnnotatedIOBuffer, b::UInt8) = write(io.io, b) + +function write(dest::AnnotatedIOBuffer, src::AnnotatedIOBuffer) + destpos = position(dest) + isappending = eof(dest) + srcpos = position(src) + nb = write(dest.io, src.io) + isappending || _clear_annotations_in_region!(dest.annotations, destpos:destpos+nb) + srcannots = [setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region) + for annot in src.annotations if first(annot.region) >= srcpos] + _insert_annotations!(dest, srcannots, destpos - srcpos) + nb +end + +# So that read/writes with `IOContext` (and any similar `AbstractPipe` wrappers) +# work as expected. +function write(io::AbstractPipe, s::Union{AnnotatedString, SubString{<:AnnotatedString}}) + if pipe_writer(io) isa AnnotatedIOBuffer + write(pipe_writer(io), s) + else + invoke(write, Tuple{IO, typeof(s)}, io, s) + end::Int +end + +# Can't be part of the `Union` above because it introduces method ambiguities +function write(io::AbstractPipe, c::AnnotatedChar) + if pipe_writer(io) isa AnnotatedIOBuffer + write(pipe_writer(io), c) + else + invoke(write, Tuple{IO, typeof(c)}, io, c) + end::Int +end + +function read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{T}}) where {T <: AbstractString} + if (start = position(io)) == 0 + AnnotatedString(read(io.io, T), copy(io.annotations)) + else + annots = [setindex(annot, UnitRange{Int}(max(1, first(annot.region) - start), last(annot.region)-start), :region) + for annot in io.annotations if last(annot.region) > start] + AnnotatedString(read(io.io, T), annots) + end +end +read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{AbstractString}}) = read(io, AnnotatedString{String}) +read(io::AnnotatedIOBuffer, ::Type{AnnotatedString}) = read(io, AnnotatedString{String}) + +function read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{T}}) where {T <: AbstractChar} + pos = position(io) + char = read(io.io, T) + annots = [NamedTuple{(:label, :value)}(annot) for annot in io.annotations if pos+1 in annot.region] + AnnotatedChar(char, annots) +end +read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{AbstractChar}}) = read(io, AnnotatedChar{Char}) +read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar}) = read(io, AnnotatedChar{Char}) + +function truncate(io::AnnotatedIOBuffer, size::Integer) + truncate(io.io, size) + filter!(ann -> first(ann.region) <= size, io.annotations) + map!(ann -> setindex(ann, first(ann.region):min(size, last(ann.region)), :region), + io.annotations, io.annotations) + io +end + +""" + _clear_annotations_in_region!(annotations::Vector{$RegionAnnotation}, span::UnitRange{Int}) + +Erase the presence of `annotations` within a certain `span`. + +This operates by removing all elements of `annotations` that are entirely +contained in `span`, truncating ranges that partially overlap, and splitting +annotations that subsume `span` to just exist either side of `span`. +""" +function _clear_annotations_in_region!(annotations::Vector{RegionAnnotation}, span::UnitRange{Int}) + # Clear out any overlapping pre-existing annotations. + filter!(ann -> first(ann.region) < first(span) || last(ann.region) > last(span), annotations) + extras = Tuple{Int, RegionAnnotation}[] + for i in eachindex(annotations) + annot = annotations[i] + region = annot.region + # Test for partial overlap + if first(region) <= first(span) <= last(region) || first(region) <= last(span) <= last(region) + annotations[i] = + setindex(annot, + if first(region) < first(span) + first(region):first(span)-1 + else + last(span)+1:last(region) + end, + :region) + # If `span` fits exactly within `region`, then we've only copied over + # the beginning overhang, but also need to conserve the end overhang. + if first(region) < first(span) && last(span) < last(region) + push!(extras, (i, setindex(annot, last(span)+1:last(region), :region))) + end + end + end + # Insert any extra entries in the appropriate position + for (offset, (i, entry)) in enumerate(extras) + insert!(annotations, i + offset, entry) + end + annotations +end + +""" + _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{$RegionAnnotation}, offset::Int = position(io)) + +Register new `annotations` in `io`, applying an `offset` to their regions. + +The largely consists of simply shifting the regions of `annotations` by `offset` +and pushing them onto `io`'s annotations. However, when it is possible to merge +the new annotations with recent annotations in accordance with the semantics +outlined in [`AnnotatedString`](@ref), we do so. More specifically, when there +is a run of the most recent annotations that are also present as the first +`annotations`, with the same value and adjacent regions, the new annotations are +merged into the existing recent annotations by simply extending their range. + +This is implemented so that one can say write an `AnnotatedString` to an +`AnnotatedIOBuffer` one character at a time without needlessly producing a +new annotation for each character. +""" +function _insert_annotations!(io::AnnotatedIOBuffer, annotations::Vector{RegionAnnotation}, offset::Int = position(io)) + run = 0 + if !isempty(io.annotations) && last(last(io.annotations).region) == offset + for i in reverse(axes(annotations, 1)) + annot = annotations[i] + first(annot.region) == 1 || continue + i <= length(io.annotations) || continue + if annot.label == last(io.annotations).label && annot.value == last(io.annotations).value + valid_run = true + for runlen in 1:i + new = annotations[begin+runlen-1] + old = io.annotations[end-i+runlen] + if last(old.region) != offset || first(new.region) != 1 || old.label != new.label || old.value != new.value + valid_run = false + break + end + end + if valid_run + run = i + break + end + end + end + end + for runindex in 0:run-1 + old_index = lastindex(io.annotations) - run + 1 + runindex + old = io.annotations[old_index] + new = annotations[begin+runindex] + io.annotations[old_index] = setindex(old, first(old.region):last(new.region)+offset, :region) + end + for index in run+1:lastindex(annotations) + annot = annotations[index] + start, stop = first(annot.region), last(annot.region) + push!(io.annotations, setindex(annotations[index], start+offset:stop+offset, :region)) + end +end + +# NOTE: This is an interim solution to the invalidations caused +# by the split styled display implementation. This should be +# replaced by a more robust solution (such as a consolidation of +# the type and method definitions) in the near future. +module AnnotatedDisplay + +using ..Base: IO, SubString, AnnotatedString, AnnotatedChar, AnnotatedIOBuffer +using ..Base: eachregion, invoke_in_world, tls_world_age + +# Write + +ansi_write(f::Function, io::IO, x::Any) = f(io, String(x)) + +ansi_write_(f::Function, io::IO, @nospecialize(x::Any)) = + invoke_in_world(tls_world_age(), ansi_write, f, io, x) + +Base.write(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = + ansi_write_(write, io, s)::Int + +Base.write(io::IO, c::AnnotatedChar) = + ansi_write_(write, io, c)::Int + +function Base.write(io::IO, aio::AnnotatedIOBuffer) + if get(io, :color, false) == true + # This does introduce an overhead that technically + # could be avoided, but I'm not sure that it's currently + # worth the effort to implement an efficient version of + # writing from a AnnotatedIOBuffer with style. + # In the meantime, by converting to an `AnnotatedString` we can just + # reuse all the work done to make that work. + ansi_write_(write, io, read(aio, AnnotatedString))::Int + else + write(io, aio.io) + end +end + +# Print + +Base.print(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = + (ansi_write_(write, io, s); nothing) + +Base.print(io::IO, s::AnnotatedChar) = + (ansi_write_(write, io, s); nothing) + +Base.print(io::AnnotatedIOBuffer, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = + (write(io, s); nothing) + +Base.print(io::AnnotatedIOBuffer, c::AnnotatedChar) = + (write(io, c); nothing) + +# Escape + +Base.escape_string(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}, + esc = ""; keep = (), ascii::Bool=false, fullhex::Bool=false) = + (ansi_write_((io, s) -> escape_string(io, s, esc; keep, ascii, fullhex), io, s); nothing) + +# Show + +show_annot(io::IO, ::Any) = nothing +show_annot(io::IO, ::MIME, ::Any) = nothing + +show_annot_(io::IO, @nospecialize(x::Any)) = + invoke_in_world(tls_world_age(), show_annot, io, x)::Nothing + +show_annot_(io::IO, m::MIME, @nospecialize(x::Any)) = + invoke_in_world(tls_world_age(), show_annot, io, m, x)::Nothing + +Base.show(io::IO, m::MIME"text/html", s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = + show_annot_(io, m, s) + +Base.show(io::IO, m::MIME"text/html", c::AnnotatedChar) = + show_annot_(io, m, c) + +end diff --git a/base/strings/basic.jl b/base/strings/basic.jl index b92c177b0ec56..352d42eb3a40f 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -463,7 +463,7 @@ end * Case `n == 1` - If `i` is in bounds in `s` return the index of the start of the character whose + If `i` is in bounds in `str` return the index of the start of the character whose encoding starts before index `i`. In other words, if `i` is the start of a character, return the start of the previous character; if `i` is not the start of a character, rewind until the start of a character and return that index. @@ -522,7 +522,7 @@ end * Case `n == 1` - If `i` is in bounds in `s` return the index of the start of the character whose + If `i` is in bounds in `str` return the index of the start of the character whose encoding starts after index `i`. In other words, if `i` is the start of a character, return the start of the next character; if `i` is not the start of a character, move forward until the start of a character and return that index. @@ -539,7 +539,7 @@ end * Case `n == 0` - Return `i` only if `i` is a valid index in `s` or is equal to `0`. + Return `i` only if `i` is a valid index in `str` or is equal to `0`. Otherwise `StringIndexError` or `BoundsError` is thrown. # Examples @@ -678,7 +678,7 @@ function filter(f, s::AbstractString) for c in s f(c) && write(out, c) end - String(_unsafe_take!(out)) + takestring!(out) end ## string first and last ## @@ -797,7 +797,7 @@ size(s::CodeUnits) = (length(s),) elsize(s::Type{<:CodeUnits{T}}) where {T} = sizeof(T) @propagate_inbounds getindex(s::CodeUnits, i::Int) = codeunit(s.s, i) IndexStyle(::Type{<:CodeUnits}) = IndexLinear() -@inline iterate(s::CodeUnits, i=1) = (i % UInt) - 1 < length(s) ? (@inbounds s[i], i + 1) : nothing +checkbounds(::Type{Bool}, s::CodeUnits, i::Integer) = checkbounds(Bool, s.s, i) write(io::IO, s::CodeUnits) = write(io, s.s) @@ -805,6 +805,8 @@ write(io::IO, s::CodeUnits) = write(io, s.s) cconvert(::Type{Ptr{T}}, s::CodeUnits{T}) where {T} = cconvert(Ptr{T}, s.s) cconvert(::Type{Ptr{Int8}}, s::CodeUnits{UInt8}) = cconvert(Ptr{Int8}, s.s) +similar(::Type{<:CodeUnits{T}}, dims::Dims) where {T} = similar(Array{T}, dims) + """ codeunits(s::AbstractString) diff --git a/base/strings/io.jl b/base/strings/io.jl index 89a5a992cbfd0..f3a0783e98a9b 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -25,7 +25,7 @@ julia> io = IOBuffer(); julia> print(io, "Hello", ' ', :World!) -julia> String(take!(io)) +julia> takestring!(io) "Hello World!" ``` """ @@ -51,7 +51,7 @@ function print(io::IO, xs...) return nothing end -setfield!(typeof(print).name.mt, :max_args, 10, :monotonic) +setfield!(typeof(print).name, :max_args, Int32(10), :monotonic) """ println([io::IO], xs...) @@ -70,13 +70,13 @@ julia> io = IOBuffer(); julia> println(io, "Hello", ',', " world.") -julia> String(take!(io)) +julia> takestring!(io) "Hello, world.\\n" ``` """ println(io::IO, xs...) = print(io, xs..., "\n") -setfield!(typeof(println).name.mt, :max_args, 10, :monotonic) +setfield!(typeof(println).name, :max_args, Int32(10), :monotonic) ## conversion of general objects to strings ## """ @@ -112,7 +112,7 @@ function sprint(f::Function, args...; context=nothing, sizehint::Integer=0) else f(s, args...) end - String(_unsafe_take!(s)) + takestring!(s) end function _str_sizehint(x) @@ -146,9 +146,9 @@ function print_to_string(xs...) for x in xs print(s, x) end - String(_unsafe_take!(s)) + takestring!(s) end -setfield!(typeof(print_to_string).name.mt, :max_args, 10, :monotonic) +setfield!(typeof(print_to_string).name, :max_args, Int32(10), :monotonic) function string_with_env(env, xs...) if isempty(xs) @@ -164,7 +164,7 @@ function string_with_env(env, xs...) for x in xs print(env_io, x) end - String(_unsafe_take!(s)) + takestring!(s) end """ @@ -294,10 +294,10 @@ Create a read-only `IOBuffer` on the data underlying the given string. ```jldoctest julia> io = IOBuffer("Haho"); -julia> String(take!(io)) +julia> takestring!(io) "Haho" -julia> String(take!(io)) +julia> takestring!(io) "Haho" ``` """ @@ -774,7 +774,7 @@ function unindent(str::AbstractString, indent::Int; tabwidth=8) print(buf, ' ') end end - String(take!(buf)) + takestring!(buf) end function String(a::AbstractVector{Char}) diff --git a/base/strings/search.jl b/base/strings/search.jl index 90e02a0e5792c..a68b5f52b9f0a 100644 --- a/base/strings/search.jl +++ b/base/strings/search.jl @@ -585,7 +585,7 @@ findlast(ch::AbstractChar, string::AbstractString) = findlast(==(ch), string) overlap::Bool = false, ) findall( - pattern::Vector{UInt8} + pattern::Vector{UInt8}, A::Vector{UInt8}; overlap::Bool = false, ) diff --git a/base/strings/string.jl b/base/strings/string.jl index c4e3edcdf0a6a..53593e3936842 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -62,8 +62,8 @@ In other cases, `Vector{UInt8}` data may be copied, but `v` is truncated anyway to guarantee consistent behavior. """ String(v::AbstractVector{UInt8}) = unsafe_takestring(copyto!(StringMemory(length(v)), v)) + function String(v::Vector{UInt8}) - #return ccall(:jl_array_to_string, Ref{String}, (Any,), v) len = length(v) len == 0 && return "" ref = v.ref @@ -78,12 +78,35 @@ function String(v::Vector{UInt8}) return str end -"Create a string re-using the memory, if possible. -Mutating or reading the memory after calling this function is undefined behaviour." +""" + unsafe_takestring(m::Memory{UInt8})::String + +Create a `String` from `m`, changing the interpretation of the contents of `m`. +This is done without copying, if possible. Thus, any access to `m` after +calling this function, either to read or to write, is undefined behavior. +""" function unsafe_takestring(m::Memory{UInt8}) isempty(m) ? "" : ccall(:jl_genericmemory_to_string, Ref{String}, (Any, Int), m, length(m)) end +""" + takestring!(x) -> String + +Create a string from the content of `x`, emptying `x`. + +# Examples +```jldoctest +julia> v = [0x61, 0x62, 0x63]; + +julia> s = takestring!(v) +"abc" + +julia> isempty(v) +true +``` +""" +takestring!(v::Vector{UInt8}) = String(v) + """ unsafe_string(p::Ptr{UInt8}, [length::Integer]) @@ -226,7 +249,7 @@ end end)(s, i, n, l) end -## checking UTF-8 & ACSII validity ## +## checking UTF-8 & ASCII validity ## #= The UTF-8 Validation is performed by a shift based DFA. ┌───────────────────────────────────────────────────────────────────┐ @@ -374,7 +397,7 @@ end ## -# Classifcations of string +# Classifications of string # 0: neither valid ASCII nor UTF-8 # 1: valid ASCII # 2: valid UTF-8 diff --git a/base/strings/strings.jl b/base/strings/strings.jl index 8dae311f475b4..32975b6ea3fc7 100644 --- a/base/strings/strings.jl +++ b/base/strings/strings.jl @@ -11,3 +11,4 @@ import .Iterators: PartitionIterator include("strings/util.jl") include("strings/io.jl") +include("strings/annotated_io.jl") diff --git a/base/strings/substring.jl b/base/strings/substring.jl index 148c860705dd3..860895207f444 100644 --- a/base/strings/substring.jl +++ b/base/strings/substring.jl @@ -135,10 +135,8 @@ end pointer(x::SubString{String}) = pointer(x.string) + x.offset pointer(x::SubString{String}, i::Integer) = pointer(x.string) + x.offset + (i-1) -function hash(s::SubString{String}, h::UInt) - h += memhash_seed - ccall(memhash, UInt, (Ptr{UInt8}, Csize_t, UInt32), s, sizeof(s), h % UInt32) + h -end +hash(data::SubString{String}, h::UInt) = + GC.@preserve data hash_bytes(pointer(data), sizeof(data), UInt64(h), HASH_SECRET) % UInt _isannotated(::SubString{T}) where {T} = _isannotated(T) diff --git a/base/strings/unicode.jl b/base/strings/unicode.jl index eb03fd0eb2837..520226aacd3cb 100644 --- a/base/strings/unicode.jl +++ b/base/strings/unicode.jl @@ -6,7 +6,7 @@ module Unicode import Base: show, ==, hash, string, Symbol, isless, length, eltype, convert, isvalid, ismalformed, isoverlong, iterate, AnnotatedString, AnnotatedChar, annotated_chartransform, - @assume_effects, annotations + @assume_effects, annotations, is_overlong_enc # whether codepoints are valid Unicode scalar values, i.e. 0-0xd7ff, 0xe000-0x10ffff @@ -262,17 +262,15 @@ julia> textwidth('⛵') 2 ``` """ -function textwidth(c::AbstractChar) - ismalformed(c) && return 1 - i = codepoint(c) - i < 0x7f && return Int(i >= 0x20) # ASCII fast path - Int(ccall(:utf8proc_charwidth, Cint, (UInt32,), i)) -end +textwidth(c::AbstractChar) = textwidth(Char(c)::Char) function textwidth(c::Char) - b = bswap(reinterpret(UInt32, c)) # from isascii(c) + u = reinterpret(UInt32, c) + b = bswap(u) # from isascii(c) b < 0x7f && return Int(b >= 0x20) # ASCII fast path - ismalformed(c) && return 1 + # We can't know a priori how terminals will render invalid UTF8 chars, + # so we conservatively decide a width of 1. + (ismalformed(c) || is_overlong_enc(u)) && return 1 Int(ccall(:utf8proc_charwidth, Cint, (UInt32,), c)) end @@ -702,7 +700,7 @@ function titlecase(s::AbstractString; wordsep::Function = !isletter, strict::Boo end c0 = c end - return String(take!(b)) + return takestring!(b) end # TODO: improve performance characteristics, room for a ~10x improvement. @@ -800,7 +798,7 @@ isgraphemebreak(c1::AbstractChar, c2::AbstractChar) = # Stateful grapheme break required by Unicode-9 rules: the string # must be processed in sequence, with state initialized to Ref{Int32}(0). # Requires utf8proc v2.0 or later. -function isgraphemebreak!(state::Ref{Int32}, c1::AbstractChar, c2::AbstractChar) +@inline function isgraphemebreak!(state::Ref{Int32}, c1::AbstractChar, c2::AbstractChar) if ismalformed(c1) || ismalformed(c2) state[] = 0 return true diff --git a/base/strings/util.jl b/base/strings/util.jl index 184fdf34a2eac..c3df790cd81e6 100644 --- a/base/strings/util.jl +++ b/base/strings/util.jl @@ -3,7 +3,7 @@ """ Base.Chars = Union{AbstractChar,Tuple{Vararg{AbstractChar}},AbstractVector{<:AbstractChar},AbstractSet{<:AbstractChar}} -An alias type for a either single character or a tuple/vector/set of characters, used to describe arguments +An alias type for either a single character or a tuple/vector/set of characters, used to describe arguments of several string-matching functions such as [`startswith`](@ref) and [`strip`](@ref). !!! compat "Julia 1.11" @@ -319,7 +319,7 @@ end """ chomp(s::AbstractString)::SubString -Remove a single trailing newline from a string. +Remove a single trailing newline (i.e. "\\r\\n" or "\\n") from a string. See also [`chop`](@ref). @@ -327,6 +327,12 @@ See also [`chop`](@ref). ```jldoctest julia> chomp("Hello\\n") "Hello" + +julia> chomp("World\\r\\n") +"World" + +julia> chomp("Julia\\r\\n\\n") +"Julia\\r\\n" ``` """ function chomp(s::AbstractString) @@ -336,17 +342,22 @@ function chomp(s::AbstractString) (j < 1 || s[j] != '\r') && (return SubString(s, 1, j)) return SubString(s, 1, prevind(s,j)) end -function chomp(s::String) - i = lastindex(s) - if i < 1 || codeunit(s,i) != 0x0a - return @inbounds SubString(s, 1, i) - elseif i < 2 || codeunit(s,i-1) != 0x0d - return @inbounds SubString(s, 1, prevind(s, i)) + +@assume_effects :removable :foldable function chomp(s::Union{String, SubString{String}}) + cu = codeunits(s) + ncu = length(cu) + len = if iszero(ncu) + 0 else - return @inbounds SubString(s, 1, prevind(s, i-1)) + has_lf = @inbounds(cu[ncu]) == 0x0a + two_bytes = ncu > 1 + has_cr = has_lf & two_bytes & (@inbounds(cu[ncu - two_bytes]) == 0x0d) + ncu - (has_lf + has_cr) end + off = s isa String ? 0 : s.offset + par = s isa String ? s : s.string + @inbounds @inline SubString{String}(par, off, len, Val{:noshift}()) end - """ lstrip([pred=isspace,] str::AbstractString)::SubString lstrip(str::AbstractString, chars)::SubString @@ -1040,7 +1051,7 @@ function _replace_(str, pat_repl::NTuple{N, Pair}, count::Int) where N return String(str) end out = IOBuffer(sizehint=floor(Int, 1.2sizeof(str))) - return String(take!(_replace_finish(out, str, count, e1, patterns, replaces, rs))) + return takestring!(_replace_finish(out, str, count, e1, patterns, replaces, rs)) end """ @@ -1062,8 +1073,11 @@ is supplied, the transformed string is instead written to `io` (returning `io`). (For example, this can be used in conjunction with an [`IOBuffer`](@ref) to re-use a pre-allocated buffer array in-place.) -Multiple patterns can be specified, and they will be applied left-to-right -simultaneously, so only one pattern will be applied to any character, and the +Multiple patterns can be specified: The input string will be scanned only once +from start (left) to end (right), and the first matching replacement +will be applied to each substring. Replacements are applied in the order of +the arguments provided if they match substrings starting at the same +input string position. Thus, only one pattern will be applied to any character, and the patterns will only be applied to the input text, not the replacements. !!! compat "Julia 1.7" @@ -1272,5 +1286,5 @@ function Base.rest(s::AbstractString, st...) for c in Iterators.rest(s, st...) print(io, c) end - return String(take!(io)) + return takestring!(io) end diff --git a/base/subarray.jl b/base/subarray.jl index eacaddc068f1f..bc68d7c555d13 100644 --- a/base/subarray.jl +++ b/base/subarray.jl @@ -128,7 +128,7 @@ _trimmedshape(i::AbstractArray{<:ScalarIndex}, rest...) = (length(i), _trimmedsh _trimmedshape(i::AbstractArray{<:AbstractCartesianIndex{0}}, rest...) = _trimmedshape(rest...) _trimmedshape(i::AbstractArray{<:AbstractCartesianIndex{N}}, rest...) where {N} = (length(i), ntuple(Returns(1), Val(N - 1))..., _trimmedshape(rest...)...) _trimmedshape() = () -# We can avoid the repeation from `AbstractArray{CartesianIndex{0}}` +# We can avoid the repetition from `AbstractArray{CartesianIndex{0}}` _trimmedpind(i, rest...) = (map(Returns(:), axes(i))..., _trimmedpind(rest...)...) _trimmedpind(i::AbstractRange, rest...) = (i, _trimmedpind(rest...)...) _trimmedpind(i::Union{UnitRange,StepRange,OneTo}, rest...) = ((:), _trimmedpind(rest...)...) @@ -522,6 +522,16 @@ function _indices_sub(i1::AbstractArray, I...) (axes(i1)..., _indices_sub(I...)...) end +axes1(::SubArray{<:Any,0}) = OneTo(1) +axes1(S::SubArray) = (@inline; _axes1_sub(S.indices...)) +_axes1_sub() = () +_axes1_sub(::Real, I...) = (@inline; _axes1_sub(I...)) +_axes1_sub(::AbstractArray{<:Any,0}, I...) = _axes1_sub(I...) +function _axes1_sub(i1::AbstractArray, I...) + @inline + axes1(i1) +end + has_offset_axes(S::SubArray) = has_offset_axes(S.indices...) function replace_in_print_matrix(S::SubArray{<:Any,2,<:AbstractMatrix}, i::Integer, j::Integer, s::AbstractString) diff --git a/base/summarysize.jl b/base/summarysize.jl index 1e4d546e675aa..9dfd1431b84c7 100644 --- a/base/summarysize.jl +++ b/base/summarysize.jl @@ -139,7 +139,7 @@ end function (ss::SummarySize)(obj::Core.TypeName) key = pointer_from_objref(obj) haskey(ss.seen, key) ? (return 0) : (ss.seen[key] = true) - return Core.sizeof(obj) + (isdefined(obj, :mt) ? ss(obj.mt) : 0) + return Core.sizeof(obj) end function (ss::SummarySize)(obj::GenericMemory) diff --git a/base/sysimg.jl b/base/sysimg.jl index c12ddcd71c66f..fd71544c205cc 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -1,19 +1,12 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -# Can be loaded on top of either an existing system image built from -# `Base_compiler.jl` or standalone, in which case we will build it now. -let had_compiler = isdefined(Main, :Base) -if had_compiler; else -include("Base_compiler.jl") -end - -Core.include(Base, "Base.jl") +Base.include("Base.jl") # finish populating Base (currently just has the Compiler) -had_compiler && ccall(:jl_init_restored_module, Cvoid, (Any,), Base) -end +# Set up Main module by importing from Base +using .Base +using .Base.MainInclude # ans, err, and sometimes Out -# Set up Main module -using Base.MainInclude # ans, err, and sometimes Out +ccall(:jl_init_restored_module, Cvoid, (Any,), Base) # These definitions calls Base._include rather than Base.include to get # one-frame stacktraces for the common case of using include(fname) in Main. @@ -22,8 +15,8 @@ using Base.MainInclude # ans, err, and sometimes Out include([mapexpr::Function,] path::AbstractString) Evaluate the contents of the input source file in the global scope of the containing module. -Every module (except those defined with `baremodule`) has its own -definition of `include`, which evaluates the file in that module. +Every `Module` (except those defined with `baremodule`) has a private 1-argument definition +of `include`, which evaluates the file in that module, for use inside that module. Returns the result of the last evaluated expression of the input file. During including, a task-local include path is set to the directory containing the file. Nested calls to `include` will search relative to that path. This function is typically used to load source @@ -42,24 +35,27 @@ Use [`Base.include`](@ref) to evaluate a file into another module. at top-level and inserts an implicit `@Core.latestworld` to make any include'd definitions visible to subsequent code. Note however that this recognition is *syntactic*. I.e. assigning `const myinclude = include` may require - and explicit `@Core.latestworld` call after `myinclude`. + an explicit `@Core.latestworld` call after `myinclude`. !!! compat "Julia 1.5" Julia 1.5 is required for passing the `mapexpr` argument. """ -const include = Base.IncludeInto(Main) +Base.IncludeInto """ eval(expr) Evaluate an expression in the global scope of the containing module. -Every `Module` (except those defined with `baremodule`) has its own 1-argument -definition of `eval`, which evaluates expressions in that module. +Every `Module` (except those defined with `baremodule`) has a private 1-argument definition +of `eval`, which evaluates expressions in that module, for use inside that module. """ +Core.EvalInto + +const include = Base.IncludeInto(Main) const eval = Core.EvalInto(Main) # Ensure this file is also tracked -pushfirst!(Base._included_files, (@__MODULE__, abspath(@__FILE__))) +pushfirst!(Base._included_files, (Main, abspath(@__FILE__))) # set up depot & load paths to be able to find stdlib packages Base.init_depot_path() diff --git a/base/sysinfo.jl b/base/sysinfo.jl index 77fd628ff2883..06e5cd298caf1 100644 --- a/base/sysinfo.jl +++ b/base/sysinfo.jl @@ -16,6 +16,7 @@ export BINDIR, JIT, cpu_info, cpu_summary, + sysimage_target, uptime, loadavg, free_memory, @@ -104,7 +105,7 @@ Standard word size on the current machine, in bits. const WORD_SIZE = Core.sizeof(Int) * 8 """ - Sys.SC_CLK_TCK: + Sys.SC_CLK_TCK::Clong The number of system "clock ticks" per second, corresponding to `sysconf(_SC_CLK_TCK)` on POSIX systems, or `0` if it is unknown. @@ -312,6 +313,23 @@ function cpu_info() return cpus end +""" + Sys.sysimage_target() + +Return the CPU target string that was used to build the current system image. + +This function returns the original CPU target specification that was passed to Julia +when the system image was compiled. This can be useful for reproducing the same +system image or for understanding what CPU features were enabled during compilation. + +If the system image was built with the default settings this will return `"native"`. + +See also [`JULIA_CPU_TARGET`](@ref). +""" +function sysimage_target() + return ccall(:jl_get_sysimage_cpu_target, Ref{String}, ()) +end + """ Sys.uptime() diff --git a/base/task.jl b/base/task.jl index c6aec03191144..4f17330bb4455 100644 --- a/base/task.jl +++ b/base/task.jl @@ -14,7 +14,7 @@ struct CapturedException <: Exception # Typically the result of a catch_backtrace() # Process bt_raw so that it can be safely serialized - bt_lines = process_backtrace(bt_raw, 100) # Limiting this to 100 lines. + bt_lines = process_backtrace(stacktrace(bt_raw))[1:min(100, end)] # Limiting this to 100 lines. CapturedException(ex, bt_lines) end @@ -95,7 +95,7 @@ function show_task_exception(io::IO, t::Task; indent = true) else show_exception_stack(IOContext(b, io), stack) end - str = String(take!(b)) + str = takestring!(b) if indent str = replace(str, "\n" => "\n ") end @@ -909,31 +909,10 @@ function list_deletefirst!(W::IntrusiveLinkedListSynchronized{T}, t::T) where T end const StickyWorkqueue = IntrusiveLinkedListSynchronized{Task} -global Workqueues::Vector{StickyWorkqueue} = [StickyWorkqueue()] -const Workqueues_lock = Threads.SpinLock() +const Workqueues = OncePerThread{StickyWorkqueue}(StickyWorkqueue) const Workqueue = Workqueues[1] # default work queue is thread 1 // TODO: deprecate this variable -function workqueue_for(tid::Int) - qs = Workqueues - if length(qs) >= tid && isassigned(qs, tid) - return @inbounds qs[tid] - end - # slow path to allocate it - @assert tid > 0 - l = Workqueues_lock - @lock l begin - qs = Workqueues - if length(qs) < tid - nt = Threads.maxthreadid() - @assert tid <= nt - global Workqueues = qs = copyto!(typeof(qs)(undef, length(qs) + nt - 1), qs) - end - if !isassigned(qs, tid) - @inbounds qs[tid] = StickyWorkqueue() - end - return @inbounds qs[tid] - end -end +workqueue_for(tid::Int) = Workqueues[tid] function enq_work(t::Task) (t._state === task_state_runnable && t.queue === nothing) || error("schedule: Task not runnable") @@ -1145,6 +1124,16 @@ function throwto(t::Task, @nospecialize exc) return try_yieldto(identity) end +@inline function wait_forever() + while true + wait() + end +end + +const get_sched_task = OncePerThread{Task}() do + Task(wait_forever) +end + function ensure_rescheduled(othertask::Task) ct = current_task() W = workqueue_for(Threads.threadid()) @@ -1181,26 +1170,29 @@ end checktaskempty = Partr.multiq_check_empty -@noinline function poptask(W::StickyWorkqueue) - task = trypoptask(W) - if !(task isa Task) - task = ccall(:jl_task_get_next, Ref{Task}, (Any, Any, Any), trypoptask, W, checktaskempty) - end - set_next_task(task) - nothing -end - function wait() ct = current_task() # [task] user_time -yield-or-done-> wait_time record_running_time!(ct) + # let GC run GC.safepoint() - W = workqueue_for(Threads.threadid()) - poptask(W) - result = try_yieldto(ensure_rescheduled) + # check for libuv events process_events() - # return when we come out of the queue - return result + + # get the next task to run + W = workqueue_for(Threads.threadid()) + task = trypoptask(W) + if task === nothing + # No tasks to run; switch to the scheduler task to run the + # thread sleep logic. + sched_task = get_sched_task() + if ct !== sched_task + return yieldto(sched_task) + end + task = ccall(:jl_task_get_next, Ref{Task}, (Any, Any, Any), trypoptask, W, checktaskempty) + end + set_next_task(task) + return try_yieldto(ensure_rescheduled) end if Sys.iswindows() diff --git a/base/terminfo.jl b/base/terminfo.jl index 8ea8387077d36..be0dd53b1ac74 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -303,16 +303,24 @@ end """ The terminfo of the current terminal. """ -current_terminfo::TermInfo = TermInfo() +const current_terminfo = OncePerProcess{TermInfo}() do + term_env = get(ENV, "TERM", @static Sys.iswindows() ? "" : "dumb") + terminfo = load_terminfo(term_env) + # Ensure setaf is set for xterm terminals + if !haskey(terminfo, :setaf) && startswith(term_env, "xterm") + # For xterm-like terminals without setaf, add a reasonable default + terminfo.strings[:setaf] = "\e[3%p1%dm" + end + return terminfo +end # Legacy/TTY methods and the `:color` parameter if Sys.iswindows() - ttyhascolor(term_type = nothing) = true + ttyhascolor() = true else - function ttyhascolor(term_type = get(ENV, "TERM", "")) - startswith(term_type, "xterm") || - haskey(current_terminfo, :setaf) + function ttyhascolor() + haskey(current_terminfo(), :setaf) end end @@ -323,8 +331,8 @@ Return a boolean signifying whether the current terminal supports 24-bit colors. Multiple conditions are taken as signifying truecolor support, specifically any of the following: - The `COLORTERM` environment variable is set to `"truecolor"` or `"24bit"` -- The current terminfo sets the [`RGB`[^1] - capability](https://invisible-island.net/ncurses/man/user_caps.5.html#h3-Recognized-Capabilities) +- The current terminfo sets the [`RGB` + capability](https://invisible-island.net/ncurses/man/user_caps.5.html#h3-Recognized-Capabilities)[^1] (or the legacy `Tc` capability[^2]) flag - The current terminfo provides `setrgbf` and `setrgbb` strings[^3] - The current terminfo has a `colors` number greater that `256`, on a unix system @@ -352,9 +360,9 @@ Multiple conditions are taken as signifying truecolor support, specifically any function ttyhastruecolor() # Lasciate ogne speranza, voi ch'intrate get(ENV, "COLORTERM", "") ∈ ("truecolor", "24bit") || - get(current_terminfo, :RGB, false) || get(current_terminfo, :Tc, false) || - (haskey(current_terminfo, :setrgbf) && haskey(current_terminfo, :setrgbb)) || - @static if Sys.isunix() get(current_terminfo, :colors, 0) > 256 else false end || + get(current_terminfo(), :RGB, false) || get(current_terminfo(), :Tc, false) || + (haskey(current_terminfo(), :setrgbf) && haskey(current_terminfo(), :setrgbb)) || + @static if Sys.isunix() get(current_terminfo(), :colors, 0) > 256 else false end || (Sys.iswindows() && Sys.windows_version() ≥ v"10.0.14931") || # See something(tryparse(Int, get(ENV, "VTE_VERSION", "")), 0) >= 3600 || # Per GNOME bug #685759 haskey(ENV, "XTERM_VERSION") || diff --git a/base/timing.jl b/base/timing.jl index 61fa73f2eff62..998103d1e78bc 100644 --- a/base/timing.jl +++ b/base/timing.jl @@ -472,6 +472,35 @@ function gc_bytes() b[] end +function allocated(f, args::Vararg{Any,N}) where {N} + b0 = Ref{Int64}(0) + b1 = Ref{Int64}(0) + Base.gc_bytes(b0) + f(args...) + Base.gc_bytes(b1) + return b1[] - b0[] +end +only(methods(allocated)).called = 0xff + +function allocations(f, args::Vararg{Any,N}) where {N} + stats = Base.gc_num() + f(args...) + diff = Base.GC_Diff(Base.gc_num(), stats) + return Base.gc_alloc_count(diff) +end +only(methods(allocations)).called = 0xff + +function is_simply_call(@nospecialize ex) + Meta.isexpr(ex, :call) || return false + for a in ex.args + a isa QuoteNode && continue + a isa Symbol && continue + isa_ast_node(a) || continue + return false + end + return true +end + """ @allocated @@ -487,15 +516,11 @@ julia> @allocated rand(10^6) ``` """ macro allocated(ex) - quote - Experimental.@force_compile - local b0 = Ref{Int64}(0) - local b1 = Ref{Int64}(0) - gc_bytes(b0) - $(esc(ex)) - gc_bytes(b1) - b1[] - b0[] + if !is_simply_call(ex) + ex = :((() -> $ex)()) end + pushfirst!(ex.args, GlobalRef(Base, :allocated)) + return esc(ex) end """ @@ -516,15 +541,14 @@ julia> @allocations rand(10^6) This macro was added in Julia 1.9. """ macro allocations(ex) - quote - Experimental.@force_compile - local stats = Base.gc_num() - $(esc(ex)) - local diff = Base.GC_Diff(Base.gc_num(), stats) - Base.gc_alloc_count(diff) + if !is_simply_call(ex) + ex = :((() -> $ex)()) end + pushfirst!(ex.args, GlobalRef(Base, :allocations)) + return esc(ex) end + """ @lock_conflicts @@ -643,33 +667,36 @@ end # here so it's possible to time/trace all imports, including InteractiveUtils and its deps macro time_imports(ex) quote - try - Base.Threads.atomic_add!(Base.TIMING_IMPORTS, 1) - $(esc(ex)) - finally + Base.Threads.atomic_add!(Base.TIMING_IMPORTS, 1) + @__tryfinally( + # try + $(esc(ex)), + # finally Base.Threads.atomic_sub!(Base.TIMING_IMPORTS, 1) - end + ) end end macro trace_compile(ex) quote - try - ccall(:jl_force_trace_compile_timing_enable, Cvoid, ()) - $(esc(ex)) - finally + ccall(:jl_force_trace_compile_timing_enable, Cvoid, ()) + @__tryfinally( + # try + $(esc(ex)), + # finally ccall(:jl_force_trace_compile_timing_disable, Cvoid, ()) - end + ) end end macro trace_dispatch(ex) quote - try - ccall(:jl_force_trace_dispatch_enable, Cvoid, ()) - $(esc(ex)) - finally + ccall(:jl_force_trace_dispatch_enable, Cvoid, ()) + @__tryfinally( + # try + $(esc(ex)), + # finally ccall(:jl_force_trace_dispatch_disable, Cvoid, ()) - end + ) end end diff --git a/base/toml_parser.jl b/base/toml_parser.jl index ddacacee5ae0e..bf13fc2b0617a 100644 --- a/base/toml_parser.jl +++ b/base/toml_parser.jl @@ -258,10 +258,10 @@ end mutable struct ParserError <: Exception type::ErrorType - # Arbitrary data to store at the + # Data to store at the # call site to be used when formatting # the error - data + data::Union{Char, Nothing} # These are filled in before returning from parse function str ::Union{String, Nothing} @@ -276,7 +276,7 @@ ParserError(type) = ParserError(type, nothing) # Defining these below can be useful when debugging code that erroneously returns a # ParserError because you get a stacktrace to where the ParserError was created #ParserError(type) = error(type) -#ParserError(type, data) = error(type,data) +#ParserError(type, data) = error(type, data) # Many functions return either a T or a ParserError const Err{T} = Union{T, ParserError} @@ -284,7 +284,7 @@ const Err{T} = Union{T, ParserError} function format_error_message_for_err_type(error::ParserError) msg = err_message[error.type] if error.type == ErrInvalidBareKeyCharacter - c_escaped = escape_string(string(error.data)::String) + c_escaped = escape_string(string(error.data::Char)) msg *= ": '$c_escaped'" end return msg @@ -315,7 +315,7 @@ function point_to_line(str::AbstractString, a::Int, b::Int, context) c == '\n' && break print(io1, c) end - return String(take!(io1.io)), String(take!(io2.io)) + return takestring!(io1.io), takestring!(io2.io) end function Base.showerror(io::IO, err::ParserError) diff --git a/base/tuple.jl b/base/tuple.jl index 4982ef1b23eb0..ac42c667269e6 100644 --- a/base/tuple.jl +++ b/base/tuple.jl @@ -149,12 +149,13 @@ nextind(@nospecialize(t::Tuple), i::Integer) = Int(i)+1 function keys(t::Tuple, t2::Tuple...) @inline - OneTo(_maxlength(t, t2...)) -end -_maxlength(t::Tuple) = length(t) -function _maxlength(t::Tuple, t2::Tuple, t3::Tuple...) - @inline - max(length(t), _maxlength(t2, t3...)) + lent = length(t) + if !all(==(lent) ∘ length, t2) + let inds = map(only ∘ axes, (t, t2...)) + throw_eachindex_mismatch_indices("indices", inds...) + end + end + Base.OneTo(lent) end # this allows partial evaluation of bounded sequences of next() calls on tuples, @@ -576,10 +577,10 @@ function _eq(t1::Any32, t2::Any32) end const tuplehash_seed = UInt === UInt64 ? 0x77cfa1eef01bca90 : 0xf01bca90 -hash(::Tuple{}, h::UInt) = h + tuplehash_seed +hash(::Tuple{}, h::UInt) = h ⊻ tuplehash_seed hash(t::Tuple, h::UInt) = hash(t[1], hash(tail(t), h)) function hash(t::Any32, h::UInt) - out = h + tuplehash_seed + out = h ⊻ tuplehash_seed for i = length(t):-1:1 out = hash(t[i], out) end diff --git a/base/util.jl b/base/util.jl index a05858305223a..fa01f1bcde498 100644 --- a/base/util.jl +++ b/base/util.jl @@ -77,7 +77,7 @@ function with_output_color(@nospecialize(f::Function), color::Union{Int, Symbol} iscolor = get(io, :color, false)::Bool try f(IOContext(buf, io), args...) finally - str = String(take!(buf)) + str = takestring!(buf) if !iscolor print(io, str) else @@ -109,7 +109,7 @@ function with_output_color(@nospecialize(f::Function), color::Union{Int, Symbol} isempty(line) && continue print(buf, enable_ansi, line, disable_ansi) end - print(io, String(take!(buf))) + print(io, takestring!(buf)) end end end diff --git a/base/version.jl b/base/version.jl index b362daa78f04f..71192916a5b22 100644 --- a/base/version.jl +++ b/base/version.jl @@ -218,7 +218,7 @@ function isless(a::VersionNumber, b::VersionNumber) end function hash(v::VersionNumber, h::UInt) - h += 0x8ff4ffdb75f9fede % UInt + h ⊻= 0x8ff4ffdb75f9fede % UInt h = hash(v.major, h) h = hash(v.minor, h) h = hash(v.patch, h) diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl index 1a98bf1ee4333..1283dc9cbc8cb 100644 --- a/base/weakkeydict.jl +++ b/base/weakkeydict.jl @@ -2,6 +2,12 @@ # weak key dictionaries +mutable struct WeakKeyDictFinalizer{T} + const d::T +end +(d::WeakKeyDictFinalizer)(k) = d.d.dirty = true + + """ WeakKeyDict([itr]) @@ -16,15 +22,15 @@ object was unreferenced anywhere before insertion. See also [`WeakRef`](@ref). """ mutable struct WeakKeyDict{K,V} <: AbstractDict{K,V} - ht::Dict{WeakRef,V} - lock::ReentrantLock - finalizer::Function + const ht::Dict{WeakRef,V} + const lock::ReentrantLock dirty::Bool + finalizer::WeakKeyDictFinalizer # Constructors mirror Dict's - function WeakKeyDict{K,V}() where V where K - t = new(Dict{WeakRef,V}(), ReentrantLock(), identity, 0) - t.finalizer = k -> t.dirty = true + function WeakKeyDict{K,V}() where {K, V} + t = new{K,V}(Dict{WeakRef,V}(), ReentrantLock(), false) + t.finalizer = WeakKeyDictFinalizer(t) return t end end @@ -70,7 +76,7 @@ function _cleanup_locked(h::WeakKeyDict) return h end -sizehint!(d::WeakKeyDict, newsz; shrink::Bool = true) = @lock d sizehint!(d.ht, newsz; shrink = shrink) +sizehint!(d::WeakKeyDict, newsz::Integer; shrink::Bool = true) = @lock d sizehint!(d.ht, newsz; shrink = shrink) empty(d::WeakKeyDict, ::Type{K}, ::Type{V}) where {K, V} = WeakKeyDict{K, V}() IteratorSize(::Type{<:WeakKeyDict}) = SizeUnknown() diff --git a/cli/loader.h b/cli/loader.h index be5195583b29f..310226c84f815 100644 --- a/cli/loader.h +++ b/cli/loader.h @@ -36,9 +36,9 @@ // Borrow definition from `support/dtypes.h` #ifdef _OS_WINDOWS_ # ifdef JL_LIBRARY_EXPORTS -# define JL_DLLEXPORT __declspec(dllexport) +# define JL_DLLEXPORT __declspec(dllexport) __attribute__ ((visibility("default"))) # endif -# define JL_DLLIMPORT __declspec(dllimport) +# define JL_DLLIMPORT __declspec(dllimport) __attribute__ ((visibility("default"))) #define JL_HIDDEN #else # define JL_DLLIMPORT __attribute__ ((visibility("default"))) diff --git a/cli/loader_win_utils.c b/cli/loader_win_utils.c index ed585a7a64ff0..34fe277fb2879 100644 --- a/cli/loader_win_utils.c +++ b/cli/loader_win_utils.c @@ -12,6 +12,10 @@ static FILE _stderr = { INVALID_HANDLE_VALUE }; FILE *stdout = &_stdout; FILE *stderr = &_stderr; +void JL_HIDDEN free(void* mem) { + HeapFree(GetProcessHeap(), 0, mem); +} + int JL_HIDDEN fwrite(const char *str, size_t nchars, FILE *out) { DWORD written; if (out->isconsole) { @@ -44,10 +48,6 @@ void JL_HIDDEN *realloc(void * mem, const size_t size) { return HeapReAlloc(GetProcessHeap(), HEAP_GENERATE_EXCEPTIONS, mem, size); } -void JL_HIDDEN free(void* mem) { - HeapFree(GetProcessHeap(), 0, mem); -} - LPWSTR *CommandLineToArgv(LPWSTR lpCmdLine, int *pNumArgs) { LPWSTR out = lpCmdLine; LPWSTR cmd = out; diff --git a/contrib/asan/Make.user.asan b/contrib/asan/Make.user.asan index 025cfad82214b..1ad8d3c8fb1f7 100644 --- a/contrib/asan/Make.user.asan +++ b/contrib/asan/Make.user.asan @@ -3,7 +3,6 @@ BINDIR=$(TOOLCHAIN)/usr/bin TOOLDIR=$(TOOLCHAIN)/usr/tools # use our new toolchain -USECLANG=1 override CC=$(TOOLDIR)/clang override CXX=$(TOOLDIR)/clang++ override PATCHELF=$(TOOLDIR)/patchelf diff --git a/contrib/check-whitespace.jl b/contrib/check-whitespace.jl index fd3106587fb0d..d7e04512e153d 100755 --- a/contrib/check-whitespace.jl +++ b/contrib/check-whitespace.jl @@ -32,10 +32,39 @@ allow_tabs(path) = endswith(path, "test/syntax.jl") || endswith(path, "test/triplequote.jl") -const errors = Set{Tuple{String,Int,String}}() - function check_whitespace() - for path in eachline(`git ls-files -- $patterns`) + # Get file list from ARGS if provided, otherwise use git ls-files + errors = Set{Tuple{String,Int,String}}() + files_to_check = filter(arg -> arg != "--fix", ARGS) + if isempty(files_to_check) + files_to_check = eachline(`git ls-files -- $patterns`) + end + + files_fixed = 0 + if "--fix" in ARGS + for path in files_to_check + content = newcontent = read(path, String) + isempty(content) && continue + if !allow_tabs(path) + tabpattern = r"^([ \t]+)"m => (x -> replace(x, r"((?: {4})*)( *\t)" => s"\1 ")) # Replace tab sequences at start of line after any number of 4-space groups + newcontent = replace(newcontent, tabpattern) + end + newcontent = replace(newcontent, + r"\s*$" => '\n', # Remove trailing whitespace and normalize line ending at eof + r"\s*?[\r\n]" => '\n', # Remove trailing whitespace and normalize line endings on each line + r"\xa0" => ' ' # Replace non-breaking spaces + ) + if content != newcontent + write(path, newcontent) + files_fixed += 1 + end + end + if files_fixed > 0 + println(stderr, "Fixed whitespace issues in $files_fixed files.") + end + end + + for path in files_to_check lineno = 0 non_blank = 0 diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index fc229c394ca02..578e27110f436 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license # Prevent this from putting anything into the Main namespace -@eval Core.Module() begin +@eval Base module __precompile_script if Threads.maxthreadid() != 1 @warn "Running this file with multiple Julia threads may lead to a build error" Threads.maxthreadid() @@ -34,13 +34,21 @@ hardcoded_precompile_statements = """ precompile(Base.unsafe_string, (Ptr{UInt8},)) precompile(Base.unsafe_string, (Ptr{Int8},)) -# loading.jl +# loading.jl - without these each precompile worker would precompile these because they're hit before pkgimages are loaded precompile(Base.__require, (Module, Symbol)) precompile(Base.__require, (Base.PkgId,)) precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int)) precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int)) precompile(Tuple{typeof(Base.Threads.atomic_add!), Base.Threads.Atomic{Int}, Int}) precompile(Tuple{typeof(Base.Threads.atomic_sub!), Base.Threads.Atomic{Int}, Int}) +precompile(Tuple{Type{Pair{A, B} where B where A}, Base.PkgId, UInt128}) +precompile(Tuple{typeof(Base.in!), Tuple{Module, String, UInt64, UInt32, Float64}, Base.Set{Any}}) +precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:allow_typevars, :volatile_inf_result), Tuple{Bool, Nothing}}, typeof(Base.Compiler.handle_match!), Array{Base.Compiler.InliningCase, 1}, Core.MethodMatch, Array{Any, 1}, Base.Compiler.CallInfo, UInt32, Base.Compiler.InliningState{Base.Compiler.NativeInterpreter}}) +precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}}) +precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:allow_typevars, :volatile_inf_result), Tuple{Bool, Base.Compiler.VolatileInferenceResult}}, typeof(Base.Compiler.handle_match!), Array{Base.Compiler.InliningCase, 1}, Core.MethodMatch, Array{Any, 1}, Base.Compiler.CallInfo, UInt32, Base.Compiler.InliningState{Base.Compiler.NativeInterpreter}}) +precompile(Tuple{typeof(Base.getindex), Type{Pair{Base.PkgId, UInt128}}, Pair{Base.PkgId, UInt128}, Pair{Base.PkgId, UInt128}, Pair{Base.PkgId, UInt128}, Vararg{Pair{Base.PkgId, UInt128}}}) +precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}, Core.SimpleVector}) +precompile(Tuple{typeof(Base.Compiler.ir_to_codeinf!), Base.Compiler.OptimizationState{Base.Compiler.NativeInterpreter}}) # LazyArtifacts (but more generally helpful) precompile(Tuple{Type{Base.Val{x} where x}, Module}) @@ -103,6 +111,7 @@ precompile(Base.CoreLogging.current_logger_for_env, (Base.CoreLogging.LogLevel, precompile(Base.CoreLogging.env_override_minlevel, (Symbol, Module)) precompile(Base.StackTraces.lookup, (Ptr{Nothing},)) precompile(Tuple{typeof(Base.run_module_init), Module, Int}) +precompile(Tuple{Type{Base.VersionNumber}, Int32, Int32, Int32}) # Presence tested in the tests precompile(Tuple{typeof(Base.print), Base.IOStream, String}) @@ -140,6 +149,9 @@ for match = Base._methods(+, (Int, Int), -1, Base.get_world_counter()) end empty!(Set()) push!(push!(Set{Union{GlobalRef,Symbol}}(), :two), GlobalRef(Base, :two)) +get!(ENV, "___DUMMY", "") +ENV["___DUMMY"] +delete!(ENV, "___DUMMY") (setindex!(Dict{String,Base.PkgId}(), Base.PkgId(Base), "file.jl"))["file.jl"] (setindex!(Dict{Symbol,Vector{Int}}(), [1], :two))[:two] (setindex!(Dict{Base.PkgId,String}(), "file.jl", Base.PkgId(Base)))[Base.PkgId(Base)] @@ -209,6 +221,10 @@ if Artifacts !== nothing end dlopen("libjulia$(Base.isdebugbuild() ? "-debug" : "")", RTLD_LAZY | RTLD_DEEPBIND) """ + hardcoded_precompile_statements *= """ + precompile(Tuple{typeof(Artifacts._artifact_str), Module, String, Base.SubString{String}, String, Base.Dict{String, Any}, Base.SHA1, Base.BinaryPlatforms.Platform, Base.Val{Artifacts}}) + precompile(Tuple{typeof(Base.tryparse), Type{Base.BinaryPlatforms.Platform}, String}) + """ end FileWatching = get(Base.loaded_modules, @@ -381,6 +397,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe if precompile(ps...) n_succeeded += 1 else + Base.get_bool_env("CI", false) && error("Precompilation failed for $statement") @warn "Failed to precompile expression" form=statement _module=nothing _file=nothing _line=0 end failed = length(statements) - n_succeeded @@ -388,6 +405,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe print_state("step3" => string("R$n_succeeded", failed > 0 ? " ($failed failed)" : "")) catch ex # See #28808 + Base.get_bool_env("CI", false) && error("Precompilation failed for $statement") @warn "Failed to precompile expression" form=statement exception=(ex,catch_backtrace()) _module=nothing _file=nothing _line=0 end end diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl deleted file mode 100644 index 363330e4a3a79..0000000000000 --- a/contrib/juliac-buildscript.jl +++ /dev/null @@ -1,301 +0,0 @@ -# Script to run in the process that generates juliac's object file output - -inputfile = ARGS[1] -output_type = ARGS[2] -add_ccallables = ARGS[3] == "true" - -# Initialize some things not usually initialized when output is requested -Sys.__init__() -Base.init_depot_path() -Base.init_load_path() -Base.init_active_project() -task = current_task() -task.rngState0 = 0x5156087469e170ab -task.rngState1 = 0x7431eaead385992c -task.rngState2 = 0x503e1d32781c2608 -task.rngState3 = 0x3a77f7189200c20b -task.rngState4 = 0x5502376d099035ae -uuid_tuple = (UInt64(0), UInt64(0)) -ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple) -if Base.get_bool_env("JULIA_USE_FLISP_PARSER", false) === false - Base.JuliaSyntax.enable_in_core!() -end - -# Patch methods in Core and Base - -@eval Core begin - DomainError(@nospecialize(val), @nospecialize(msg::AbstractString)) = (@noinline; $(Expr(:new, :DomainError, :val, :msg))) -end - -(f::Base.RedirectStdStream)(io::Core.CoreSTDOUT) = Base._redirect_io_global(io, f.unix_fd) - -@eval Base begin - depwarn(msg, funcsym; force::Bool=false) = nothing - _assert_tostring(msg) = "" - reinit_stdio() = nothing - JuliaSyntax.enable_in_core!() = nothing - init_active_project() = ACTIVE_PROJECT[] = nothing - set_active_project(projfile::Union{AbstractString,Nothing}) = ACTIVE_PROJECT[] = projfile - disable_library_threading() = nothing - start_profile_listener() = nothing - invokelatest_trimmed(f, args...; kwargs...) = f(args...; kwargs...) - const invokelatest = invokelatest_trimmed - function sprint(f::F, args::Vararg{Any,N}; context=nothing, sizehint::Integer=0) where {F<:Function,N} - s = IOBuffer(sizehint=sizehint) - if context isa Tuple - f(IOContext(s, context...), args...) - elseif context !== nothing - f(IOContext(s, context), args...) - else - f(s, args...) - end - String(_unsafe_take!(s)) - end - function show_typeish(io::IO, @nospecialize(T)) - if T isa Type - show(io, T) - elseif T isa TypeVar - print(io, (T::TypeVar).name) - else - print(io, "?") - end - end - function show(io::IO, T::Type) - if T isa DataType - print(io, T.name.name) - if T !== T.name.wrapper && length(T.parameters) > 0 - print(io, "{") - first = true - for p in T.parameters - if !first - print(io, ", ") - end - first = false - if p isa Int - show(io, p) - elseif p isa Type - show(io, p) - elseif p isa Symbol - print(io, ":") - print(io, p) - elseif p isa TypeVar - print(io, p.name) - else - print(io, "?") - end - end - print(io, "}") - end - elseif T isa Union - print(io, "Union{") - show_typeish(io, T.a) - print(io, ", ") - show_typeish(io, T.b) - print(io, "}") - elseif T isa UnionAll - print(io, T.body::Type) - print(io, " where ") - print(io, T.var.name) - end - end - show_type_name(io::IO, tn::Core.TypeName) = print(io, tn.name) - - mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted; dims=:, init=_InitialValue()) where {F, F2} = - _mapreduce_dim(f, op, init, A, dims) - mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted...; kw...) where {F, F2} = - reduce(op, map(f, A...); kw...) - - _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = - mapfoldl_impl(f, op, nt, A) - - _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, ::Colon) where {F, F2} = - _mapreduce(f, op, IndexStyle(A), A) - - _mapreduce_dim(f::F, op::F2, nt, A::AbstractArrayOrBroadcasted, dims) where {F, F2} = - mapreducedim!(f, op, reducedim_initarray(A, dims, nt), A) - - _mapreduce_dim(f::F, op::F2, ::_InitialValue, A::AbstractArrayOrBroadcasted, dims) where {F,F2} = - mapreducedim!(f, op, reducedim_init(f, op, A, dims), A) - - mapreduce_empty_iter(f::F, op::F2, itr, ItrEltype) where {F, F2} = - reduce_empty_iter(MappingRF(f, op), itr, ItrEltype) - mapreduce_first(f::F, op::F2, x) where {F,F2} = reduce_first(op, f(x)) - - _mapreduce(f::F, op::F2, A::AbstractArrayOrBroadcasted) where {F,F2} = _mapreduce(f, op, IndexStyle(A), A) - mapreduce_empty(::typeof(identity), op::F, T) where {F} = reduce_empty(op, T) - mapreduce_empty(::typeof(abs), op::F, T) where {F} = abs(reduce_empty(op, T)) - mapreduce_empty(::typeof(abs2), op::F, T) where {F} = abs2(reduce_empty(op, T)) -end -@eval Base.Sys begin - __init_build() = nothing -end -@eval Base.GMP begin - function __init__() - try - ccall((:__gmp_set_memory_functions, libgmp), Cvoid, - (Ptr{Cvoid},Ptr{Cvoid},Ptr{Cvoid}), - cglobal(:jl_gc_counted_malloc), - cglobal(:jl_gc_counted_realloc_with_old_size), - cglobal(:jl_gc_counted_free_with_size)) - ZERO.alloc, ZERO.size, ZERO.d = 0, 0, C_NULL - ONE.alloc, ONE.size, ONE.d = 1, 1, pointer(_ONE) - catch ex - Base.showerror_nostdio(ex, "WARNING: Error during initialization of module GMP") - end - # This only works with a patched version of GMP, ignore otherwise - try - ccall((:__gmp_set_alloc_overflow_function, libgmp), Cvoid, - (Ptr{Cvoid},), - cglobal(:jl_throw_out_of_memory_error)) - ALLOC_OVERFLOW_FUNCTION[] = true - catch ex - # ErrorException("ccall: could not find function...") - if typeof(ex) != ErrorException - rethrow() - end - end - end -end -@eval Base.Sort begin - issorted(itr; - lt::T=isless, by::F=identity, rev::Union{Bool,Nothing}=nothing, order::Ordering=Forward) where {T,F} = - issorted(itr, ord(lt,by,rev,order)) -end -@eval Base.TOML begin - function try_return_datetime(p, year, month, day, h, m, s, ms) - return DateTime(year, month, day, h, m, s, ms) - end - function try_return_date(p, year, month, day) - return Date(year, month, day) - end - function parse_local_time(l::Parser) - h = @try parse_int(l, false) - h in 0:23 || return ParserError(ErrParsingDateTime) - _, m, s, ms = @try _parse_local_time(l, true) - # TODO: Could potentially parse greater accuracy for the - # fractional seconds here. - return try_return_time(l, h, m, s, ms) - end - function try_return_time(p, h, m, s, ms) - return Time(h, m, s, ms) - end -end - -# Load user code - -import Base.Experimental.entrypoint - -let mod = Base.include(Base.__toplevel__, inputfile) - if !isa(mod, Module) - mod = Main - end - Core.@latestworld - if output_type == "--output-exe" && isdefined(mod, :main) && !add_ccallables - entrypoint(mod.main, ()) - end - #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{Base.SubString{String}, 1}, String)) - #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{String, 1}, Char)) - entrypoint(Base.task_done_hook, (Task,)) - entrypoint(Base.wait, ()) - entrypoint(Base.trypoptask, (Base.StickyWorkqueue,)) - entrypoint(Base.checktaskempty, ()) - if add_ccallables - ccall(:jl_add_ccallable_entrypoints, Cvoid, ()) - end -end - -# Additional method patches depending on whether user code loads certain stdlibs -let - find_loaded_root_module(key::Base.PkgId) = Base.maybe_root_module(key) - - SparseArrays = find_loaded_root_module(Base.PkgId( - Base.UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf"), "SparseArrays")) - if SparseArrays !== nothing - @eval SparseArrays.CHOLMOD begin - function __init__() - ccall((:SuiteSparse_config_malloc_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_malloc, Ptr{Cvoid})) - ccall((:SuiteSparse_config_calloc_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_calloc, Ptr{Cvoid})) - ccall((:SuiteSparse_config_realloc_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_realloc, Ptr{Cvoid})) - ccall((:SuiteSparse_config_free_func_set, :libsuitesparseconfig), - Cvoid, (Ptr{Cvoid},), cglobal(:jl_free, Ptr{Cvoid})) - end - end - end - - Artifacts = find_loaded_root_module(Base.PkgId( - Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts")) - if Artifacts !== nothing - @eval Artifacts begin - function _artifact_str( - __module__, - artifacts_toml, - name, - path_tail, - artifact_dict, - hash, - platform, - _::Val{LazyArtifacts} - ) where LazyArtifacts - # If the artifact exists, we're in the happy path and we can immediately - # return the path to the artifact: - dirs = artifacts_dirs(bytes2hex(hash.bytes)) - for dir in dirs - if isdir(dir) - return jointail(dir, path_tail) - end - end - error("Artifact not found") - end - end - end - - Pkg = find_loaded_root_module(Base.PkgId( - Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) - if Pkg !== nothing - @eval Pkg begin - __init__() = rand() #TODO, methods that do nothing don't get codegened - end - end - - StyledStrings = find_loaded_root_module(Base.PkgId( - Base.UUID("f489334b-da3d-4c2e-b8f0-e476e12c162b"), "StyledStrings")) - if StyledStrings !== nothing - @eval StyledStrings begin - __init__() = rand() - end - end - - Markdown = find_loaded_root_module(Base.PkgId( - Base.UUID("d6f4376e-aef5-505a-96c1-9c027394607a"), "Markdown")) - if Markdown !== nothing - @eval Markdown begin - __init__() = rand() - end - end - - JuliaSyntaxHighlighting = find_loaded_root_module(Base.PkgId( - Base.UUID("ac6e5ff7-fb65-4e79-a425-ec3bc9c03011"), "JuliaSyntaxHighlighting")) - if JuliaSyntaxHighlighting !== nothing - @eval JuliaSyntaxHighlighting begin - __init__() = rand() - end - end -end - -empty!(Core.ARGS) -empty!(Base.ARGS) -empty!(LOAD_PATH) -empty!(DEPOT_PATH) -empty!(Base.TOML_CACHE.d) -Base.TOML.reinit!(Base.TOML_CACHE.p, "") -Base.ACTIVE_PROJECT[] = nothing -@eval Base begin - PROGRAM_FILE = "" -end -@eval Sys begin - BINDIR = "" - STDLIB = "" -end diff --git a/contrib/juliac/Artifacts.toml b/contrib/juliac/Artifacts.toml new file mode 100644 index 0000000000000..54771b41b21f7 --- /dev/null +++ b/contrib/juliac/Artifacts.toml @@ -0,0 +1,19 @@ +[[mingw-w64]] +arch = "x86_64" +git-tree-sha1 = "b17bda08a19173572926f43a48aad5ef3d845e7c" +os = "windows" +lazy = true + + [[mingw-w64.download]] + sha256 = "53645e06775a55733580426341395c67dda20a664af83bcda76a1d052b618b59" + url = "https://github.com/JuliaLang/PackageCompiler.jl/releases/download/v2.1.24/x86_64-14.2.0-release-posix-seh-msvcrt-rt_v12-rev0.tar.gz" + +[[mingw-w64]] +arch = "i686" +git-tree-sha1 = "76b9f278e7de1d7dfdfe3a786afbe9c1e29003ea" +os = "windows" +lazy = true + + [[mingw-w64.download]] + sha256 = "d049bd771e01b02f2ca9274435f0e6f9f4f295bf2af72a8059dd851c52144910" + url = "https://github.com/JuliaLang/PackageCompiler.jl/releases/download/v2.1.24/i686-14.2.0-release-posix-dwarf-msvcrt-rt_v12-rev0.tar.gz" diff --git a/contrib/juliac/juliac-buildscript.jl b/contrib/juliac/juliac-buildscript.jl new file mode 100644 index 0000000000000..80a3fd756bcd0 --- /dev/null +++ b/contrib/juliac/juliac-buildscript.jl @@ -0,0 +1,100 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Script to run in the process that generates juliac's object file output + +# Run the verifier in the current world (before modifications), so that error +# messages and types print in their usual way. +Core.Compiler._verify_trim_world_age[] = Base.get_world_counter() + +# Initialize some things not usually initialized when output is requested +Sys.__init__() +Base.init_depot_path() +Base.init_load_path() +Base.init_active_project() +task = current_task() +task.rngState0 = 0x5156087469e170ab +task.rngState1 = 0x7431eaead385992c +task.rngState2 = 0x503e1d32781c2608 +task.rngState3 = 0x3a77f7189200c20b +task.rngState4 = 0x5502376d099035ae +uuid_tuple = (UInt64(0), UInt64(0)) +ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple) +if Base.get_bool_env("JULIA_USE_FLISP_PARSER", false) === false + Base.JuliaSyntax.enable_in_core!() +end + +if Base.JLOptions().trim != 0 + include(joinpath(@__DIR__, "juliac-trim-base.jl")) +end + +# Load user code + +import Base.Experimental.entrypoint + +# for use as C main if needed +function _main(argc::Cint, argv::Ptr{Ptr{Cchar}})::Cint + args = ccall(:jl_set_ARGS, Any, (Cint, Ptr{Ptr{Cchar}}), argc, argv)::Vector{String} + return Main.main(args) +end + +let include_result = Base.include(Main, ARGS[1]) + Core.@latestworld + if ARGS[2] == "--output-exe" + have_cmain = false + if isdefined(Main, :main) + for m in methods(Main.main) + if isdefined(m, :ccallable) + # TODO: possibly check signature and return type + have_cmain = true + break + end + end + elseif include_result isa Module && isdefined(include_result, :main) + error(""" + The `main` function must be defined in `Main`. If you are defining it inside a + module, try adding `import .$(nameof(include_result)).main` to $(ARGS[1]). + """) + end + if !have_cmain + if Base.should_use_main_entrypoint() + if hasmethod(Main.main, Tuple{Vector{String}}) + entrypoint(_main, (Cint, Ptr{Ptr{Cchar}})) + Base._ccallable("main", Cint, Tuple{typeof(_main), Cint, Ptr{Ptr{Cchar}}}) + else + error("`@main` must accept a `Vector{String}` argument.") + end + else + error("To generate an executable a `@main` function must be defined in the `Main` module.") + end + end + end + #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{Base.SubString{String}, 1}, String)) + #entrypoint(join, (Base.GenericIOBuffer{Memory{UInt8}}, Array{String, 1}, Char)) + entrypoint(Base.task_done_hook, (Task,)) + entrypoint(Base.wait, ()) + entrypoint(Base.wait_forever, ()) + entrypoint(Base.trypoptask, (Base.StickyWorkqueue,)) + entrypoint(Base.checktaskempty, ()) + if ARGS[3] == "true" + ccall(:jl_add_ccallable_entrypoints, Cvoid, ()) + end +end + +if Base.JLOptions().trim != 0 + include(joinpath(@__DIR__, "juliac-trim-stdlib.jl")) +end + +empty!(Core.ARGS) +empty!(Base.ARGS) +empty!(LOAD_PATH) +empty!(DEPOT_PATH) +empty!(Base.TOML_CACHE.d) +Base.TOML.reinit!(Base.TOML_CACHE.p, "") +Base.ACTIVE_PROJECT[] = nothing +@eval Base begin + PROGRAM_FILE = "" +end +@eval Sys begin + BINDIR = "" + STDLIB = "" +end diff --git a/contrib/juliac/juliac-trim-base.jl b/contrib/juliac/juliac-trim-base.jl new file mode 100644 index 0000000000000..1d4d9558e0bf5 --- /dev/null +++ b/contrib/juliac/juliac-trim-base.jl @@ -0,0 +1,111 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Patches to Base needed for trimming + +@eval Core begin + DomainError(@nospecialize(val), @nospecialize(msg::AbstractString)) = (@noinline; $(Expr(:new, :DomainError, :val, :msg))) +end + +(f::Base.RedirectStdStream)(io::Core.CoreSTDOUT) = Base._redirect_io_global(io, f.unix_fd) + +@eval Base begin + depwarn(msg, funcsym; force::Bool=false) = nothing + _assert_tostring(msg) = "" + reinit_stdio() = nothing + JuliaSyntax.enable_in_core!() = nothing + init_active_project() = ACTIVE_PROJECT[] = nothing + set_active_project(projfile::Union{AbstractString,Nothing}) = ACTIVE_PROJECT[] = projfile + disable_library_threading() = nothing + start_profile_listener() = nothing + invokelatest_trimmed(f, args...; kwargs...) = f(args...; kwargs...) + const invokelatest = invokelatest_trimmed + function sprint(f::F, args::Vararg{Any,N}; context=nothing, sizehint::Integer=0) where {F<:Function,N} + s = IOBuffer(sizehint=sizehint) + if context isa Tuple + f(IOContext(s, context...), args...) + elseif context !== nothing + f(IOContext(s, context), args...) + else + f(s, args...) + end + String(_unsafe_take!(s)) + end + function show_typeish(io::IO, @nospecialize(T)) + if T isa Type + show(io, T) + elseif T isa TypeVar + print(io, (T::TypeVar).name) + else + print(io, "?") + end + end + function show(io::IO, T::Type) + if T isa DataType + print(io, T.name.name) + if T !== T.name.wrapper && length(T.parameters) > 0 + print(io, "{") + first = true + for p in T.parameters + if !first + print(io, ", ") + end + first = false + if p isa Int + show(io, p) + elseif p isa Type + show(io, p) + elseif p isa Symbol + print(io, ":") + print(io, p) + elseif p isa TypeVar + print(io, p.name) + else + print(io, "?") + end + end + print(io, "}") + end + elseif T isa Union + print(io, "Union{") + show_typeish(io, T.a) + print(io, ", ") + show_typeish(io, T.b) + print(io, "}") + elseif T isa UnionAll + print(io, T.body::Type) + print(io, " where ") + print(io, T.var.name) + end + end + show_type_name(io::IO, tn::Core.TypeName) = print(io, tn.name) +end +@eval Base.Sys begin + __init_build() = nothing # VersionNumber parsing is not supported yet +end +@eval Base.GMP begin + function __init__() # VersionNumber parsing is not supported yet + try + ccall((:__gmp_set_memory_functions, libgmp), Cvoid, + (Ptr{Cvoid},Ptr{Cvoid},Ptr{Cvoid}), + cglobal(:jl_gc_counted_malloc), + cglobal(:jl_gc_counted_realloc_with_old_size), + cglobal(:jl_gc_counted_free_with_size)) + ZERO.alloc, ZERO.size, ZERO.d = 0, 0, C_NULL + ONE.alloc, ONE.size, ONE.d = 1, 1, pointer(_ONE) + catch ex + Base.showerror_nostdio(ex, "WARNING: Error during initialization of module GMP") + end + # This only works with a patched version of GMP, ignore otherwise + try + ccall((:__gmp_set_alloc_overflow_function, libgmp), Cvoid, + (Ptr{Cvoid},), + cglobal(:jl_throw_out_of_memory_error)) + ALLOC_OVERFLOW_FUNCTION[] = true + catch ex + # ErrorException("ccall: could not find function...") + if typeof(ex) != ErrorException + rethrow() + end + end + end +end diff --git a/contrib/juliac/juliac-trim-stdlib.jl b/contrib/juliac/juliac-trim-stdlib.jl new file mode 100644 index 0000000000000..0cc3f01aa92f8 --- /dev/null +++ b/contrib/juliac/juliac-trim-stdlib.jl @@ -0,0 +1,83 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Patches to stdlib needed for trimming + +let + find_loaded_root_module(key::Base.PkgId) = Base.maybe_root_module(key) + + SparseArrays = find_loaded_root_module(Base.PkgId( + Base.UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf"), "SparseArrays")) + if SparseArrays !== nothing + @eval SparseArrays.CHOLMOD begin + function __init__() + ccall((:SuiteSparse_config_malloc_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_malloc, Ptr{Cvoid})) + ccall((:SuiteSparse_config_calloc_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_calloc, Ptr{Cvoid})) + ccall((:SuiteSparse_config_realloc_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_realloc, Ptr{Cvoid})) + ccall((:SuiteSparse_config_free_func_set, :libsuitesparseconfig), + Cvoid, (Ptr{Cvoid},), cglobal(:jl_free, Ptr{Cvoid})) + end + end + end + + Artifacts = find_loaded_root_module(Base.PkgId( + Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts")) + if Artifacts !== nothing + @eval Artifacts begin + function _artifact_str( + __module__, + artifacts_toml, + name, + path_tail, + artifact_dict, + hash, + platform, + _::Val{LazyArtifacts} + ) where LazyArtifacts + # If the artifact exists, we're in the happy path and we can immediately + # return the path to the artifact: + dirs = artifacts_dirs(bytes2hex(hash.bytes)) + for dir in dirs + if isdir(dir) + return jointail(dir, path_tail) + end + end + error("Artifact not found") + end + end + end + + Pkg = find_loaded_root_module(Base.PkgId( + Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) + if Pkg !== nothing + @eval Pkg begin + __init__() = nothing # Assume the Pkg is not actually used + end + end + + StyledStrings = find_loaded_root_module(Base.PkgId( + Base.UUID("f489334b-da3d-4c2e-b8f0-e476e12c162b"), "StyledStrings")) + if StyledStrings !== nothing + @eval StyledStrings begin + __init__() = nothing # Assume that StyledStrings are not actually used + end + end + + Markdown = find_loaded_root_module(Base.PkgId( + Base.UUID("d6f4376e-aef5-505a-96c1-9c027394607a"), "Markdown")) + if Markdown !== nothing + @eval Markdown begin + __init__() = nothing # Assume that Markdown is not actually used with StyledStrings + end + end + + JuliaSyntaxHighlighting = find_loaded_root_module(Base.PkgId( + Base.UUID("ac6e5ff7-fb65-4e79-a425-ec3bc9c03011"), "JuliaSyntaxHighlighting")) + if JuliaSyntaxHighlighting !== nothing + @eval JuliaSyntaxHighlighting begin + __init__() = nothing # Assume the JuliaSyntaxHighlighting is not actually used with StyledStrings + end + end +end diff --git a/contrib/juliac.jl b/contrib/juliac/juliac.jl similarity index 60% rename from contrib/juliac.jl rename to contrib/juliac/juliac.jl index 7087462afc7a1..eb6277785c789 100644 --- a/contrib/juliac.jl +++ b/contrib/juliac/juliac.jl @@ -1,8 +1,12 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + # Julia compiler wrapper script # NOTE: The interface and location of this script are considered unstable/experimental +using LazyArtifacts + module JuliaConfig - include(joinpath(@__DIR__, "julia-config.jl")) + include(joinpath(@__DIR__, "..", "julia-config.jl")) end julia_cmd = `$(Base.julia_cmd()) --startup-file=no --history-file=no` @@ -28,9 +32,61 @@ if help !== nothing exit(0) end +# Copied from PackageCompiler +# https://github.com/JuliaLang/PackageCompiler.jl/blob/1c35331d8ef81494f054bbc71214811253101993/src/PackageCompiler.jl#L147-L190 +function get_compiler_cmd(; cplusplus::Bool=false) + cc = get(ENV, "JULIA_CC", nothing) + path = nothing + @static if Sys.iswindows() + path = joinpath(LazyArtifacts.artifact"mingw-w64", + "extracted_files", + (Int==Int64 ? "mingw64" : "mingw32"), + "bin", + cplusplus ? "g++.exe" : "gcc.exe") + compiler_cmd = `$path` + end + if cc !== nothing + compiler_cmd = Cmd(Base.shell_split(cc)) + path = nothing + elseif !Sys.iswindows() + compilers_cpp = ("g++", "clang++") + compilers_c = ("gcc", "clang") + found_compiler = false + if cplusplus + for compiler in compilers_cpp + if Sys.which(compiler) !== nothing + compiler_cmd = `$compiler` + found_compiler = true + break + end + end + end + if !found_compiler + for compiler in compilers_c + if Sys.which(compiler) !== nothing + compiler_cmd = `$compiler` + found_compiler = true + if cplusplus && !WARNED_CPP_COMPILER[] + @warn "could not find a c++ compiler (g++ or clang++), falling back to $compiler, this might cause link errors" + WARNED_CPP_COMPILER[] = true + end + break + end + end + end + found_compiler || error("could not find a compiler, looked for ", + join(((cplusplus ? compilers_cpp : ())..., compilers_c...), ", ", " and ")) + end + if path !== nothing + compiler_cmd = addenv(compiler_cmd, "PATH" => string(ENV["PATH"], ";", dirname(path))) + end + return compiler_cmd +end + # arguments to forward to julia compilation process julia_args = [] enable_trim::Bool = false +project::String = "--project=$(Base.active_project())" let i = 1 while i <= length(ARGS) @@ -52,6 +108,8 @@ let i = 1 push!(julia_args, arg) # forwarded arg elseif arg == "--experimental" push!(julia_args, arg) # forwarded arg + elseif startswith(arg, "--proj") + global project = arg else if arg[1] == '-' || !isnothing(file) println("Unexpected argument `$arg`") @@ -80,6 +138,7 @@ function get_rpath(; relative::Bool = false) end end +cc = get_compiler_cmd() absfile = abspath(file) cflags = JuliaConfig.cflags(; framework=false) cflags = Base.shell_split(cflags) @@ -88,16 +147,13 @@ allflags = Base.shell_split(allflags) rpath = get_rpath(; relative = relative_rpath) rpath = Base.shell_split(rpath) tmpdir = mktempdir(cleanup=false) -initsrc_path = joinpath(tmpdir, "init.c") -init_path = joinpath(tmpdir, "init.a") img_path = joinpath(tmpdir, "img.a") bc_path = joinpath(tmpdir, "img-bc.a") - function precompile_env() # Pre-compile the environment # (otherwise obscure error messages will occur) - cmd = addenv(`$julia_cmd --project=$(Base.active_project()) -e "using Pkg; Pkg.precompile()"`) + cmd = addenv(`$julia_cmd $project -e "using Pkg; Pkg.precompile()"`) verbose && println("Running: $cmd") if !success(pipeline(cmd; stdout, stderr)) println(stderr, "\nError encountered during pre-compilation of environment.") @@ -115,26 +171,12 @@ function compile_products(enable_trim::Bool) end # Compile the Julia code - cmd = addenv(`$julia_cmd_target --project=$(Base.active_project()) --output-o $img_path --output-incremental=no $strip_args $julia_args $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) + cmd = addenv(`$julia_cmd_target $project --output-o $img_path --output-incremental=no $strip_args $julia_args $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) verbose && println("Running: $cmd") if !success(pipeline(cmd; stdout, stderr)) println(stderr, "\nFailed to compile $file") exit(1) end - - # Compile the initialization code - open(initsrc_path, "w") do io - print(io, """ - #include - __attribute__((constructor)) void static_init(void) { - if (jl_is_initialized()) - return; - julia_init(JL_IMAGE_IN_MEMORY); - jl_exception_clear(); - } - """) - end - run(`cc $(cflags) -g -c -o $init_path $initsrc_path`) end function link_products() @@ -150,11 +192,11 @@ function link_products() julia_libs = Base.shell_split(Base.isdebugbuild() ? "-ljulia-debug -ljulia-internal-debug" : "-ljulia -ljulia-internal") try if output_type == "--output-lib" - cmd2 = `cc $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $init_path $(julia_libs)` + cmd2 = `$(cc) $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` elseif output_type == "--output-sysimage" - cmd2 = `cc $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` + cmd2 = `$(cc) $(allflags) $(rpath) -o $outname -shared -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` else - cmd2 = `cc $(allflags) $(rpath) -o $outname -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $init_path $(julia_libs)` + cmd2 = `$(cc) $(allflags) $(rpath) -o $outname -Wl,$(Base.Linking.WHOLE_ARCHIVE) $img_path -Wl,$(Base.Linking.NO_WHOLE_ARCHIVE) $(julia_libs)` end verbose && println("Running: $cmd2") run(cmd2) diff --git a/contrib/mac/app/Makefile b/contrib/mac/app/Makefile index 81b7e47cdf2cf..70436a857c265 100644 --- a/contrib/mac/app/Makefile +++ b/contrib/mac/app/Makefile @@ -47,8 +47,8 @@ dmg/$(APP_NAME): startup.applescript julia.icns plutil -insert CFBundleVersion -string "$(JULIA_VERSION_OPT_COMMIT)" $@/Contents/Info.plist plutil -insert NSHumanReadableCopyright -string "$(APP_COPYRIGHT)" $@/Contents/Info.plist -mkdir -p $@/Contents/Resources/julia - make -C $(JULIAHOME) binary-dist - tar zxf $(JULIAHOME)/$(JULIA_BINARYDIST_FILENAME).tar.gz -C $@/Contents/Resources/julia --strip-components 1 + $(MAKE) -C $(JULIAHOME) binary-dist + $(TAR) -xzf $(JULIAHOME)/$(JULIA_BINARYDIST_FILENAME).tar.gz -C $@/Contents/Resources/julia --strip-components 1 find $@/Contents/Resources/julia -type f -exec chmod -w {} \; # Even though the tarball may already be signed, we re-sign here to make it easier to add # unsigned executables (like the app launcher) and whatnot, without needing to maintain lists diff --git a/contrib/mac/app/startup.applescript b/contrib/mac/app/startup.applescript index 9964049f34ed6..d7b46cec1a89d 100644 --- a/contrib/mac/app/startup.applescript +++ b/contrib/mac/app/startup.applescript @@ -1,4 +1,3 @@ set RootPath to (path to me) set JuliaPath to POSIX path of ((RootPath as text) & "Contents:Resources:julia:bin:julia") -set JuliaFile to POSIX file JuliaPath -tell application id "com.apple.finder" to open JuliaFile +do shell script "open -a Terminal '" & JuliaPath & "'" diff --git a/contrib/pgo-lto/Makefile b/contrib/pgo-lto/Makefile index ddd86f5d5b39a..5902d4ad08151 100644 --- a/contrib/pgo-lto/Makefile +++ b/contrib/pgo-lto/Makefile @@ -32,7 +32,7 @@ STAGE2_FLAGS:=LDFLAGS="-fuse-ld=lld -flto=thin -Wl,--undefined-version -fprofile CFLAGS="-fprofile-use=$(PROFILE_FILE)" $\ CXXFLAGS="-fprofile-use=$(PROFILE_FILE)" -COMMON_FLAGS:=USECLANG=1 USE_BINARYBUILDER_LLVM=0 +COMMON_FLAGS:=USE_BINARYBUILDER_LLVM=0 all: stage2 # Default target as first in file diff --git a/contrib/print_sorted_stdlibs.jl b/contrib/print_sorted_stdlibs.jl index 6bc2023c4f1cc..c4cf391efb623 100644 --- a/contrib/print_sorted_stdlibs.jl +++ b/contrib/print_sorted_stdlibs.jl @@ -12,12 +12,13 @@ function check_flag(flag) end if check_flag("--help") || check_flag("-h") - println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls] [--exclude-sysimage]") + println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls] [--exclude-sysimage] [--only-sysimg]") end # Allow users to ask for JLL or no JLLs exclude_jlls = check_flag("--exclude-jlls") exclude_sysimage = check_flag("--exclude-sysimage") +only_sysimage = check_flag("--only-sysimg") # Default to the `stdlib/vX.Y` directory STDLIB_DIR = get(ARGS, 1, joinpath(@__DIR__, "..", "usr", "share", "julia", "stdlib")) @@ -81,9 +82,19 @@ if exclude_jlls filter!(p -> !endswith(p, "_jll"), sorted_projects) end -if exclude_sysimage - loaded_modules = Set(map(k->k.name, collect(keys(Base.loaded_modules)))) - filter!(p->!in(p, loaded_modules), sorted_projects) +if only_sysimage && exclude_sysimage + println(stderr, "Warning: --only-sysimg and --exclude-sysimage are mutually exclusive. Prioritizing --only-sysimg.") + exclude_sysimage = false +end + +if only_sysimage || exclude_sysimage + loaded_modules_set = Set(map(k->k.name, collect(keys(Base.loaded_modules)))) + + if only_sysimage + filter!(p -> in(p, loaded_modules_set), sorted_projects) + else + filter!(p -> !in(p, loaded_modules_set), sorted_projects) + end end # Print out sorted projects, ready to be pasted into `sysimg.jl` @@ -92,6 +103,9 @@ println(" # Stdlibs sorted in dependency, then alphabetical, order by contrib if exclude_jlls println(" # Run with the `--exclude-jlls` option to filter out all JLL packages") end +if only_sysimage + println(" # Run with the `--only-sysimg` option to filter for only packages included in the system image") +end if exclude_sysimage println(" # Run with the `--exclude-sysimage` option to filter out all packages included in the system image") end diff --git a/contrib/refresh_checksums.mk b/contrib/refresh_checksums.mk index fa7cddc705958..77921858f2b6e 100644 --- a/contrib/refresh_checksums.mk +++ b/contrib/refresh_checksums.mk @@ -24,7 +24,7 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS)) NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS)) # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded: -BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia +BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib zstd libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia BB_GCC_EXPANDED_PROJECTS=openblas csl BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld # These are non-BB source-only deps @@ -58,10 +58,6 @@ checksum-$(1)-$(2)-$(3): clean-$(1) # Add this guy to his project target checksum-$(1): checksum-$(1)-$(2)-$(3) -# Add a dependency to the pack target -# TODO: can we make this so it only adds an ordering but not a dependency? -pack-checksum-$(1): | checksum-$(1) - # Add this guy to the `checksum` and `pack-checksum` default targets (e.g. `make -f contrib/refresh_checksums.mk openblas`) checksum: checksum-$1 $1 pack-checksum: pack-checksum-$1 @@ -100,7 +96,7 @@ checksum-doc-unicodedata: all: checksum-doc-unicodedata .PHONY: checksum-doc-unicodedata -# merge substring project names to avoid races +# merge substring project names (llvm and llvm-tools, libsuitesparse and suitesparse) to avoid races pack-checksum-llvm-tools: | pack-checksum-llvm @# nothing to do but disable the prefix rule pack-checksum-llvm: | checksum-llvm-tools @@ -110,18 +106,21 @@ pack-checksum-compilersupportlibraries: | checksum-csl pack-checksum-libsuitesparse: | pack-checksum-suitesparse @# nothing to do but disable the prefix rule pack-checksum-suitesparse: | checksum-libsuitesparse -# This is a bit tricky: we want llvmunwind to be separate from unwind and llvm, +# This is a bit tricky: we want llvmunwind, clang, and lld to be separate from unwind and llvm, # so we add a rule to process those first pack-checksum-llvm pack-checksum-unwind: | pack-checksum-llvmunwind -# and the name for LLVMLibUnwind is awkward, so handle that with a regex -pack-checksum-llvmunwind: | pack-checksum-llvm.*unwind +pack-checksum-llvm: | pack-checksum-clang pack-checksum-lld +# and the name for LLVMLibUnwind is awkward, so handle that packing with a regex +checksum-llvm.*unwind: checksum-llvmunwind + @# nothing to do but disable the prefix rule +pack-checksum-llvmunwind: | pack-checksum-llvm.*unwind # override general rule below cd "$(JULIAHOME)/deps/checksums" && mv 'llvm.*unwind' llvmunwind clean-%: FORCE -rm "$(JULIAHOME)/deps/checksums"/'$*' # define how to pack parallel checksums into a single file format -pack-checksum-%: FORCE +pack-checksum-%: FORCE | checksum-% @echo making "$(JULIAHOME)/deps/checksums/"'$*' @cd "$(JULIAHOME)/deps/checksums" && \ for each in $$(ls | grep -i '$*'); do \ diff --git a/contrib/tsan/Make.user.tsan b/contrib/tsan/Make.user.tsan index b192c36e4cfee..252a17ba86497 100644 --- a/contrib/tsan/Make.user.tsan +++ b/contrib/tsan/Make.user.tsan @@ -3,7 +3,6 @@ BINDIR=$(TOOLCHAIN)/usr/bin TOOLDIR=$(TOOLCHAIN)/usr/tools # use our new toolchain -USECLANG=1 override CC=$(TOOLDIR)/clang override CXX=$(TOOLDIR)/clang++ @@ -11,3 +10,5 @@ USE_BINARYBUILDER_LLVM=1 override SANITIZE=1 override SANITIZE_THREAD=1 +override CROSS_BOOTSTRAP_JULIA=$(BUILDROOT)/../bootstrap/usr/bin/julia +override CROSS_BOOTSTRAP_SYSBASE=$(BUILDROOT)/../bootstrap/usr/lib/julia/sysbase.$(SHLIB_EXT) diff --git a/contrib/tsan/build.sh b/contrib/tsan/build.sh index 2c4ba3b1bde95..de3951fbe8bd5 100755 --- a/contrib/tsan/build.sh +++ b/contrib/tsan/build.sh @@ -3,7 +3,7 @@ # # Usage: -# contrib/tsan/build.sh [...] +# contrib/tsan/build.sh [-j] [...] # # Build TSAN-enabled julia. Given a workspace directory , build # TSAN-enabled julia in /tsan. Required toolss are install under @@ -13,6 +13,16 @@ # make target is `debug`. set -ue +set -x + +JOBS=1 +while getopts j: opt +do + case $opt in + j) JOBS="$OPTARG";; + esac +done +shift $((OPTIND-1)) # `$WORKSPACE` is a directory in which we create `toolchain` and `tsan` # sub-directories. @@ -44,6 +54,12 @@ fi make -C "$TOOLCHAIN/deps" install-clang install-llvm-tools +echo +echo "Building bootstrap Julia..." +BUILD="$WORKSPACE/bootstrap" + +make -j "$JOBS" O="$BUILD" julia-src-release julia-sysbase-release + echo echo "Building Julia..." @@ -54,4 +70,6 @@ if [ ! -d "$BUILD" ]; then fi cd "$BUILD" # so that we can pass `-C src` to `make` -make "$@" +# Reporting tsan warnings will interfere with bootstrapping. +export TSAN_OPTIONS="report_bugs=0 exitcode=0" +make -j "$JOBS" "$@" diff --git a/deps/Makefile b/deps/Makefile index 4ad4df9ec4ba4..392b4fad2b2e2 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -42,9 +42,10 @@ ifeq ($(USE_SYSTEM_LIBBLASTRAMPOLINE), 0) DEP_LIBS += blastrampoline endif -ifeq ($(USE_SYSTEM_CSL), 0) +# We need to run this whether or not USE_SYSTEM_CSL is set. +# If it is, this target copies the system CSLs into the location our +# build system expects. DEP_LIBS += csl -endif ifeq ($(SANITIZE), 1) DEP_LIBS += sanitizers @@ -72,10 +73,12 @@ endif endif endif +PATCHELF_MANIFEST := ifneq (,$(findstring $(OS),Linux FreeBSD OpenBSD)) ifeq ($(USE_SYSTEM_PATCHELF), 0) DEP_LIBS += patchelf PATCHELF:=$(build_depsbindir)/patchelf +PATCHELF_MANIFEST:=$(build_prefix)/manifest/patchelf else PATCHELF:=patchelf endif @@ -157,6 +160,10 @@ ifeq ($(USE_SYSTEM_ZLIB), 0) DEP_LIBS += zlib endif +ifeq ($(USE_SYSTEM_ZSTD), 0) +DEP_LIBS += zstd +endif + ifeq ($(USE_SYSTEM_P7ZIP), 0) DEP_LIBS += p7zip endif @@ -204,7 +211,7 @@ DEP_LIBS_STAGED := $(DEP_LIBS) # list all targets DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \ openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \ - objconv openssl libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \ + objconv openssl libssh2 nghttp2 curl libgit2 libwhich zlib zstd p7zip csl \ sanitizers libsuitesparse lld libtracyclient ittapi nvtx JuliaSyntax \ terminfo mmtk_julia DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL) @@ -256,6 +263,7 @@ include $(SRCDIR)/openblas.mk include $(SRCDIR)/utf8proc.mk include $(SRCDIR)/libsuitesparse.mk include $(SRCDIR)/zlib.mk +include $(SRCDIR)/zstd.mk include $(SRCDIR)/unwind.mk include $(SRCDIR)/gmp.mk include $(SRCDIR)/mpfr.mk diff --git a/deps/blastrampoline.version b/deps/blastrampoline.version index 1e4a75305a4dd..bb711cfcd67ec 100644 --- a/deps/blastrampoline.version +++ b/deps/blastrampoline.version @@ -4,6 +4,6 @@ BLASTRAMPOLINE_JLL_NAME := libblastrampoline ## source build -BLASTRAMPOLINE_VER := 5.12.0 -BLASTRAMPOLINE_BRANCH=v5.12.0 -BLASTRAMPOLINE_SHA1=b127bc8dd4758ffc064340fff2aef4ead552f386 +BLASTRAMPOLINE_VER := 5.13.1 +BLASTRAMPOLINE_BRANCH=v5.13.1 +BLASTRAMPOLINE_SHA1=f26278e83ddc9035ae7695da597f1a5b26a4c62b diff --git a/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 new file mode 100644 index 0000000000000..092778b7bb675 --- /dev/null +++ b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/md5 @@ -0,0 +1 @@ +eef0f8463d4a7ba1c697f666e815b0da diff --git a/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 new file mode 100644 index 0000000000000..abbec567981d6 --- /dev/null +++ b/deps/checksums/DelimitedFiles-a982d5cf46061593c98b4d80fcc64dd42e6cba74.tar.gz/sha512 @@ -0,0 +1 @@ +e119db9546e698872e5e4cdf027a1c31e694f36c1e775a7d9361ad0db5d6a8e1804732090e68b40765a519ec3f7aa45d40c8c5e22275262303c100367bd681b3 diff --git a/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/md5 b/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/md5 deleted file mode 100644 index 9c6e4e44927fe..0000000000000 --- a/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ee5afca99801e37fd3a42a9455ae986b diff --git a/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/sha512 b/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/sha512 deleted file mode 100644 index 69a50a7282781..0000000000000 --- a/deps/checksums/DelimitedFiles-db79c842f95f55b1f8d8037c0d3363ab21cd3b90.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -2adec92de521df1668eb13f2903ffdb01efd6afa5f04ce6fbd1737caa4948f7b629cdda7f75a895853a0cd49dccf8b388860d5c19c29e4d4aad6c7f8fa6b7209 diff --git a/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 new file mode 100644 index 0000000000000..ca3d9730181f4 --- /dev/null +++ b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/md5 @@ -0,0 +1 @@ +fdf2e62fcaed6aa5ad69bca405329675 diff --git a/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 new file mode 100644 index 0000000000000..186dfb34221b6 --- /dev/null +++ b/deps/checksums/Distributed-3679026d7b510befdedfa8c6497e3cb032f9cea1.tar.gz/sha512 @@ -0,0 +1 @@ +2361fc4ccad83139cf728f14fa38466f075fbf93dddfac533af8f5c0c35d6b86511881b7b97a95ee59f858ba9a33428d3514ac3bd605b745b002a673acfc3190 diff --git a/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/md5 b/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/md5 deleted file mode 100644 index cdf885890db7c..0000000000000 --- a/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -bf358a22ed4aa0d57b8672ef81fb2b44 diff --git a/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/sha512 b/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/sha512 deleted file mode 100644 index 171fd2fdbf512..0000000000000 --- a/deps/checksums/Distributed-51e52978481835413d15b589919aba80dd85f890.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -399ab073d8c8cd1404e2e89ce2753486c782aa18646d56d8508c0a929c6a312e8bac2c791eddc01966f99cec1af34e56bbdccd8d75196b26f0b1bbb1fa8bd9ac diff --git a/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 new file mode 100644 index 0000000000000..ed84729b00910 --- /dev/null +++ b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/md5 @@ -0,0 +1 @@ +ac2209576b09a9c7a4da02a058e9ec87 diff --git a/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 new file mode 100644 index 0000000000000..aa81c3a943f3f --- /dev/null +++ b/deps/checksums/Downloads-06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3.tar.gz/sha512 @@ -0,0 +1 @@ +0636b4e7f17d8747408949b77470f5f7f1b4c4b618dc905e20d84d0a5ca4713834e8a4d6a10bcccf7a28982edefe03eeff11bb0aca7d8cb49cb230b7e1ddc06f diff --git a/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/md5 b/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/md5 deleted file mode 100644 index 09301a8f7a65e..0000000000000 --- a/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ad1074d6518982b6e1a5ecb6537aece1 diff --git a/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/sha512 b/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/sha512 deleted file mode 100644 index 5ef1b996bb785..0000000000000 --- a/deps/checksums/Downloads-1199e9039dcce6072601c3f29cea8da4a0acbff6.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -7fc7ec01a354d0c4afd6d8c7ad5423afae2d128683f1e2006b0a37b6f34e6b49ad2ed6286c7fc0b5d74392fdbad00e12cc8871c55b28def078d7f962313840a7 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 new file mode 100644 index 0000000000000..4a586ce45dbcc --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/md5 @@ -0,0 +1 @@ +778d62517cab8b4a95920337631f9439 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 new file mode 100644 index 0000000000000..5a88e30a10a3f --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c.tar.gz/sha512 @@ -0,0 +1 @@ +95db08cd6775920271e347bae3ee4a68ef532ec6dceb834a63bc8f918b785c042c2bed9babf9ca76ff610ee2782130a69b4ecf158e4beae6361115cdef57dc51 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 deleted file mode 100644 index 2a4b55e15ab2d..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ed0ccc4434fc70b06e8ea1ddb8141511 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 deleted file mode 100644 index 456f1ee64ca0b..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -04efea853a1c1bfbf5baf4d2908ce492a5ff3029bca73a004280aa116157b6b678a5f9fd6a115f9c57a625d0841d3fb96c8d68ec467e5bc4a743272bee84c8c7 diff --git a/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 new file mode 100644 index 0000000000000..264c8e5ea6293 --- /dev/null +++ b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/md5 @@ -0,0 +1 @@ +b3a67e92f5a9d5832699623e34abf1b2 diff --git a/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 new file mode 100644 index 0000000000000..78f04bb4becb8 --- /dev/null +++ b/deps/checksums/LibCURL-038790a793203248362cf2bd8d85e42f8c56a72d.tar.gz/sha512 @@ -0,0 +1 @@ +d558321aa6099ddb61cc402f85d3ea7f0d5c875aee1371f10e6b50c53d8b578045a132a3fd361b96fabf44658b364496d0377371d1ec232962fc8450eb242217 diff --git a/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/md5 b/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/md5 deleted file mode 100644 index f14b87c21f5ed..0000000000000 --- a/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -e8c53aa3fb963c80921787d5d565eb2c diff --git a/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/sha512 b/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/sha512 deleted file mode 100644 index ab24e6a9516c3..0000000000000 --- a/deps/checksums/LibCURL-a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -8e442ea834299df9c02acb87226c121395ad8e550025ac5ee1103df09c6ff43817e9e48dd1bcbc92c80331ef3ddff531962430269115179acbec2bab2de5b011 diff --git a/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/md5 b/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/md5 deleted file mode 100644 index cf11a21c45995..0000000000000 --- a/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -33933c31906af2086702d38b9d8eb90b diff --git a/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/sha512 b/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/sha512 deleted file mode 100644 index e11b0c3957407..0000000000000 --- a/deps/checksums/LinearAlgebra-1ce842652c07b33289046236b09bc59bad43dbeb.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -3783b8c3a46c4db5cf750a99c6753e1bf377b2d979a85df8e26c81f41cb3c75a41c28a487fce332e19ba8287c2531d440b248a6f5aedc93e8fa6673d60456c33 diff --git a/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 new file mode 100644 index 0000000000000..63847447093fd --- /dev/null +++ b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/md5 @@ -0,0 +1 @@ +6ea57484fcb60da4bf9d7c646fe73b2e diff --git a/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 new file mode 100644 index 0000000000000..fd0479374f265 --- /dev/null +++ b/deps/checksums/LinearAlgebra-2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023.tar.gz/sha512 @@ -0,0 +1 @@ +7b75d79804e73a3fc7140fe0636b06a4d0267c96ee6d6b1dbe8eedeefd79a1f8a82501bed4a3038ba309284dc0291e7424463ced1f1b76eb20f5bd57c7c0a7b5 diff --git a/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 new file mode 100644 index 0000000000000..fbb7d5b86627f --- /dev/null +++ b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/md5 @@ -0,0 +1 @@ +afab093d162a62d5a488894f33d3b396 diff --git a/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 new file mode 100644 index 0000000000000..146d3a3d1bad8 --- /dev/null +++ b/deps/checksums/NetworkOptions-532992fcc0f1d02df48374969cbae37e34c01360.tar.gz/sha512 @@ -0,0 +1 @@ +14b41cc9c93e2f9eeaa9499d65b8c42ee80691cbd533ef6cafabdb6e94c7cf31eee00fb603ca70dfe86930c871419cf17a8f05c0a76bd379a8bbf705b875dfe2 diff --git a/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/md5 b/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/md5 deleted file mode 100644 index 87111ac121562..0000000000000 --- a/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -b851cab503506c37af6e4c861d81b8ce diff --git a/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/sha512 b/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/sha512 deleted file mode 100644 index 79f9e269ff599..0000000000000 --- a/deps/checksums/NetworkOptions-c090626d3feee6d6a5c476346d22d6147c9c6d2d.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -4cba5c531e5e7205bb6d7f0179da8b29ca7c4dcf42f27de5f70be7674efc1fa92ea22e134e6584743e2905edbd754838d8a02f6ba7811c7a5b99ab9db3bde596 diff --git a/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/md5 b/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/md5 deleted file mode 100644 index b9ff96ecde0ea..0000000000000 --- a/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -0d36bf3edd61c39515a647719e5af8b9 diff --git a/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/sha512 b/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/sha512 deleted file mode 100644 index 093e0a263c7e5..0000000000000 --- a/deps/checksums/Pkg-26bdeeeb10f3ab0c6cf7478cb8454a800b4578be.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -c5c91b56f3548a13851081e589ec7347a7ca8ab1de792645b1ec04ee0e2e57678fdb2721d6243450e278457b0edc42ece105d205a61ab9cdcb6ea293bbcbba9f diff --git a/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 new file mode 100644 index 0000000000000..f1a89122167f9 --- /dev/null +++ b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/md5 @@ -0,0 +1 @@ +474392cbf2fe976bca300944fc62f5ab diff --git a/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 new file mode 100644 index 0000000000000..e3a190082f455 --- /dev/null +++ b/deps/checksums/Pkg-d94f8a1d946f90b00b836afc1dedf034604c6627.tar.gz/sha512 @@ -0,0 +1 @@ +99c1b57efd80eade5f89f350d244bedc80717809c108056bf28b2f6213dba040bbed2ef4869f97a624efa1ca8474504c087481860758bb366025093a8bec34b5 diff --git a/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 new file mode 100644 index 0000000000000..ef6edc13c3a2c --- /dev/null +++ b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/md5 @@ -0,0 +1 @@ +ab91b3b95af44071020c564f6cb2b83f diff --git a/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 new file mode 100644 index 0000000000000..26889e062b48d --- /dev/null +++ b/deps/checksums/SHA-169a3369026ee767c454e5b1ca70c62c4db5a933.tar.gz/sha512 @@ -0,0 +1 @@ +ea90fdf05a0c345ff697b91bea45ab5458123cb7d7bad53cebeea2304822545ec422012f116090d2c9491c185a9de61ad44b2eb71237a69e959737a140d8cf71 diff --git a/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/md5 b/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/md5 deleted file mode 100644 index cbd2acb2c6f66..0000000000000 --- a/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -7b511b7dab411685206d0d90cc1fb56e diff --git a/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/sha512 b/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/sha512 deleted file mode 100644 index 5201bbdcc40f9..0000000000000 --- a/deps/checksums/SHA-4451e1362e425bcbc1652ecf55fc0e525b18fb63.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -960367406d80e46e8e742bcb0f7f0e4b089b664c2321ca82953eb760b325693ae57f431d891ccf56c3ab9146bc29682d2d1767bc635f4dbe6dd4d80030a42487 diff --git a/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 new file mode 100644 index 0000000000000..a660fb96bd9d8 --- /dev/null +++ b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/md5 @@ -0,0 +1 @@ +3615e2464cfab2c7d00ae53dedc0510b diff --git a/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 new file mode 100644 index 0000000000000..be8fbd8d2e1a3 --- /dev/null +++ b/deps/checksums/SparseArrays-30201abcb41e558a4b5cc23b11dc5676c5655c0b.tar.gz/sha512 @@ -0,0 +1 @@ +1998b694bd9451ac92517b70c25fc2683b1482ca1d547500cec8e09eecbb615167aea94c16cae899a32b5cb2fcb842ce081fa90641a2921ea3cfa0666e9ed385 diff --git a/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/md5 b/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/md5 deleted file mode 100644 index 541d74f9eee17..0000000000000 --- a/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ab4ff8bd5749c1c5915ecf9ac1ae82a3 diff --git a/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/sha512 b/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/sha512 deleted file mode 100644 index 46bcc3ea9668f..0000000000000 --- a/deps/checksums/SparseArrays-f3610c07fe0403792743d9c9802a25642a5f2d18.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -0586606d86bca352f6610ba2baeecbe997b8c90ee798c9ab89f42bf3c94c5f23c56ea75fe49d8ccc39d8b25f02592db879c95affa30f98cb45024b177157249a diff --git a/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 new file mode 100644 index 0000000000000..c5f56d9064e92 --- /dev/null +++ b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/md5 @@ -0,0 +1 @@ +0b60da1286ca8a978cf3c27b8fbc0601 diff --git a/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 new file mode 100644 index 0000000000000..8cd97202f18ba --- /dev/null +++ b/deps/checksums/Statistics-22dee82f9824d6045e87aa4b97e1d64fe6f01d8d.tar.gz/sha512 @@ -0,0 +1 @@ +2e03fe3b79dfb299caa0ac23e045bf26addeb3d38ef0b5e5430966dc227e771cfd84722d8bb80aaaedc0a988fbd2228bebf56c3e7d80b3e8993c623d8436660c diff --git a/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/md5 b/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/md5 deleted file mode 100644 index 600c561d0cf14..0000000000000 --- a/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -5235ac479da042d5dc3c572c473b7219 diff --git a/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/sha512 b/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/sha512 deleted file mode 100644 index 2f663a3d7c44d..0000000000000 --- a/deps/checksums/Statistics-77bd5707f143eb624721a7df28ddef470e70ecef.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -0c02ccf1b4988fc701209afb949f27e6f675f37a628385d3f28dc9ea333fed38ce1ca77b001e58fdbe15af833bbe98598cbf478cef21a98b37d54acfe52270b6 diff --git a/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 new file mode 100644 index 0000000000000..0285925aaaa2a --- /dev/null +++ b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/md5 @@ -0,0 +1 @@ +1762267d32633457c7c64067c1c9d871 diff --git a/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 new file mode 100644 index 0000000000000..f7d817c2f1e84 --- /dev/null +++ b/deps/checksums/StyledStrings-1aafc2f3abb6a977ee36a87206f9ce6446a8ae86.tar.gz/sha512 @@ -0,0 +1 @@ +72ca1b52b8f0c4e3d3ca58526cd34a3f1486c4fe373930bf513570f056700cddc65a4666d5b7767305b115ec0352639c4caaf03b9378911d596f6add3cbb762b diff --git a/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/md5 b/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/md5 deleted file mode 100644 index 0fd8e8966e068..0000000000000 --- a/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -411277f3701cc3e286ec8a84ccdf6f11 diff --git a/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/sha512 b/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/sha512 deleted file mode 100644 index 0b495aefef55d..0000000000000 --- a/deps/checksums/StyledStrings-8985a37ac054c37d084a03ad2837208244824877.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -95a7e92389f6fd02d3bec17ec0201ba41316aa2d7c321b14af88ccce8246fd0000ed2c0cc818f87cb81f7134304233db897f656426a00caac1bc7635056260c2 diff --git a/deps/checksums/UnicodeData-13.0.0.txt/md5 b/deps/checksums/UnicodeData-13.0.0.txt/md5 deleted file mode 100644 index 2b3ffc179ce01..0000000000000 --- a/deps/checksums/UnicodeData-13.0.0.txt/md5 +++ /dev/null @@ -1 +0,0 @@ -85879f1976cc8eb739ee5585a93938e2 diff --git a/deps/checksums/UnicodeData-13.0.0.txt/sha512 b/deps/checksums/UnicodeData-13.0.0.txt/sha512 deleted file mode 100644 index a93ba01e7ddda..0000000000000 --- a/deps/checksums/UnicodeData-13.0.0.txt/sha512 +++ /dev/null @@ -1 +0,0 @@ -1a4a662e2ab33469976bf5f91aa6933ed9b73f6d4179a2daffb349e1869d7d6cfa885b164e82d15dcdad7458cd451c81add58d875eb0c70de854589dc97b2055 diff --git a/deps/checksums/UnicodeData-16.0.0.txt/md5 b/deps/checksums/UnicodeData-16.0.0.txt/md5 new file mode 100644 index 0000000000000..79ae5d27eff0e --- /dev/null +++ b/deps/checksums/UnicodeData-16.0.0.txt/md5 @@ -0,0 +1 @@ +f50a0495d2000b7d6dd979cb40e00ba2 diff --git a/deps/checksums/UnicodeData-16.0.0.txt/sha512 b/deps/checksums/UnicodeData-16.0.0.txt/sha512 new file mode 100644 index 0000000000000..05b998b3724ba --- /dev/null +++ b/deps/checksums/UnicodeData-16.0.0.txt/sha512 @@ -0,0 +1 @@ +963e5a1e7a480873c6e66d53e9288232b5029942477a694a0bfafa7e994c55189cb9c2f8d00255de84b82b72ff6066932e5531e3664fb422eeef9c69ea25d80e diff --git a/deps/checksums/blastrampoline b/deps/checksums/blastrampoline index 9e007f6055cf9..7870242560f34 100644 --- a/deps/checksums/blastrampoline +++ b/deps/checksums/blastrampoline @@ -1,38 +1,38 @@ -blastrampoline-b127bc8dd4758ffc064340fff2aef4ead552f386.tar.gz/md5/395f2035bcb52e886b55ac926a7bf183 -blastrampoline-b127bc8dd4758ffc064340fff2aef4ead552f386.tar.gz/sha512/9ae0fe2ca75dc0b2c784d5b7248caca29ed6d44258743ee2b32827032734757e9078dd6bcdf80a02b042deb5c7ca7b4e5be392be6700efde91427091fb53a03f -libblastrampoline.v5.12.0+0.aarch64-apple-darwin.tar.gz/md5/9a18b39bb575d0112834992043d302c0 -libblastrampoline.v5.12.0+0.aarch64-apple-darwin.tar.gz/sha512/4e406b155149414d3e4fd5db49ab56a87ed13577ebb399eaf8a251692c0b84e639c6e1a4eb20863e2638c31add0241ca916e57f91bb5a4aed07e2c56cc580870 -libblastrampoline.v5.12.0+0.aarch64-linux-gnu.tar.gz/md5/e100e93f0d6a104fc66c9f78a67150c5 -libblastrampoline.v5.12.0+0.aarch64-linux-gnu.tar.gz/sha512/f7e0c379e32d8163dbb4919b77e9637e1b16cf26618b9260222cf985bfab9ca3f36bebccd0e8360af68db925035c82127ba85d46b4a6578961dde6a049c7cf93 -libblastrampoline.v5.12.0+0.aarch64-linux-musl.tar.gz/md5/814a79e8cfe8744ca5a2a722f007fcaa -libblastrampoline.v5.12.0+0.aarch64-linux-musl.tar.gz/sha512/bc886b199500fc4245a95446d4c862fc636711e0875a9d5cf9aef661d819d00324adfd3e037d9c03e274be26034353d033fb041e7608ecef222e1d154f38337d -libblastrampoline.v5.12.0+0.aarch64-unknown-freebsd.tar.gz/md5/9b9a7fe0e45a73009bb9f8044f4a27a2 -libblastrampoline.v5.12.0+0.aarch64-unknown-freebsd.tar.gz/sha512/51d52afb13e326ef4750bdcad800aaf3db2c9e068b4c38bd148e312c63358b2228b81d23626d18b8983534a8a6f24df1b64b4e7121779d2535574ea907bd18ba -libblastrampoline.v5.12.0+0.armv6l-linux-gnueabihf.tar.gz/md5/1b6fd062d133b13e8efc63f08528fb51 -libblastrampoline.v5.12.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/78d525f425ee27068b94b94f89ef44a51ffac9f642ffe66e177434804e59b4ac3ba875190aceee386a8d740f7903e979e5b91f0973138d0fc7753061c6f5f26d -libblastrampoline.v5.12.0+0.armv6l-linux-musleabihf.tar.gz/md5/506be2b7669aa171efcc541388cb5444 -libblastrampoline.v5.12.0+0.armv6l-linux-musleabihf.tar.gz/sha512/2975136376c3f61b8f227676c4e1368d1847d85ff469dddbc0a330635eac77c00072c7544ae4aa9981d16a4ab04d494be54fc951b434a56fbf14003c42626579 -libblastrampoline.v5.12.0+0.armv7l-linux-gnueabihf.tar.gz/md5/99403eae880f52aa97884143e2ca7215 -libblastrampoline.v5.12.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/986dfcf5fe3ac731df3c71eb6b0bf3d7525511952d22cc9128ff35e6fcb330acf69e897aeb97920ebabd1ccccd1dd6ce9b6c16d0dbf661d39a103ce5b477462f -libblastrampoline.v5.12.0+0.armv7l-linux-musleabihf.tar.gz/md5/20adf8d2ef348f5362cb03e1a2780476 -libblastrampoline.v5.12.0+0.armv7l-linux-musleabihf.tar.gz/sha512/95068a3b5bcf17bd5f13373a2730a6508d3992f0aa83a91629527821cf038b9607327843cc44fb72730b63c01d3d70e2eb488eca8f48ed9444d7736f67745d02 -libblastrampoline.v5.12.0+0.i686-linux-gnu.tar.gz/md5/a56f833ad986fc3e9e64e5abdb16915f -libblastrampoline.v5.12.0+0.i686-linux-gnu.tar.gz/sha512/d478b4981dc17afb8aa8625fdbb23139f1c3edaa9aaa179e70d274984a056147b2e65e9f473b007733d094369f448823c33aa95fadd228016ecf9dfbf17f06bb -libblastrampoline.v5.12.0+0.i686-linux-musl.tar.gz/md5/8578119b3b3e84393e6324996e9506aa -libblastrampoline.v5.12.0+0.i686-linux-musl.tar.gz/sha512/b546de6687755ce43680f312008a23a8f9df422603098807f33e2ae969c9e9de0ca32a3319067d4f8fa1f782f21b6465638cd59e4c86fc6261fb4180f0ed116f -libblastrampoline.v5.12.0+0.i686-w64-mingw32.tar.gz/md5/b9e2800b8758d3fa0ac0597f738c399c -libblastrampoline.v5.12.0+0.i686-w64-mingw32.tar.gz/sha512/e0aa0ee2a750cfe702e0bd5861e352f97f433f67444dbc6e5814055fb32f571de318f640ac670c91bad233f8af85f0421daef71b7768a710de5b15febee28b27 -libblastrampoline.v5.12.0+0.powerpc64le-linux-gnu.tar.gz/md5/bab2048857c7c1ba4a6c3d540b9275c6 -libblastrampoline.v5.12.0+0.powerpc64le-linux-gnu.tar.gz/sha512/576026c970b19cc00480d7bb9439933c5bb432eec17def66b22f5c0dfd418bcf75bb10ccfc1b01fef48e8d504ebf953c5f6c63d504713315c43d9579ab5fa2e4 -libblastrampoline.v5.12.0+0.riscv64-linux-gnu.tar.gz/md5/f37e2849a948a8c8c8bfa6055e30909c -libblastrampoline.v5.12.0+0.riscv64-linux-gnu.tar.gz/sha512/89f30d52f1a1dcc0aa38b4b343534b7fadcff12d788f455172c043ea2511c03b2735fdacf8f794a6f62156cb5d82fb0e9e0edd04bb9c57a1ca3e680410456b17 -libblastrampoline.v5.12.0+0.x86_64-apple-darwin.tar.gz/md5/b07c42b602b91bf2229b1a5cfd8e37b3 -libblastrampoline.v5.12.0+0.x86_64-apple-darwin.tar.gz/sha512/ab064dff373826776f9b64a4a77e3418461d53d5119798a5e702967e4ac4f68c58cd8c3c0cc01bda3edeb613cf50b9d3171d9141c91ff9ef3a2c88a8e8f00a37 -libblastrampoline.v5.12.0+0.x86_64-linux-gnu.tar.gz/md5/c37b01242012e51e124711d5ad10cf97 -libblastrampoline.v5.12.0+0.x86_64-linux-gnu.tar.gz/sha512/3f9015bec4aaddc677cb3f3aebd432db8bad89b3f6e563634a37569afeb9fb0efa4f214166c984c2c1926831d5cd79fcd4d605d40675e0d1a7e494a76c066f02 -libblastrampoline.v5.12.0+0.x86_64-linux-musl.tar.gz/md5/c24e440a1757a45f087a2e1ac649fb45 -libblastrampoline.v5.12.0+0.x86_64-linux-musl.tar.gz/sha512/824b930d50df929fd22ead6dffad06593d2aad9fcb149f07f1c2f6d4b7b34911e89c2be5a1e9b8ad5ad8292ac29f9e5dbe6d7bb205d2b207432ade61ae5f8b68 -libblastrampoline.v5.12.0+0.x86_64-unknown-freebsd.tar.gz/md5/5721328a24473cefbb3e77ba85e46922 -libblastrampoline.v5.12.0+0.x86_64-unknown-freebsd.tar.gz/sha512/3537ea491828492f1cb68fa961dc5574b63a88b49abf19eb86f9d1a4544e1398fcd84d6338c6dcb9550ee3abcdcab0654f5cc2b85699c5ed5b3b31a1c35a199d -libblastrampoline.v5.12.0+0.x86_64-w64-mingw32.tar.gz/md5/450afb701cc2899c7c083bd3f3e580a0 -libblastrampoline.v5.12.0+0.x86_64-w64-mingw32.tar.gz/sha512/e4d1785a06b051a4f16edd7343021eed61ac45cf45d26b4e3ef1e54cfaadb44da2e74b7d854e31b05a733dbb3004f3e85644967316c4f41d1ad64400fed126f2 +blastrampoline-f26278e83ddc9035ae7695da597f1a5b26a4c62b.tar.gz/md5/855b7723a6e9eb8885876eb675d48329 +blastrampoline-f26278e83ddc9035ae7695da597f1a5b26a4c62b.tar.gz/sha512/29cbd060c8f5eb17ef486d0a10ee4b221eeceec3a2ab0f9f98f60880f3d19a2247d93ac0dc0d32ec568ef876acd30f6c0642aaf704757580c2e17884e425607f +libblastrampoline.v5.13.1+0.aarch64-apple-darwin.tar.gz/md5/d8dc0f092f86b379b2fb9da97382be70 +libblastrampoline.v5.13.1+0.aarch64-apple-darwin.tar.gz/sha512/d9fc0439565afaabe53f56f64c20aeddb846c991dafeafdef6c2369bd7a359c1a6b49cdf8d63eaae2730a336509854b5c306e630eb520445712efc4e41c0263e +libblastrampoline.v5.13.1+0.aarch64-linux-gnu.tar.gz/md5/c181e51a6ca4cde0da3d036d561e24dc +libblastrampoline.v5.13.1+0.aarch64-linux-gnu.tar.gz/sha512/fe4a86bb4c94ef86c2307adad528bb58d0508a33c194c64190fffe7902f5b915592567d9e0cc35414633c5ab9067def2fa20cf669a2f4309265744180a5ec51a +libblastrampoline.v5.13.1+0.aarch64-linux-musl.tar.gz/md5/6f9eb8d73a0e61f3a2b97dba7105086e +libblastrampoline.v5.13.1+0.aarch64-linux-musl.tar.gz/sha512/9c3db080155729a91b5dd47df91d3852539aefc331d4dc51167fccaf3b01e601b36911ec259c53e211fe192c108e839a1f14b837009fa4f7d88ed82d658f80ff +libblastrampoline.v5.13.1+0.aarch64-unknown-freebsd.tar.gz/md5/68f65db9da9938929d510eea3540335b +libblastrampoline.v5.13.1+0.aarch64-unknown-freebsd.tar.gz/sha512/2fc7b375a751f3bb201504e0417828602fe014a2c8626137779c09ca7264ac6d39d44db0d1d32e0dc506284f56b49e23791922b0cc1237021473fb505fbf06bd +libblastrampoline.v5.13.1+0.armv6l-linux-gnueabihf.tar.gz/md5/a377fa4e5751fbeb3c42f319cb6341de +libblastrampoline.v5.13.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/9ddb1e2f4daab45d65b66dafc00df6ca7f788cb919cd6699c4aa0deca3e99a86d9ced10c3741610a6e480093d483e8a02c1d9165f91a7179632c1e2ae1abcfb7 +libblastrampoline.v5.13.1+0.armv6l-linux-musleabihf.tar.gz/md5/42c841baa05f80f17ea1b1d4f3405bef +libblastrampoline.v5.13.1+0.armv6l-linux-musleabihf.tar.gz/sha512/0c3ed42bd48f8f1ee9b1dc18faa7afa6e2fb27cffc59b9a420e29b5e6cdf8fb3bde36b82f3086075f8f7f329614aeb91ca5f173b1683e30e9530076f341ea2e0 +libblastrampoline.v5.13.1+0.armv7l-linux-gnueabihf.tar.gz/md5/61e515ec1223c99705175a26e6fbaf87 +libblastrampoline.v5.13.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/92260dcc563ece74719f21921a7cb51266884ed01b50c97fa997b4a98737e900ec9eaa8012d2c4c67ab479c4080bd1cf2708612eaaaddbba28e4f9147f3708ea +libblastrampoline.v5.13.1+0.armv7l-linux-musleabihf.tar.gz/md5/d45816d705dd46572d85105567bc060e +libblastrampoline.v5.13.1+0.armv7l-linux-musleabihf.tar.gz/sha512/45cba07050b818cd85c67acdfc29515e1fe416eb4e0c219171f2c0c026f7412903c3a9367d48258259a16e89f36c1e8f9fa054e455759720f1c6c5e8e27be476 +libblastrampoline.v5.13.1+0.i686-linux-gnu.tar.gz/md5/c8d3fd5f314353133934396361857c92 +libblastrampoline.v5.13.1+0.i686-linux-gnu.tar.gz/sha512/a949b3c0655ad9d6f8d53fd8a3f0b4ab504046e49a373039defc94e832b7faf90c77520f3912c4d6db8b0829951d85b4fc2a4021b3d8bb2c399d1ad04ce77ab0 +libblastrampoline.v5.13.1+0.i686-linux-musl.tar.gz/md5/a7bbd2233366d180ce8aa61fd3568c11 +libblastrampoline.v5.13.1+0.i686-linux-musl.tar.gz/sha512/e78cbef5b3bcfa93a86e14eebf0d704a94ac7b1f5c7030706d1f4a960de888c42e3daddb65395c7102e08dfd444efbfb40273e58a5f1de199d44ad55fd3ae658 +libblastrampoline.v5.13.1+0.i686-w64-mingw32.tar.gz/md5/4ca5cf3f855d04d3e8bdbd15321944ad +libblastrampoline.v5.13.1+0.i686-w64-mingw32.tar.gz/sha512/33160caa000c6c44cedd594195e1f2efefb950459653ee12ad2be4cedf0b833874772512f1812948d753f075ee7b8fe5629e5f9bd753a3da7804c4a6e1b0e0a8 +libblastrampoline.v5.13.1+0.powerpc64le-linux-gnu.tar.gz/md5/8be947c20f7d35ec22247f9a11ccce43 +libblastrampoline.v5.13.1+0.powerpc64le-linux-gnu.tar.gz/sha512/56f4246f96d2f49b03f5e5f3b996660a48d50b3784f89df7cd1dc52bab6efea0c120a65015040041a51d18fc6f361f89486f77317d771fbf588a1ba7565d77a2 +libblastrampoline.v5.13.1+0.riscv64-linux-gnu.tar.gz/md5/85e1f70a3235097158b4884a58a58154 +libblastrampoline.v5.13.1+0.riscv64-linux-gnu.tar.gz/sha512/11f1f5c2a409dbdab11d6bc968610b5700e9b0cb95094e348fe43ddca5586eda47bda1c382fb1f4b5a15aa741a6fc2b31f58f9b08bfe46631b5471e864bc009b +libblastrampoline.v5.13.1+0.x86_64-apple-darwin.tar.gz/md5/c6756ca8b6778ce2a4a440f63355c32e +libblastrampoline.v5.13.1+0.x86_64-apple-darwin.tar.gz/sha512/895d9bba75a9a0861809dca48b3dae7b5ffc5d866a518729ffd52f70fa1742a41a4b8b4e03bb354cba12d9ad11a33f3f112fa69a30ab3f945a9dede0d59d92b3 +libblastrampoline.v5.13.1+0.x86_64-linux-gnu.tar.gz/md5/1326a406aa98b6045f7459d7fb237894 +libblastrampoline.v5.13.1+0.x86_64-linux-gnu.tar.gz/sha512/4965baa1de5532425ea57b8100e369cf44b55963340cd144c0359f845560f27a1bea1597e4c72ec541917f71aaff8a4863f47d01a095c2e761a68212bfb08d1e +libblastrampoline.v5.13.1+0.x86_64-linux-musl.tar.gz/md5/5103983b7fecc7b87f495cd3b6c4d7a5 +libblastrampoline.v5.13.1+0.x86_64-linux-musl.tar.gz/sha512/f3243d84a0a0a191abad9e3850c37be78892eb5905b63b47bfb3e5a4148e0dae672ee72d311c5c764ad0fffe57d39c10dfd2086466efd76b5030118941d36a00 +libblastrampoline.v5.13.1+0.x86_64-unknown-freebsd.tar.gz/md5/a001ecd07b5178ce724a4f78996dc43e +libblastrampoline.v5.13.1+0.x86_64-unknown-freebsd.tar.gz/sha512/508866d54a9a49df2ef7eaa5d807173016c6dfaec59c4c89d5b37cd3faa7384302d2d4d39aca1975d79a948414657b7ec048a3ebdf6bf5c938037aa89303013a +libblastrampoline.v5.13.1+0.x86_64-w64-mingw32.tar.gz/md5/14fc4ec99e72e5bb646f5e6e8410fe01 +libblastrampoline.v5.13.1+0.x86_64-w64-mingw32.tar.gz/sha512/b7d07218047917fe217736b3c97d2b0565f6c904cd9cf6de96e38c66552aeec13b3cde714775fce1eb5a230db0ec0f2822572de8f0e166cb042552a16beb2b79 diff --git a/deps/checksums/cacert-2024-12-31.pem/md5 b/deps/checksums/cacert-2024-12-31.pem/md5 deleted file mode 100644 index b01bf68ddc247..0000000000000 --- a/deps/checksums/cacert-2024-12-31.pem/md5 +++ /dev/null @@ -1 +0,0 @@ -d9178b626f8b87f51b47987418d012bf diff --git a/deps/checksums/cacert-2024-12-31.pem/sha512 b/deps/checksums/cacert-2024-12-31.pem/sha512 deleted file mode 100644 index c12b8215a7855..0000000000000 --- a/deps/checksums/cacert-2024-12-31.pem/sha512 +++ /dev/null @@ -1 +0,0 @@ -bf578937d7826106bae1ebe74a70bfbc439387445a1f41ef57430de9d9aea6fcfa1884381bf0ef14632f6b89e9543642c9b774fcca93837efffdc557c4958dbd diff --git a/deps/checksums/cacert-2025-07-15.pem/md5 b/deps/checksums/cacert-2025-07-15.pem/md5 new file mode 100644 index 0000000000000..084a7e65d544f --- /dev/null +++ b/deps/checksums/cacert-2025-07-15.pem/md5 @@ -0,0 +1 @@ +ce594ef75f07eed66c538f7d4d83eafd diff --git a/deps/checksums/cacert-2025-07-15.pem/sha512 b/deps/checksums/cacert-2025-07-15.pem/sha512 new file mode 100644 index 0000000000000..7fb658508c700 --- /dev/null +++ b/deps/checksums/cacert-2025-07-15.pem/sha512 @@ -0,0 +1 @@ +743e96c112787f9b415310cb762ef9fec845534974e7e27bcec07472c346b1085fcbe89dad1f97699123800b37e29cb97f3fa1ffa6bb005af9743b4c74771fae diff --git a/deps/checksums/clang b/deps/checksums/clang new file mode 100644 index 0000000000000..2d1b19d45c84f --- /dev/null +++ b/deps/checksums/clang @@ -0,0 +1,120 @@ +Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/744ad2230a1594acee8781be442cbc65 +Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/118f67c2fbd6b11e3d850a38ea331f10403cbef563f22bb1102ff5960d1dcb3e264dcb36e71534828d422e669de1495acdc14c9d452991fb74b68810acc721b2 +Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/6b208a76bb52b83bdd3239c93fc37876 +Clang.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/8eb5d93c894a830321f58ab18e4aad58a52895bfe8976728249758f41b439e249beb35990daa2fc635a4fd5369aa89e710771d06d7242ee7c34002a18c939629 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/69d3fd7ba2103fc9074af330e30b0397 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/f29d2552626a7f3a91a50984bdd396beae9beca65d119e3e25f427334723d40dea4672e27e6b2057bc1ce354eacda57dd06b6718347d9d075f455dbffbd972f3 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/778dcf8e95ff94a946726eee2987f3f8 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/31ac240f96d0081a20284de2c18c919ef74cb1734426051567eabedc18752f18550a1fe48ccb66fd89f3605a50ded57b61631aa48ff528c20a07dfb73b2bb063 +Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/4239b43accecdfc6fcb96317a84f267a +Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/1df612a15d6e9f59235a0d97f8aec1b1252320077b8b3ddc5623bbfa531b7b089663f4c826e1b881401715a288cc170d79b9953ad987caabda234b3c1fbf9efb +Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/bc3e688489a6440a4eef0cb3731d979f +Clang.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/7c0b96a9b927e3679ec79a45db388a3bf79d773ddc3d85da9f42a92c197c5e9886d13cbf00e7189e24b1d3a2ad942ce7c0cc50d3666a14c6e051b1d01e639145 +Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/770f427ecd0f6900eaab9bdcdb7b1964 +Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/8263c16d644c94ee21d5b46c0b515fad5b73dab7bd0fe658eef239bdd83b9c10e26546706cd22a892ea9d8d43e054123b04c68f586407634a4eb2826ea93946c +Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/c02058697c68d62d94b68392b00da335 +Clang.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/ca4de5088f9f016ad0b782b9816a06ddd2dd2413e28c587cb8968308b89ef1295c7432de271c1839a0d436d670e06c64b7989f44f53ffbfd1ad82ca35ef910f7 +Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/edba25d64e92d9bcdb6380c92779ff68 +Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/2d080e0f159ebb3e91ade714343f597d0da9733c361ca998dca9cb436f5702604ca0c20b6ec547b230f77f963d232e38cd139f7ed4203c3fca96bfbe9818bbee +Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/414dd96ffffc43202293d0f2987ce478 +Clang.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/41aeff9a58036e38c5d714ef03a7850b52b93dd84e78327075dade971027a3d714d0f3ede85144a4ec7b10449ce5d5dfccf6c1123c04a7da74abb5b25a38ca79 +Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/7c8b3eb507228e61f7251d7e8b6dd578 +Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/0acbd2fc5cc95830ba1cb2efa24b0a7b0bfe66c9f2592a8fd166a2318727346fa4827097601dbb9d915473b90e8ef5cd88c8d0c73947e27f8ce1df9a74879d03 +Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/74f4afcdc240f3e3c56edc6058f04530 +Clang.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/b2b815a323994b3bf5a386d10d126f5bd5572ea4e883c5b28ad68cdadab4c8c5d4eb7ba9799eff97abe556de65f725c62426dfc44492240f444206f7ae666abb +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/ffa2f021083b5d27e9de06b618364079 +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/9295f6f080eeeba21570165435e937a7347867977b6a5e5abbc4be7150396808e938b27e7cef5b0ddd9d1e538a04e17af795fa3c3508f179ac8916157fc61f52 +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/4b33633c5f9dbdede7721f815c39b9e7 +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/a358b4898a34a2882029652769faa8a4ce0a2f5ed36c853ed40a5875797dcd14ece37dadd620a5f7a25afec1bbc426fcf5710eb9be54023c03cb0c50087ead9c +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/25076268c657945ed492d46c2dd819eb +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/e8bc9533ef8fdbee5eb938f9a764d69de52b47220d8ed97b9a23a7ddcb62206e54e479abeedc14339fb2082ed451f861d1b19eb93830dd0a06d609c8b85fdec4 +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/e0a95901306ab2c66a6cbdf86ebb0c5c +Clang.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/8713ac54a84e44b1d565db8334befc271a8d5ca8646f46f75b43754c4681f433b8ecc239b3103d917ae28ad258c33f6e656db895d449a0b97c3af3adb3893ac9 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/c52e8e1667745e3886223c214abe8191 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/abb67bf8041cf61a81f37f109a30fafc595def65b11d5628a7a7cfd095b0c6b2303913a9149df7f1a2996893233f6f8152a33a26943368c62763efced7ef8e83 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/e51f6b1a3bdc815c164440976b3eb380 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/2633d7c1bde3bae99d81c2865644f15e02e935587b7cbadd63b3bc90eb9539dd455dfe592283574aa56030b70681b0e4e1950f17643b352e4053d63e61cfc93e +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/b1c191536aa62bb9f514b0f5615bd322 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/0973319c7d8a3f3058cf8b710afc0efc7d28679cc3182982bfb1feee431fb52eab88c3387aed5e351294e2bf4155775bf2b20ae6c35e21c5f123937140bfbf40 +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/2ddbb22f341c0fc79e02ddd2bc60abae +Clang.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/e829147ede7726dfd018183b1c89cf976e2160cf33f5f39eda0f0c9389e31dbe52cc127cbc3f43c19253fcde84ba6d8bfa4eecf6d6a3d6704519b578e77778ab +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/739d03b8f5f11ef6ac54054fc767548b +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/af3fdce8bc0920e028adf5126dcc442f8f8658707f1f5b58ce4179dffe3351b3337e91908ab50a309789a568e0110930fe6a64ace2384e5631f960847ee696ff +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/1c1a6e5abd46f5b82d16af33f1f04055 +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/d2758bdc071eb5b5cd9bd6e0fb215aca8989429b6fbdb27a2874b1b3c7988624c311d9f71cdc76d8fda2fae113ada8d18212180ffdef80071ea25123686d16b7 +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/fc073b15200b2a4514289c1670761a32 +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/7be72926f46764b7b106e4a7623ec7cf54da57feb9b12b4310ca565324bd277b06197fadb3cdf4aeb8ae804b5e68c66f8d18d0b9e8c07a0b5f2acef6befd7001 +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/894be9034932a5ddd8868030470aa1fe +Clang.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/7460e69dac9c61942494d4f2a48d3cc52e4741395b4185fbad72b418a86a825f4b872e80b89324e465a2b83e8fcf3532aa081f9706c29c7040c61ac6fb1071cb +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/1ad205872d5fee1e4d5821c6569ed8b8 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/e3004ec1a7e1424e1dfc44f3b8f4e87edc1c9d490866bfb22fcbfc79d9494e5cbb73b9d7859de061ce2c8bd08715eb1b5d9b0ba23e0231f4a607ba7097370b88 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/2ff49e5d12a9f5db5758deb116fe62ee +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/29ee0cb52dfe643f99f92a50e0f60d6fc35aae8311e0419106862b9d14c08ee162fe7b0a2b04a291176f72cb789d006eae3aa41a50e9383d6acb03aee99c2c9e +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/89c6e3312234835ae172ec9f7697b888 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/44e20f9ef9dae0e961bd5b469e930cd4d701f228a48c867d0655a7eff220bc060a8e769a520af7dcfa1019bb058516c95ed9bd25564488264604609f76f60a68 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/ed6b213ee57408a819912740dd5d2248 +Clang.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/43d68c8712bff0fc260be31df1ccfc657d725971f5b115645d5efeafeaf10209401752c5d5fe0b0f3e4be003d68a2d75c08f97df7fb8d8aa0ed1a49c2b148a75 +Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/f813d4d256ec68b000be8e7dd24172bf +Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6a433e3459ae36b6a131aed5fde1bf35af7abf825d5d5014cd82be0117087fe138a3d3ee883c12a56cbfe5a0ccd797508dd38587abcf80bd1e06847a522c6acf +Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/1fe9b72f78ac84c375cc5ecbae608e14 +Clang.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/51663a78e81635ae0a65fb0c7c4ebf66080fc919271736ab0895d3661686d500c6dfcf4181865ff6e0fa645a359a24224f84e0d2aa6c9b9a81431e4905f81cf1 +Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a3bfe33b1a5ecc815b81cc5719c5df3a +Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/c7d341cc38fc895b6cd884bffdbeb1d2cdeda6cdd90fe3e85ee2fc909ea6e6c2267c1d165035f8f8c1ae7e5d96eeec61c65e803149f267cc1ea54599c49463ac +Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/b77d8afd01a62b31eb5f62a39ea7c834 +Clang.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/141248c50ecb5178722c2be6540e12e7e62f99903d06bf7f1e401f9ea27ccb42dd146aa66e08c790e9fe663c167a3ea48809023daf95cf7e32fb0f177147f4a6 +Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/08b9ef36900ec33066898bcd173a5d33 +Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/1e2b9f4011873cdabfcd3cd154bf47b93da682f4ffc1bd41cadcee47e693f921ea62d7792f23b07793b2731434cf6cee91bd11432bab6f6b0ac8969aef507e84 +Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/c48a8ee67bcbb13a5263f79e757d2fbd +Clang.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/5f24c4d8b76fbf9036bcc3344436a5906eb025a9949118780e4586a5da6afe3d5a326011fdc01bdca7595940d2eb94d668dabf2df65af33e31637d9ce9ca374a +Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/ec43a0ba8d6957e071936d85eb5e360e +Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/9a145a4ec60f3a3826001cbe4b089e7053e10a9443182145b44fb9b0a0c9051c0d277632e17eb989cfac9746441cbe3fd3d8e68726e48727d63aefdcdc1d012b +Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/08507753c774304274f1531c36f12426 +Clang.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/73fde12aec5fa8a6cd1037767a24dbed2c496071b2dac6df98624d01316c3cde85f7fa176e9572b5b74c2f2a7010fb2545bc4b949d27291c926e749284ddead6 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/a118c6ef27c7b69f2877b10f532bf2e5 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/620727a97254c81d16ef4e1bf3ca4013aeff75c069f12de4b5d013e1df28e9d1804f2dcac95c116c19a537db39ce190064a2e39db9d2f10f5b83e93145ebf7f3 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/13a9e37303fe1e793d2782690159403a +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/4a24a1cf630dc16c9dce2b9eedb7ef6327b82eed288cb76e42d227db9651694acb52b9c59b96736af7a19a5d4be122935e5694a0384184bf28d2b4419de8a9c6 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/402a534e78daddcdf42966ba51696663 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/c3835b05b621b9a4f0b7a2aae44a7d53e64164010a054785ff4264a2a369d1f5b9df84c1a4e2522df55f240790cf1a939becadd13893c70c3a9139d2703fcf5b +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/9b7a6b511b728972c0ccc3e1b33855c4 +Clang.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/cd9fa85402fbe04ee55f6b69ffb87d3a4923374c40176199d4ffe10f8e8aba54e24645ac30ad0e74ed60e9ab5119bd75f089d8dec4a132ba29a6960fbd0338be +Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/fadabaefeabb7c1df445473ed5ece280 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/aa6556a9d27703fa8b49d682c60a1ca843ae887b400c3ecdd3a78e8ef456b1feac5b1b46cedc6124f04760ca485d028defb404da808a96bdeb0b65a704928d06 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/c7dfffb39c3d193785b4b31ce58dab0a +Clang.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/3c64e13b5846f21ab9796326f2858eadd77bb8825a840ed14383012e651f93c6d74fbed3cee9c324cd3d9256feb243dce15e10bd7962ffd668fe83c9772c6beb +Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a4596e099c5edeee4a204305b74be573 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/bae493823a3eacf6a6f7c0c53a78903425d8ea71e846e5d28b583ee067cf9c8dc7c0103f53f30b08d271f4867a2f3513a187c04b4e9136eb7c4fbe1334be5240 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/f3d54eaf5770a465a510c4370ee96548 +Clang.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/cbdb8e0c1592f9c4bd8b1e6a15ed15decc92502009d11c75be7ca8840bea066f341e6bfdd4b60dc51e2098103425b324096870358bbca36f8e3af9eb4b1b9559 +Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/d697a3ea0444d2a583ee37b609ddd545 +Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/49651c7e38e41aac7645c7f8e8fd5f789b742ee8e945b59181675b40bcbfcba4aea8d8d54e7dd703ee40fc91cd76d02f1f7999ca040beaf49c2d605a1f76b818 +Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/43a0fa128b939182277f285b9d6f4007 +Clang.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/c7ebdadf877fa46f55e2366b5ccc18499a15fb01888e6373f1596ba10530c2c7d60f248def10b6c196fcb5d50c9bf9e27bad4b7a0c3e5291f411ef36a604328a +Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/1a43bec0e1fc4cb6b94885d4201e75f2 +Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/76295abc2d2e1c4c2f56719f6926dee6002f09472266f77be2d097d2f2f1da706174215d9de59cf2f88807e6df3dd38050935e82f8e66b0c41f5cd8a1dc2ba2b +Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/c3ac86b91ee81032410e732722be1301 +Clang.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/dd370f92c522ef8fcaf85c2a87cddfdff74fee92bcff8c3b1d5ae8e12bb3f90186ead91d22808f96d1262ca84806938506333fb3dcb5bbab51ca845de6d256e9 +Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/60d67e563b4f8d5660692da70a399b92 +Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/55c135642fb4b9e61eac6fca3cee9b2d0f9f83456f9670bb90428e1fc0c28642f285a2dfce9cfa32a585f9174645ab6b2ff8943367b9653792233f4c32a78b0a +Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/2448604ea7a346df8532511e36d9a59d +Clang.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/6627693edb541e700596212ea580d18a064d8e765642353d4b5e474f394bd6d9120b781f5c02840bd327b084ce0228b05ff17a58bafb6161b933a789759eb0ac +Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/de3ab95295e91a8dc55beece1c094489 +Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/4b2f1d6026fd8680a5929de517a42bf84f66fc6cbd32f8becc8b0601d3b67de66c78ab67a94662184ba5becc28bc4fcc2644f17731b838b8de46f909e4773252 +Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/7d3eec07c5d71a2a90fff278116f5b09 +Clang.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/8d837e4f24fc80f3860716aceb23961ef33d499cbb6ffb5bc89b422b795ef46c60a787038e79a1728471ee2f0807906b28e9aed7fdf3772fa13d99afda0d2332 +Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/82298efca03959f3c01c91d0da4b8b07 +Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/90bdf33dfb8eff0a3ad9cec730067900283a5e31f05a008a873c8673a603e8a3e704048729c15b841f0adc29c96daaf1085cdcb3029cffd7d42f8b99116dcc86 +Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/c9671bf1a659a52fa7866716472dacf9 +Clang.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/7d6296bc6c44311b763d70a3a2617c7d1e7835a0df1c232b5de453611d71eb42dd43f659fa2bfb320a100d87594e71647b17c7f108772d444581daae567b152d +Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/e9457d3c788a66624a2d34c20d1c300c +Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/4aa67eec6c4f8db62fd88e568f7ac5a5b24ba5ae92ce6a779dac570a4b95a373bce2ca70b46ae1f919459e44697f9aecc861e9375dfa39511bfdb43b8bca0539 +Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/21dc84915705793f2c0045db6430391b +Clang.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/d2a1d55d1cc04f4ea0c890dd52ac2c0961d64c3f9295a2ddc8b681f8a13cfd32b0b05d34aabbba5708fa9b0e68f5802e33267456dbe42154771d5c610d5d1f02 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/37091fed96267a3865f837d637ce9140 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/046ac9f2414eb9e163e7520d7d57e2a6caf5279ca6bb7297eaddf5b36b2218ad83ea222c276455111bf1b32e16a56e40ea419698ba2f18f93ec271ffda9ae433 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/4fb68bfc1f784051b212f13f1c766aed +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/c32936a923b4af1106293e24ea07311bfa728b97609540f55bb65acca8d2fc937fb718796cc785fad2260c22631e09e96782ceff8a7e3b2b2659f6eb55e3c74f +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/edfca70811c32f08b758cab451b6d254 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/8eec4b27eea12a40c77c47f19fc0ce50f88aaf83822dc76c274ee9b051da8b41d37bf4a7fd3d0a6124f1a6fd58c52ec86cd8ef72f7844a688554744add4daf1f +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/97ecfeef24bf469e594360dd1cbcc177 +Clang.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/e6ee100c2a944dd7d6cca577f8cd01fd1e0236720310667191bfe09990a516b09d663889ab29d8819f8341906d397d3871c28f351c88c144de9d238215c985df diff --git a/deps/checksums/curl b/deps/checksums/curl index d85f7796bb0a9..c7f63bfb9dc2d 100644 --- a/deps/checksums/curl +++ b/deps/checksums/curl @@ -1,38 +1,38 @@ -LibCURL.v8.12.1+1.aarch64-apple-darwin.tar.gz/md5/a6a5a1a360d210e3e03f1815efc97260 -LibCURL.v8.12.1+1.aarch64-apple-darwin.tar.gz/sha512/8493cc358fff3123fa6a8e6bc42a72c5219a259d568a2e1ff3c216c7bbd4c10a4057bfe2d2f7c571532d51ea876450c97c0f03f901ede4c7d668916265631f80 -LibCURL.v8.12.1+1.aarch64-linux-gnu.tar.gz/md5/7ebe2338198069b12716df084cd07d9b -LibCURL.v8.12.1+1.aarch64-linux-gnu.tar.gz/sha512/4b4ca81dcf0098ad8ce4f9190c559dba9d92ef330e0d18c50356e13280a6c71dfdaaed0de762b216380af22338d762aba4cd7039e95f3ee84b2bb02d55dbb49d -LibCURL.v8.12.1+1.aarch64-linux-musl.tar.gz/md5/9ff411aa7296780c3bbd2369597c898b -LibCURL.v8.12.1+1.aarch64-linux-musl.tar.gz/sha512/d3a4ec96254f5a8816d92492ea1dff9f6d3e3dbb5c6db06472dac41b38a7671c84959e934676857262751e89ae9d0e17ba567f8e4fab3a190c9a0f4393c190aa -LibCURL.v8.12.1+1.aarch64-unknown-freebsd.tar.gz/md5/7baca39040ac317f18b48d851beadf3b -LibCURL.v8.12.1+1.aarch64-unknown-freebsd.tar.gz/sha512/23158572ecb7378e92fd60977f73c9bca31a95683be3c820f0e2f60b2289125e726e8066f7b32e5159ddb4c76400774d41c299e6c8db5468cc6e3ba98ca3c1eb -LibCURL.v8.12.1+1.armv6l-linux-gnueabihf.tar.gz/md5/3a3f0ce770e66ec4afa313ccd747bd8f -LibCURL.v8.12.1+1.armv6l-linux-gnueabihf.tar.gz/sha512/68d62659448e79add970982065cbcf0f18c82c52ed837d83b954ac241268a82624532f0f291a8bafe0aafb4a700f180ad2c57c02dc2edc2c00da626befd0da92 -LibCURL.v8.12.1+1.armv6l-linux-musleabihf.tar.gz/md5/d7d77c98169e0c8067215c4f063df040 -LibCURL.v8.12.1+1.armv6l-linux-musleabihf.tar.gz/sha512/ff53a1cc55f6d1eac8688ca9c7dddf19949157fdde8a36221747b2e97b0bce2595862213dc35cdae211ead6b27372599c1eef1ebdfd4161f4b8b7be23f6534b3 -LibCURL.v8.12.1+1.armv7l-linux-gnueabihf.tar.gz/md5/2226abe022064c9f7464297a0a19f334 -LibCURL.v8.12.1+1.armv7l-linux-gnueabihf.tar.gz/sha512/f49feb8e27b732fed0f7ae5c8e00d6787f93ce450e063cec62300f6f42e6ecd6b59a2fa148e1d9fd9ff142689fc1f8fca8c9df21baea95301859b8177cffd7d6 -LibCURL.v8.12.1+1.armv7l-linux-musleabihf.tar.gz/md5/9d82cf23338ce84dcc03201f73ecc300 -LibCURL.v8.12.1+1.armv7l-linux-musleabihf.tar.gz/sha512/302fba7ac037d7ba2d849fc6d5da0877f5876871453837afcccc6e59c944f7dbafe0c271ec55062437578bb572bc0fe34a2e7a2599250c85d42f82dfb6d81d7b -LibCURL.v8.12.1+1.i686-linux-gnu.tar.gz/md5/a6bd8eaca6d45b0ceacfeefa3dbb6dbc -LibCURL.v8.12.1+1.i686-linux-gnu.tar.gz/sha512/dbbe92fd917e6388fa0e9454c19a71de842c349fc80fb530f2ead6c1e9b6af46d56723a0d8d4f451dd88e8bebc0d06be06eca90afc4bff5c3a23c84fbf36acb1 -LibCURL.v8.12.1+1.i686-linux-musl.tar.gz/md5/895552ab1aa470d87534a239b4f62aa1 -LibCURL.v8.12.1+1.i686-linux-musl.tar.gz/sha512/af05a032cec927adba166093af876fde691f15ff89c3e1978e4aa07f3589aeca495620fae201e8067bf17d776aaa5985ae4893e2875b52d6fb7e412e12b11a59 -LibCURL.v8.12.1+1.i686-w64-mingw32.tar.gz/md5/c10aa57b3fe2c8d6591813cb609908e1 -LibCURL.v8.12.1+1.i686-w64-mingw32.tar.gz/sha512/dcb943a2a405d6989aa90d8de3d3a3c259c12932be7707915d49e2016ea6c66b58e2b4d505bb171ffd6f59b1524a2d34c676c3fe61117082179c309bf64be61d -LibCURL.v8.12.1+1.powerpc64le-linux-gnu.tar.gz/md5/8233682e6d1a00449cb15f41434dfe04 -LibCURL.v8.12.1+1.powerpc64le-linux-gnu.tar.gz/sha512/3bb32f53ae2188b9373ee364394a73c497138ac986bd642e393ff7140adc533adc518e489a1b5255dafed7d16ec3e7c06bd4825e27ee473ffdbe2a0f28562586 -LibCURL.v8.12.1+1.riscv64-linux-gnu.tar.gz/md5/4d49a478301a8ede00dcced9d94775b4 -LibCURL.v8.12.1+1.riscv64-linux-gnu.tar.gz/sha512/69dade7cf6b628cd33571ca07ce22c7394f25b472a2d4d2df20a4f0c13e84c9e1bf53600209444b900cb401ee8ff2649023350c7bf9f14bc78bf4aec8ddad9cb -LibCURL.v8.12.1+1.x86_64-apple-darwin.tar.gz/md5/e6de55f6cdef86653ac96c2d884aea28 -LibCURL.v8.12.1+1.x86_64-apple-darwin.tar.gz/sha512/0193b54b153d9673c182fd534cef7abbf03987073aacec55d1852dc29d696ab3005b7619b9c1abb6f8879f5856b66df7ca7841908c7372684f8cc9b32c311473 -LibCURL.v8.12.1+1.x86_64-linux-gnu.tar.gz/md5/c6f439d4e0b92ee342d2badf009d9e8e -LibCURL.v8.12.1+1.x86_64-linux-gnu.tar.gz/sha512/cc9415d69732f9174d64f7903a3fc43a270ad007016adf80e10314182fd8147dcf4eb7cd0df907a2b9ff9cc82016ed905d700c66fd6a8b8089b8e27106d9ccb1 -LibCURL.v8.12.1+1.x86_64-linux-musl.tar.gz/md5/cb3ea4385e3799004266ea77d02632d3 -LibCURL.v8.12.1+1.x86_64-linux-musl.tar.gz/sha512/a52c7f6f5a4c9d9423e1e0e9bf563791f369bd979e5dce22b0de32d4f339dd42ab2f3f21cd0c933d4c7841fb7b244a56849936136841c2f5d7995b3a92a811de -LibCURL.v8.12.1+1.x86_64-unknown-freebsd.tar.gz/md5/0532f49c6b32859cf39ab3bcb5646d02 -LibCURL.v8.12.1+1.x86_64-unknown-freebsd.tar.gz/sha512/dea1a8de19cf74b8f95811397b8d615f64610b00678ad93482db765df0547c080de03211cb9e50eeaf1bab425119fe170ee70bde1ca58c185e2cde34fbd1d945 -LibCURL.v8.12.1+1.x86_64-w64-mingw32.tar.gz/md5/311dc58c0e5969f2d547a129e86f97ed -LibCURL.v8.12.1+1.x86_64-w64-mingw32.tar.gz/sha512/caa39cb4da20de98df2c28a2038763e332729894c1cbef3cb33df12351294559f5cb0c7f405ffec2bfb9bc27bcb4c3400916745f4ac09c0c94be6b9775f47021 -curl-8.12.1.tar.bz2/md5/c53f2409d83c7afa138cae7efed5ac4a -curl-8.12.1.tar.bz2/sha512/6a30f6eda94c03d58a287b4bfa0af39faddb13a3c3d1f86bb874c3e7147d76ce1dc7323741e28ccd8dc48b59a19d444c3b92512f80b3b8a4cf209d5b89109a4e +LibCURL.v8.15.0+1.aarch64-apple-darwin.tar.gz/md5/6e0789ffb84bb7baae49814c36db354f +LibCURL.v8.15.0+1.aarch64-apple-darwin.tar.gz/sha512/9d64ef5d61db74f3cc67cdcaf9b41d3e7e6a0a3b51deea97dbb3da1928adc9d8aa9558de786af1497de42df43dd184af057465f57471fd01cc1a83020dddbd6f +LibCURL.v8.15.0+1.aarch64-linux-gnu.tar.gz/md5/54214152cfef71c9296a150594c8c405 +LibCURL.v8.15.0+1.aarch64-linux-gnu.tar.gz/sha512/9a1c96e943313af3151763104bb2bed465722d81a4a5c97f8eefbcb4c566718e2a426bb9e81fafb17129a9cc5de93fd8cba5f83f7314dfa4aca6ad817011b627 +LibCURL.v8.15.0+1.aarch64-linux-musl.tar.gz/md5/be2288fc32572680434756ac1e9774e3 +LibCURL.v8.15.0+1.aarch64-linux-musl.tar.gz/sha512/d15a5a7e0f08fe9a0ad1b95f86c53167e3f26356be8010fa332a1a7003d879b5506a8029b7ab36e221821033012cd3a5581c30023fc950c711b517b49d891a3f +LibCURL.v8.15.0+1.aarch64-unknown-freebsd.tar.gz/md5/a18e1e712923540b1e8ccf8c67e0e735 +LibCURL.v8.15.0+1.aarch64-unknown-freebsd.tar.gz/sha512/cb0b2f4041abdcd3b8f15f8fa1a894f8c4aacaaea9f5b4d5213cee53bee2dfe8884299df2cc1f51c31f058b19c57c6b9e3c3cfd63f84ce2db27af14ebc4c0c5f +LibCURL.v8.15.0+1.armv6l-linux-gnueabihf.tar.gz/md5/389f85ab11b49689a941de10853072a3 +LibCURL.v8.15.0+1.armv6l-linux-gnueabihf.tar.gz/sha512/2e1f70f804b540159baf720953716badea9854e7e92783b2f3a8805057c430b2701cd662c24f0cf4a8d11e79cd80a352fd59f23cbf705dd09ffd9ab7dcabef8e +LibCURL.v8.15.0+1.armv6l-linux-musleabihf.tar.gz/md5/4b071a0e0af2a64ad1cde498f54a7177 +LibCURL.v8.15.0+1.armv6l-linux-musleabihf.tar.gz/sha512/90eaf004296fd1ebb3dd02120e92ef574a8c37456c5c99477764e91cdcf0326d92de852d0c8261693403958a3d0e40c4b6b3cffda79f517ecb4f1f852a812c02 +LibCURL.v8.15.0+1.armv7l-linux-gnueabihf.tar.gz/md5/b3da6e896d2fc03ff384abda6f27baf1 +LibCURL.v8.15.0+1.armv7l-linux-gnueabihf.tar.gz/sha512/21c7bb94b94bc1840c5bcda0cd3fd01e4850e3d9e43804c7f2ba49f2f97793603191476c2f764bd5e23f4288c1b04d1645a1c55655cc09971ec15925aef7fd77 +LibCURL.v8.15.0+1.armv7l-linux-musleabihf.tar.gz/md5/37d6ff69f83338c222d65f5132025112 +LibCURL.v8.15.0+1.armv7l-linux-musleabihf.tar.gz/sha512/f483d2fce926272fa7cabbcec8a269f681150b4bdb1cd25e638d9cff358e770a5014ada6394a2b42a1fabe6f6830d131924442003eeb3841dafbc8f2b9288112 +LibCURL.v8.15.0+1.i686-linux-gnu.tar.gz/md5/24bf5ca0680cb664b2a9d11e5356c4b7 +LibCURL.v8.15.0+1.i686-linux-gnu.tar.gz/sha512/21b000f346458079d7091f3766a2ee06dda90b1c4e48d236e768181013088b9ee8daf51eaf966c6c88cf3d9380b204ea7f9f827336de841f4b5efb38644ea5c6 +LibCURL.v8.15.0+1.i686-linux-musl.tar.gz/md5/c158455a2fb3e75d8ddc08e253bafb82 +LibCURL.v8.15.0+1.i686-linux-musl.tar.gz/sha512/8b59acb7bc33cc04fa39107d85c5ef3296a901d138bf4951001b7cd7d5780d84e6728b68862173501f9f4720c30736611f24c6334a01d271649ab1a03ca7a05c +LibCURL.v8.15.0+1.i686-w64-mingw32.tar.gz/md5/e681eea3a492df7cfb0c0e8d1152a56c +LibCURL.v8.15.0+1.i686-w64-mingw32.tar.gz/sha512/41b9c183099b00eadefd7f121d17611972f201cf21ecdebec538c5ee9620646bd8fb7610888df4b9d1a1ab3055d9edb6d345e8f787d9df4c1b739cf72312d5dc +LibCURL.v8.15.0+1.powerpc64le-linux-gnu.tar.gz/md5/9c71b40f94b7ffd65c182dd96da76d04 +LibCURL.v8.15.0+1.powerpc64le-linux-gnu.tar.gz/sha512/763cf590b7fc8cc509b4762f8c3fcd1f0534f756407472bc86063b70a8a144fd651990d95f9300bc7eb30682875abafab399a19449af6a8a4d9d9fb548b2a5d8 +LibCURL.v8.15.0+1.riscv64-linux-gnu.tar.gz/md5/d84d6ecb606d7de45a7e0e91d29f8e06 +LibCURL.v8.15.0+1.riscv64-linux-gnu.tar.gz/sha512/5c786779c838797182824849f682d54cf57f248ea7b7f80c21d34540324df16996796647f82e3b3d1cb274278be6c935c080658edb780fd76dbfe1d04928c74b +LibCURL.v8.15.0+1.x86_64-apple-darwin.tar.gz/md5/2738d7814b7e75a292301c5bce181a0e +LibCURL.v8.15.0+1.x86_64-apple-darwin.tar.gz/sha512/7ffe3b616f6a6591608dcd3a17cf67420b830b17e4d5cd9bd0441d104910d1a5ce28484238fa7c32dee0306c8f2ac9ffaf673a28ceca646e6879e5a552e3065d +LibCURL.v8.15.0+1.x86_64-linux-gnu.tar.gz/md5/a945b3ed85f8fb6ec27c551fbfe2946b +LibCURL.v8.15.0+1.x86_64-linux-gnu.tar.gz/sha512/1646773b43433ea9f403c0242fe0efb0595f16507554cfe326d775ef3f0bcd400a04a7a7db5b184649f84531d47938ec03d79d335c53929647a50cef9011fbcc +LibCURL.v8.15.0+1.x86_64-linux-musl.tar.gz/md5/6aa66d59e057fdb7da74a359648fa41d +LibCURL.v8.15.0+1.x86_64-linux-musl.tar.gz/sha512/1a422ca310ffea3eb70d27472440631031a2f7c8f089e261cecbb3bcb1bc7161fa41fcad91d3eb01cd688af30c8b43ee75b565f09bdff8c2fb0ab64fab571eb9 +LibCURL.v8.15.0+1.x86_64-unknown-freebsd.tar.gz/md5/d95c02c6c541689250e0015eec5a8577 +LibCURL.v8.15.0+1.x86_64-unknown-freebsd.tar.gz/sha512/0b6e0790779370f721ba251f6cd5f3464b62361fdc23bc1ca9ae22c82c4efbf2c7fc8409277b90c1b2e73996eb4378c188924b083d4b9ba255eaaeeff339e84d +LibCURL.v8.15.0+1.x86_64-w64-mingw32.tar.gz/md5/690e63d716b19e109b5487dfb4ef580d +LibCURL.v8.15.0+1.x86_64-w64-mingw32.tar.gz/sha512/2e763b8acbfc622dac431fa742c9cfdb12cb95436b0eba2e409a7a1dede107e99952f394c92d8f3aa55f3febf039f5c16058ee122d88e0eac5a75ae98f2e9018 +curl-8.15.0.tar.bz2/md5/8b475c3eec74c5e78a9fb45a902dae76 +curl-8.15.0.tar.bz2/sha512/797fc9af599de88ceb36c8bc284d3f1a2a1ca0703c7bdd377d67ce6da4317ca673ba4a946c61f3bd5a66febee37b5aa88826f26fbbb398f5e20630769a0de033 diff --git a/deps/checksums/libgit2 b/deps/checksums/libgit2 index c1906c995cc73..2339d381a7079 100644 --- a/deps/checksums/libgit2 +++ b/deps/checksums/libgit2 @@ -1,38 +1,38 @@ -LibGit2.v1.9.0+0.aarch64-apple-darwin.tar.gz/md5/1e22c2cf3e6003addd9bf16026ac4a06 -LibGit2.v1.9.0+0.aarch64-apple-darwin.tar.gz/sha512/78d5e5d246534164e1d70cf69dea273bbb8386df24c13fc3c3571762df15f2714307e7ff4cae6f977eee9def121c94cfe33cfcd44a60905a8161d65d17565e90 -LibGit2.v1.9.0+0.aarch64-linux-gnu.tar.gz/md5/70bfe9da256442ea2c295a016a89d3b9 -LibGit2.v1.9.0+0.aarch64-linux-gnu.tar.gz/sha512/14916a5521aa1281b443e61beee2573bc55b76d88810a3bec8bdea677d95763da82f1a527975cdabcdaa213e69aa1640201a03656bdb505b886906795aad0c74 -LibGit2.v1.9.0+0.aarch64-linux-musl.tar.gz/md5/62f6e885de29a345cc5ee3e773c74471 -LibGit2.v1.9.0+0.aarch64-linux-musl.tar.gz/sha512/09e793209505ea954e608c609138b8865d8a1630340fa8ff032a55234bfb8277d2c3c31f26048ae4993bf8c3d8f165abd0b4ccd80526c61efca0807f634572df -LibGit2.v1.9.0+0.aarch64-unknown-freebsd.tar.gz/md5/6fcba6e43265aa7a1ea5bba85977d622 -LibGit2.v1.9.0+0.aarch64-unknown-freebsd.tar.gz/sha512/34b836d3c22436e74963141dbe1f9372cb7ee695ebb2054ee0af1353d4401e1dfb855e91341a1d06a24ce18d57caaa3aa1e2bc7063000fa4f9be40130eb6ff95 -LibGit2.v1.9.0+0.armv6l-linux-gnueabihf.tar.gz/md5/75ede2c2c7312adf06a2a9859cd6310f -LibGit2.v1.9.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/9de567bee3aad33eebac51ad5b57b4fefaa4b778ce8510b2524a55cd223bfaf3051fd48c8713741e799d1464b308469580716dcb847a6eb97fd632727ca22a7d -LibGit2.v1.9.0+0.armv6l-linux-musleabihf.tar.gz/md5/e5341f0c76c89273c465cb43cbf0f284 -LibGit2.v1.9.0+0.armv6l-linux-musleabihf.tar.gz/sha512/1029d47c82ce20223b1c108da77a1a32ef0b91b9645040c1d941e7abdd161011736a81f4ad25006b32d83d4c07c548fcf1c8a3326cf3cb91d56fd443e2e9ced7 -LibGit2.v1.9.0+0.armv7l-linux-gnueabihf.tar.gz/md5/03191a1c4ff1c1ae764092b26c941783 -LibGit2.v1.9.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/6bb113c722b550fb28fc84033a3a38565ed5305a7fa193eeb4949b979fcf4599b84c748f50dad2ad47481827138a6e405eaf727f719d219984a809088bbb2948 -LibGit2.v1.9.0+0.armv7l-linux-musleabihf.tar.gz/md5/1678d6e57aa887963b27917c884cbf36 -LibGit2.v1.9.0+0.armv7l-linux-musleabihf.tar.gz/sha512/52590e9ca4118e0dec70191353b2c76155363df77df6c0bb5741dfb3f333539a8ad75339796748a744c342b51c15869726cfe9bbf6ca78d524e7d2ccce4a4622 -LibGit2.v1.9.0+0.i686-linux-gnu.tar.gz/md5/3fc50746cb80e0455f8e7c7622cd433a -LibGit2.v1.9.0+0.i686-linux-gnu.tar.gz/sha512/20c97e1a816456267a16759378a5e968e6bca122d1e0dc7cc282cad2bf2a8e3929e90373752065d91dfb6688e39ac6db660d9bdbb3277f1b9cb04b5d3f46fd8c -LibGit2.v1.9.0+0.i686-linux-musl.tar.gz/md5/fadb5e051e3b21e68a61b2a3049f65c7 -LibGit2.v1.9.0+0.i686-linux-musl.tar.gz/sha512/369c8c64df89149e9ed600028c1ac96db24e7b2c1977146667b8aeba93aa7a3b4787a49734411448680654188ece33e740fa475108b80b876a5082edad722925 -LibGit2.v1.9.0+0.i686-w64-mingw32.tar.gz/md5/610da247e41070b73e71df7e41267846 -LibGit2.v1.9.0+0.i686-w64-mingw32.tar.gz/sha512/d5b61c885133e3002e48e0fc37ceed0bfeef070e8fc6b2d78ec5f3069ad80966ea5b3a2b3aeae1ca478e9a2f839309fd67c3a186ecf751f4642ff4cb4ca3cb38 -LibGit2.v1.9.0+0.powerpc64le-linux-gnu.tar.gz/md5/f05f5f07de55fd297c564b6cd4e54747 -LibGit2.v1.9.0+0.powerpc64le-linux-gnu.tar.gz/sha512/57b740ca3ef6b18994386d74f1cf39c97c1f58f5a63e749c1a0dcef8c43a915f13cc093a8e1d06cef1d1c60cf484ba0e38d20a96344df69dfc997daa63ee1137 -LibGit2.v1.9.0+0.riscv64-linux-gnu.tar.gz/md5/b043226b10e5cbbe4914be3392f5bf72 -LibGit2.v1.9.0+0.riscv64-linux-gnu.tar.gz/sha512/a580795dd9a7ee237cd1d51d55f5079588686b1adfe391a017de743946e1bd4e7d5e4f8b79a6f84f0ce165733ca1b67ea740d06fa18547c29616df2f73e3f289 -LibGit2.v1.9.0+0.x86_64-apple-darwin.tar.gz/md5/bad8607d4997ef82cd43edfc7579d0fb -LibGit2.v1.9.0+0.x86_64-apple-darwin.tar.gz/sha512/c7359d79949a6727973b1df2264b672bfcd1617b6d4c74d281ef70ac93bcadfe47f99f7a5d031eed36b65077668ba12f2b31bbe6d491542b6938816659070317 -LibGit2.v1.9.0+0.x86_64-linux-gnu.tar.gz/md5/21e5fd214a6358f643477973c22ec70c -LibGit2.v1.9.0+0.x86_64-linux-gnu.tar.gz/sha512/9e68cb6d25d85ad272fcb0d77deedce2daa9c62d7ce2fd7e9221647d021aa00e372f490ad29211d7ca2b5ddefb4addcc4733e25e3df038aaf26fe3cb269d8f56 -LibGit2.v1.9.0+0.x86_64-linux-musl.tar.gz/md5/e9ad320825b22ee378b33856ca266b12 -LibGit2.v1.9.0+0.x86_64-linux-musl.tar.gz/sha512/bd33b4d31a7622a0440bd0979ecc7bbdef7ba7a52bfc911f880c9430d57d2b9ea1c6c4e57697b5a2b63c2e00e07673b3dad6feac056a4f345ed6e3b0ef7aef77 -LibGit2.v1.9.0+0.x86_64-unknown-freebsd.tar.gz/md5/501c63c8810616e6764ff80c23fff0b5 -LibGit2.v1.9.0+0.x86_64-unknown-freebsd.tar.gz/sha512/109e5676899ba6992a68fcff6d7503f49cc3b748b4b0faffcf951f318f9730e242914b57a7848111e229642070fdbce29bc181cbc79ac2e794c6ef489bb27293 -LibGit2.v1.9.0+0.x86_64-w64-mingw32.tar.gz/md5/4e76fa8356407a7065b50298817ad462 -LibGit2.v1.9.0+0.x86_64-w64-mingw32.tar.gz/sha512/01204b29ff2f90a9204d2e91fb7d48a3b6bea008a77984e3e67423a04f630690073d648a7200168809999aa5885fa6035c5b099256724b0379229c257ef19b9f -libgit2-338e6fb681369ff0537719095e22ce9dc602dbf0.tar.gz/md5/0ce4a212921ef1752ea057a3be45e384 -libgit2-338e6fb681369ff0537719095e22ce9dc602dbf0.tar.gz/sha512/4eb018a85a59c6ac0514f09f19a40813d8a4bc5ea230bf54897aa2ef5f584796e8c680a27ac68985a3457e1ea1f554ba4af803b430d67a9065cf51ff317d7a89 +LibGit2.v1.9.1+0.aarch64-apple-darwin.tar.gz/md5/1281d4cfc44ab26054b83355a053c42b +LibGit2.v1.9.1+0.aarch64-apple-darwin.tar.gz/sha512/d4040d56c588333c222d66488944c2988a3ed1bb096ffe2bac67061b0ce20c44c5a803741fdc53a14a2523d0bd3fe5b3a6f247b0c366d0d3676c48bac44d3c05 +LibGit2.v1.9.1+0.aarch64-linux-gnu.tar.gz/md5/b2571cd9b20a307fc12c9d7979f42e81 +LibGit2.v1.9.1+0.aarch64-linux-gnu.tar.gz/sha512/52a2301f1311a8fb3c091b158e99d29267175156979779896d346cff5a5dac8b784f113793ca86bbd41f92c937c4eb64b8b8238df55ae95a6b1320cc3455b379 +LibGit2.v1.9.1+0.aarch64-linux-musl.tar.gz/md5/b53fa9546706075f41cf12c39c4969ed +LibGit2.v1.9.1+0.aarch64-linux-musl.tar.gz/sha512/9d818007520f1780eff4d0b6b2d8c7c1fc09763f36280b55d3795895dfa908d87db1699e9dc265162f831b9b1f03879cd187c020f8082c77eda57320eca14ff0 +LibGit2.v1.9.1+0.aarch64-unknown-freebsd.tar.gz/md5/5975e2abf13b3cefbc8cfd3d1d24a956 +LibGit2.v1.9.1+0.aarch64-unknown-freebsd.tar.gz/sha512/05337be9595c5d94816d168cf245289f2c7b2717c49ea000ba872b582c919b327be3a8cfa877eaf7c541a38897db3f57e11a8f49df9b9bfbf8f3110b39389e0e +LibGit2.v1.9.1+0.armv6l-linux-gnueabihf.tar.gz/md5/a4de1b1a529f130d85a75b56f82551e8 +LibGit2.v1.9.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/cfdb3aef71f88405e7ec3c31df1f84617ad01dd227d0d6dde9a7dcd5e4a59590437f4ae90b91b63ab14a5716b366e554ed8d2e403eca2bcc9f4a3f7b813d3df3 +LibGit2.v1.9.1+0.armv6l-linux-musleabihf.tar.gz/md5/6a5deeaf7dc9f4352dc018df12c1ec61 +LibGit2.v1.9.1+0.armv6l-linux-musleabihf.tar.gz/sha512/cec900743c68c1b2580d21e51abe1852ae44ecadd682e48300d0b656ce955f71c4706c1462c35d301e331d79c37330bb4abface88709dee8b4154d1b65ea1217 +LibGit2.v1.9.1+0.armv7l-linux-gnueabihf.tar.gz/md5/90d0cab30f58b6e1d75b006c12f3ce0f +LibGit2.v1.9.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/735174f87677795e313b8338a50f97ce438058386ff594ae91f7b8bed3085e8579c4da5e305e3e70821e30cbf6d878deb1c161b695b1f19f58bc924663bf2949 +LibGit2.v1.9.1+0.armv7l-linux-musleabihf.tar.gz/md5/56c495c7cd018e3942ee5879d0e397fe +LibGit2.v1.9.1+0.armv7l-linux-musleabihf.tar.gz/sha512/2ea7cafb587c2f2f693789f37c1962fdcd7d4815b064e676c876178f42f8dab400a91d48767bbfd1758646174bf060cad2978cd0cc74fa810955acb1e96b6ba0 +LibGit2.v1.9.1+0.i686-linux-gnu.tar.gz/md5/d8fd641059379d0a153aefc806adc12a +LibGit2.v1.9.1+0.i686-linux-gnu.tar.gz/sha512/bbe3b2903c981037d71d841a1376d744068a5f1ac9d476f833e3f31d7c22284d35262f04ce0fd885d63102c0bb68558e29ec2743adafa6dc618b13b061be402a +LibGit2.v1.9.1+0.i686-linux-musl.tar.gz/md5/c7e31adcdb6735e4a658e8c75d5cf213 +LibGit2.v1.9.1+0.i686-linux-musl.tar.gz/sha512/1f1ec4a3f1cbe14ff1b94ddb1d4c693b0fefec4bff51ab18920ed84eb12406d778af624baebf75a82c6411540cb62bbd3e00d975b38281a7ef51d7b7cfe255ff +LibGit2.v1.9.1+0.i686-w64-mingw32.tar.gz/md5/d28b2507115bcbceab8fe2c1a3141e72 +LibGit2.v1.9.1+0.i686-w64-mingw32.tar.gz/sha512/42d3ba8a06207dbe4a9862981f18a08c77fbf7eb0da047b2631d4cc46352e6403c8016fa6490b6732165c8286e341bd167f1d4953a77607102fceec276f37eed +LibGit2.v1.9.1+0.powerpc64le-linux-gnu.tar.gz/md5/ef21df7cd15d2eb71e9b1ed00d30a008 +LibGit2.v1.9.1+0.powerpc64le-linux-gnu.tar.gz/sha512/1b6b76d098d1b3bb48ece5c95b529e067b815aa95115f8df604a9169ed243d8b1b1c86598425f34193dce594dba19d7094ae95860ad7fed723d891df3cbf53c1 +LibGit2.v1.9.1+0.riscv64-linux-gnu.tar.gz/md5/52eb7e4c1057c704a5abf4a3526ff6f9 +LibGit2.v1.9.1+0.riscv64-linux-gnu.tar.gz/sha512/1d1378143ca1f1cb61335785b6d7db655d00da721ba5a5837c735b3dbd2a5be3207ee1e8bd2cfbcbd08e41f8aa61c028335affad7854cfe499b0210b5642c124 +LibGit2.v1.9.1+0.x86_64-apple-darwin.tar.gz/md5/a571908803d937204a84d0418c49284e +LibGit2.v1.9.1+0.x86_64-apple-darwin.tar.gz/sha512/bc3f19cc22889f7d082363274e6fd70487e0e5c1c0e156098d6f4f1350b78b327f8d7891f3c5c476628e85649a44ef0b2ec51899afa8721fbb26363fd52c1586 +LibGit2.v1.9.1+0.x86_64-linux-gnu.tar.gz/md5/9823d346d9cfebb7d14f537e7d4f66da +LibGit2.v1.9.1+0.x86_64-linux-gnu.tar.gz/sha512/5ac7a780861907554e3586ec2bd2109603af93ef3a6a3d8a9746af65013d3d745f00289dd24312b6b6bbe6154f0e66fd7925e4907873e712f3e765b960a36771 +LibGit2.v1.9.1+0.x86_64-linux-musl.tar.gz/md5/d12787cdb2407c9668dd1c9fe5cef246 +LibGit2.v1.9.1+0.x86_64-linux-musl.tar.gz/sha512/d7b0a08bc154639ca0c3f12ebf3b5d1725edc45a177ac27e91abb83add19c3e3b2c414c955477d9ace6ceeddb5e1f7257a56baccee8134812b5d47932162089c +LibGit2.v1.9.1+0.x86_64-unknown-freebsd.tar.gz/md5/0aecfa9999032119fe0ee6aee13a55c1 +LibGit2.v1.9.1+0.x86_64-unknown-freebsd.tar.gz/sha512/b2b42ff038abf32fb4e3c15d4b4760578410218685fcd2761c91efb366d5430e2dcad267f51c28205040b2714dbe91aaa060742ac66b77690744367de14484d7 +LibGit2.v1.9.1+0.x86_64-w64-mingw32.tar.gz/md5/d610537c9abd49eec7d861c814226c2d +LibGit2.v1.9.1+0.x86_64-w64-mingw32.tar.gz/sha512/5bea03406ef4d2bf89bd82fa6cdd8fa52fb3cc6efdc7e5016a1e3a33e60603be23f867630d7a5b14d2fe435fcade5229e10067a86945a5e46abaa20ce1f7d9dd +libgit2-0060d9cf5666f015b1067129bd874c6cc4c9c7ac.tar.gz/md5/904b3287acfbd1098a447e50b957f76d +libgit2-0060d9cf5666f015b1067129bd874c6cc4c9c7ac.tar.gz/sha512/e3f49faa709456d506993328177226932e3673fce3cf9ac0d0b0063206aa9ce0a6d780348c35f5ad28f59ae50f777c9f4fa54ec35af04582742cf7e36af09b56 diff --git a/deps/checksums/lld b/deps/checksums/lld new file mode 100644 index 0000000000000..6e8fab41edbcb --- /dev/null +++ b/deps/checksums/lld @@ -0,0 +1,120 @@ +LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/8cdfcae09219dd5c42a575ee95dbe933 +LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/050783b4d922a806a566e6d5e5afc77c5d0bd935cc8c5538f388b6221d37f387bc8e81682196c8f07b56100ab2a0aec331738dc5815925cd09bb1b5b0cd68ebd +LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/4beca49829fe5fdce2674c778df340c8 +LLD.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/c7593ade99be7e6da301845525428ac8feab063b2349bce266e02325dca806e875fd8586ba1f9252bb5e7f701c1e750dc26dd7ddb4fffce3f48be918d89152a9 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/34d2f188679bf5abf4c28c7ff4e30703 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/e89e8156ac291b3373f5b723638affefe43a7a6845ea49720d6e39387e3cdeb34e4790de5221df1ccb50939eb8b41141af94fd4436bcafca49e24e3007bf1c16 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/3a431153ad307ff728b3c07f10aaeb17 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/5526f8e004d93b54689f47b62067d689ffc5471289d9f43f2ddacc6c17275d7133e86837de1895b40d0cf969006d07b8fed2ecc96b7b65472d6fd4972f4ac082 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/e9d0903886a7be9fdf5421540899418d +LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/42f29306911257a2982b0cb9630a4ea81b50add6567d7135c9f3098162c699c2d234256cff80ef789a0cdc6ebafdc2ceed9009bb84aa18bafdba5d5d69874210 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/97ef4ddf834a725a7d13faa1ddcc3906 +LLD.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/09a557558f51b3e22fb8a93c5dd458c5d0b397858df2acfd2cfcca11e8fce11666eae867b8e3f51c3b296a175d72fa5716135f0c0612c859bc293d8fbf98d226 +LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/8e5b12a175c78b24ce43e442c735924e +LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/00492672666c6ece6a8273811e1cd0ff1a4fac3f32ae33ff003a978c44d2f6e1024e24f9e29bb970882f766f922987f7fcacc79d2c81c29acea5b397406fbba4 +LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/b68c5ed9e9f52c3954d1664c49ad1755 +LLD.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/5a2cb3cd347ebaa05c0aeef636e8dac0878114e73004f2f903608a87567a2a4bfd36d633df7cf48e20b60965fbb2c8e9760180f0ab03e51101775c75029f5a97 +LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/53f810e9d3d0442f2290954c70f93f37 +LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/417dff21f10eed36ed0b911341903914658240c157ffece632b2e0d28b65b0aeda376b1dbeb1f4677ee0b434c63d763fd44c9d6195b192a94ca575314fbfbbd7 +LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/25713a33485613c69d718518aeb49259 +LLD.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/4542b3d1f98a6357ff7fbe686803103223c82c56096696bc90d89e1378455d3eda19fe45835f6e6d5035e754ad5aa51f24678ad20ec24fcbaee873cc729fd189 +LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/978b490710efe5019486c88d19dece8c +LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/7b47a5216b765f7c54a8f48006d351a01ee38ff379b003f62880894b4a5523c5842fb1205edfc2dc71420f7d19cf31f417e6aea4f6b30fcd905671dafb73dd2e +LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/52562639d81bfca12e3586740f4b35bd +LLD.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/b230117d657a8d18d0c43a2d3da025890f4df3b11ceabd7fd883621c3baff6ce1e8dcbce213edc2fdc733a59022889619a56fa7c1d773d6039150b6e2e44eea2 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/01efa3cd3ecdfe45803800b8b8bcddf8 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/8af2ed095fb79e831b613cb5ad83393b5299cd9df1748f1a6b5771c51d3df2d14af286d00e0fff432b2e97be5991bc6d515fbf2089f3f58ea0cefdf87eb0622e +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/1f5c758df3e347960a5429ec7cb84849 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/1a38fd706b55e202044db984be82f96fe21da67ae39edd396e49a5dc86daaad47185877a1863f7ec21e1303b103fc533b0729221fc87533d3c02efb9e05338c6 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/bc0e9ff37b2bd42460b2627e973f115d +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/1926cdf82fb5807019647133a602ad34a57dc6b45ef072443ce0a5eca02ff7466d5daa93b45e5fc861113e2aa8818f789afabf86a99a431d73bf29e5c0951a10 +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/5426e6022fb0646d61a8b48f4e91f50c +LLD.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/b4a0dd3f2b3b9d4d643405ac9ccac3448ebfcf199568ddf7079b6b77c5f29b42d9210ac2a6881416449b50d4d52b822dbba3ffaa007cc79028c8ef49c07f0c31 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/08563d88b2ced2f5ff6f2a876b10e88a +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/4bb9d42394c5d6895dae473257af00950692be4d07a7d393e70d8a7397535389f095c4c65d5b54342e51068c76157d9737c404b01e18f7648d54ee4aaf1b36d8 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/2c20b768d5c77175613b59104b848086 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/122177d274de1867fd383a4d90e25564d5079f9e7cd09e5167fd449ab0d0c5cefaca867623921a6163872fccf6b65dd0640a3b22d33f30412e3e2f458f9f4f9e +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/d301952a3da16a4d6a32e5027d946982 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/90dde9fa203766ac128c80a167046508b9a1e31121b5498e986e469559876a6c587c01016aef6e1ebc328fd15f8f479dfeff94042f8e172b6a4e8582c3696995 +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/364812da3ab7e29e11c14503d676d01d +LLD.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/0a0a819023643ab2d06800e7dc75bbd0b738b91a11e2257dc752f56b8d7e6438b85ab5695daa72672d4797d871b912c210b6df2c80f516429f9e8335a97a18bf +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/7d77b4217d5bdb192dc545749cba9fe7 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/16cc0e199d37d4537505b1cca4e0c648d5e9e49b2135dbb2b96355501d9121ffab32ba1a076ae1b77a57378be3bc32822d2109bd85a4c6a5747e50e1ccac3965 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/efd07e543cc5bbb3767f9d64c83eb385 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/5ecd6029bec97fe1cb56dbbc1b1eea39df5c049b23465c8eb1232c281e733d59f730dc38107368468ad56a07b3c864ae4a0d51fd2503d3d4077923e7f1ba49b8 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/cca1d7d104f95c432ddde95969f43c09 +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/1ba73da3f31e1fdbcde0289e399fcd34b7064f626d32cab724559f8d0a893287a1aa67965be5ef85c9f7e4ba645f338e852e0f93ae8942706deb5e7654a6e38c +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/9414dd3e1cd61b52f3ee312e5fa4faec +LLD.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/d60b7482c276a76e288360a37c885689dcbea4a00b555768cad635a5d04227f294e85fbca6f586036ee3a33ec77bf9f9f3bfd110fcc697cee8f4586935067ddb +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/22fee78e5d5fbfc99d426382d1f1d7df +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/195c19aa3ebff8795cc6c1862b9d2c8a577809a94da20a860de57e4200baac1c2e76683c278eac6d54db4afd976e11dfbf055a008e7e44d22579847cab7f2eb6 +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/356ea4ed4353c3d7245987e55a5f6897 +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/fc071585b97db5900a6d72c109d78596c731a2a0971f46ffe26b4423bb1da3488df350aa5eb9e7ee03b5b2fd66c3c80950c764e0703a7c6fd6a4015af05b994e +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/3dfb7a907a0b18a971e4227f82009dbd +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/bfe5601d4d4654f15330eae9168327f9dcb91bc658cf7514526776d13f4b74ce95f99a5dafe48d59deca645f22a4f5b6103bbb0e35ebfcec85e82767ff5543a4 +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/fcc8d611a6bb893408da646268faa061 +LLD.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/7d09101bbd6ee53bcc4ecbeb0883a5a4fe440de49902595c53e6f56d8e0c67a350b923210e17d22618569ec087ecabc097719307ce18db0cb7cea7c16e0b805e +LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/74faaf27eddb69690ab6bfd66479ab05 +LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/23aca5a1f80fe35fd1ce7a3ea563bf490215c9e2fb82dbbe7fcfbecb6b5a73dd1643e019d0446c57c6f31e9f7a1f5c52c8264555f23ba3c5fa8275edb6715ad9 +LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/6ce4b175bfd0242be28b02d22b8da351 +LLD.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/e0b0e9f27f0546df44985e6b1cce85711efa9c3023d9bad88c25111ae86107a72af5c2921014173f8449eebe9f57ecdf6e1afb0b10d5593ce9b90e0415aa4954 +LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/83c135cffa8563f67f0ec1bd8cf271c5 +LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/1ab03926e7be1691e9e79b15785caacd00de08ee666a0fca264fff5fab1c277e368d7c6eb77beca32709fecdac730f7d9df03191ec2240d0ba8bbf6ad9494330 +LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/5242e12e1c74e2dae73c6b4b7d4307e8 +LLD.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/0f88cb79e838b2b92111473638fc7f2e7973b6301597f80caf66e378077cd306768d6cd18035c26262eb6e79525f1690dedc6d6db2b7dfaa1483ab2008bc7d9a +LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/7290bc875a2fcac989aa8fc3122ae6b6 +LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/a332d69b92ee11fa2ab5ba909c8d3632ebda2bb86d74b1a4a93caa62d6cdff5e21356d92b6e5a5744a5d0b8bd94c0fa1a4e69ada9c5e006cd35ddb658e37352b +LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/645a495655eeed199af92f26c0c5325e +LLD.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/5c6b16c22322bd92a0de52a393a87085d1a3836f85a75257675194a6fdf6783aea9e77670e176371cceb637b9450c7c5b2b4d03f82e1580b813548b078f7b254 +LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/f060e69be07766a078cc9d8158abdde5 +LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/e4f006aa955304144983cd3437fa2ed213e0e27d904d69deb916ce82f352809311e443522d31d928e7f4c1907c0bc82828d422f5b1f0c9647abdc0c4fae7c6b4 +LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/ee06aa4b93bcb60c4434339eb68715ea +LLD.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/cc6f319855af8e35c282428bb3c9012f0c4c9c4a8dcca1065d2d88c732bbeef6cd21cd42708968335ef4625fcc50e80d126e1caf545dd79cac9a23b4753bb16d +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/80757f0836a949560b66c03891472b07 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/ab3f0bad55b902837cffc1e747a39a7a0887ac5dccbfc6fe31571de58bc88340fd2b848d7cf806768e0dba4b895ae9c945214757789c9ef134f4b7e48afdbee2 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/716af34d7c11db49857a31b685bf30df +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/9e05fd855b39a8b33781902d801244471f38bcb766b234dfa15f71f3a3bc0a1500b2f2a56c137b71edc4c1f3bd456145ec589a188640a2d0eb2ece0c3c361075 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/44f9e0d5ca28b85641b6c06daab74733 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/778d85ab80755b95a6b2b61b676026487cb8dccde3c68a71935e9b3db4cd113577d5072d4db1d09f0c7233a24bbca5f1e4bc1fc192b3a7839fbdc0a68b649b2d +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/d2b52996d60e0e4f3b3012d481840842 +LLD.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/082980ba92a796db4d9b757445ed9400099f1b2167e79b868da3aaa5d3d2e627fb0980b8e9351d8f2fce9845839aca6d386ea59a1639f5aeb5f7060904f6cc1f +LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/df3f2fc3722ddf530462247bc484e6cf +LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/cc881793eb7f2ebd9bd37058b50e87afb3fccebfa545a3422ba2019d51d17457d4a0cb6a49b8c91e74e76ec9cf923aa1d8ec0d8e8e285d149a521622b4c0add8 +LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/05cebf19f1eb5c1871efd39de205d490 +LLD.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/0cfb6fdeada331e5d0ede9a7c50b1677d3b4bb5a36149a24f957cca4944e6852bf016f4ca2aa99dc7d9a590a28dfc2e02399feb32846064bf3e3065920400198 +LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/bf911ffeef7eb44907d288b0cf18f434 +LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/66fe8f97f4e200eed12d3e7dfd79f1062318af5f0d4d96fadd389f7582146fc45a6dab593d8346ecf3bc23f02a84035a22ed42cabb7222baf406890a5f80856b +LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/45556dcd1655ae34564a17bd4fec150e +LLD.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/bbee6388e40007eec26c766dda6a6e4f3ccd84656157e1e81ff103295cc1dbbb42db6c858f9f5519434945c27807d880934ad791eeb8e13579003edc065a272d +LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/0647f954526b0ff65dc35a1b65b4b756 +LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/d614f0ae0da90505f6d8710077c7e9a06e815f4a7192c167f791488ce5fb8d0ed2e6e3cba1262c8d78917217315e820dd180f8838cc9fd739eb4dfe8b7b7793b +LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/e13e78f28632eb19ed1451ef1f243328 +LLD.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/d035ee204120915c9ff7fe3385f1066993cb7c0b21df970be62ff573a4adda395341b0539984f579c236a973e18f05ab8e8a4befc2bae34fde08b03e9b975807 +LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/6442326b50f9b7cca3edcce9e671e544 +LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/294b2bb952ebb724c57c9f3f25e66a87224d9686053b3e4d517ca82ad728093371d193852450b579a586bbf2c2e9b4fbbc703657335524253a23a3106ebab9cb +LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/51b96db9df5fed624bff8a89bca2e96e +LLD.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/27086b42e8ccbbfad398250853cfb6941f5e6a2d81d6dfeb99b6a08e456c017cba3ec7978e4cdf2f48b3ce771b7015bac7e5953648db0a1e65c601e9dca76b56 +LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/9485f990275be736e6ed8fe9c905db2a +LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/58e4f4ae61ac789030322125ffb6b11a16eda792d1f9e8d01aadf5e399375c40d263184f05143e2d01402bb60df068c6028ba2c3bc9757037f776c0f63de1be8 +LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/1bccb2ceb7070060122dc2c10811be4b +LLD.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/06adc0285afd6f5de86b533ab73cc5005769d44562e748b55062ef0c195e8d6b0c287a20b99582e6096ab0c989a072a1c7bb1d347a37cbd9634d165fdb78c7ac +LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/ad160553bf0f20984759686db7b87243 +LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/f64d9d3ba6b9595ec745f9326eb7f1564d99330975e2997f387f969c21bbe757fa8f63faf583a115887a311f8806e70ec3b9a87ba6a8894a09c0e9fb8496c357 +LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/0e5433d428daca35c954fe261c82a874 +LLD.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/67de9856e05cdfca1ec25ce9661337a04b01ee49f6f2eb05ab970d19ee585beea425c5d8c6cd72a95c8d5da48ac7968a1ac2ffeca9817fc695d2be020149e76b +LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/c8a10b058232cef8e9ffd0d1ccfa6c4b +LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/15d54ad7337f337a340a8ed0b2ef2bf9b75687ce691369fd9780e50a3e064e1866295cfc9460889f095501a3d48b7f891e1af22d8d197097640ceb94bd257aa4 +LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/4eda180a515b58400fe5230fa309bc54 +LLD.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/87c0a59e425768fdcda295ca8323eb6eec2690aa9784dfd8d6200a69820c2dab8d8fdd7497c2b53e255a6a5bd855ec3ada9d8e1f65fd12322833c58363772137 +LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/f971f5bb4255f544511e689bde1a66ef +LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/43367f52d2278cf8f96bfa60aedca4174e9a97770de4b23f4a586d644439bcd437ae9cbf4a802ddb22ebb6f5411084d1fbada762f29a34ba8ea5d0859f1100f9 +LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/6ff7a32c9089f1c0ddd192ff037fe1fa +LLD.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/af9e2fd3f6cf68d45a6469f1458e2b3b21ddf8522fe96a6060a511e80e9da562c7c13ddd4ecac0f77d2e432618f617f829514684f9f7cab197adb9ef8f68e01b +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/d3cfeb84b4b46d430a859cd2460f1a89 +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/bdd8c1503844f8c99960fd7d72faad2993ea56ba1d0b3af65b37a4fb3df586779e864379cc53a1387b50be55259a8f2f6d9f1f2544c1e09365d2ae65528139e4 +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/32d94428edf18af0a887570b1a757a3a +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/55df0a8f4dc2ca2df5926562e40065fc01ef1f9a3f88d5ec0e6dd40458ac66922a6292091c26b48bd1bbb599a081c1e73ccf34093b11f2888b97ff979dc75dae +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/462735ba1292bb622c82ca8652d0d587 +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/f0fdb9764db4870e4db9bdf2fde99caea1016c3b499dcd406a188a5c973a5d3e804625fabeab8bcbba51a45a503fa9157a02aa9c772538d5a0b8b7953642fc0a +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/0c86dee0baa0e9e71c5c703e45d1d726 +LLD.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/28f9d913805287661765bbd24c15466a39c6485ab96699db27cf359c18165dc8121e95359065c16f16376369fc76a63ac67a451ab2c53f189b064c637dfdd0a7 diff --git a/deps/checksums/llvm b/deps/checksums/llvm index 1c9598e36d5e2..552904b4cc204 100644 --- a/deps/checksums/llvm +++ b/deps/checksums/llvm @@ -1,482 +1,244 @@ -Clang.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/d24be00d258e18c03f5dee3f866438ec -Clang.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/ba4e7fee70883076b3b42c0999a34ca9ee2070ee25134848c8d045f37f121ebe0843d2581160536ce2e09065a988dd203081638cc63f59344e025fdbd0b669d2 -Clang.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/14d4438d746cc21a9e3658d306cd9ce2 -Clang.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/cf1152f18498bb407e61b927fbd99c6699fd2f9e39040769541961d6775da2f6b41f2e61591642ba56cb221ab64354bf20973616d30a47ebdb46550df0400721 -Clang.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/54f14052b86c7ad407f5b8fb9fb4654d -Clang.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/de0782647aa7d1c48932165b9b5327a46e9f22319d26752fefa4900afd561bf075bb7d86165e54b032be00ff4169348bd95ff846a5e6f0407832fbdada386c0d -Clang.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/d18298df54c86f54b95000006fb941a7 -Clang.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/12232f07920d9b5bdea27d3b465ec5391867279fddf98c10a7c93244a773864999c2f3b4256a94ca80eba5309c9d5cbb0378e1a5392bee429a386da0691b0896 -Clang.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/2be14c518b0281b18d3fb5201d01e44e -Clang.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/4b164e20ca8248aeb2cf4fb2330d518a325cff42dd7376cbd53e2063bd1f297bf6977f1115c8e893a8c37a31934c4012768fe1890601a82a21ac801cb45a23eb -Clang.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/e88aa91527728e3b04f97ff03cbfb2b0 -Clang.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/472ea39ff1023d00761364e9f760f26fb1c23b42a437319004ad41970f1a43a5911292e4be2f8e9c599bfac068276287bf7af6b0562e77e65a30907de18a1780 -Clang.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/451b4bd9fea31a0f9b37629e550d5269 -Clang.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/37332d2ad5022327bfc5d86f7ec8d3fa36396bf830c8a9c95602d8cdf30408e9d6d5acfba1c5f2768b481d7b72aa1bf5ff3f72c1752a00dd4a562042e586111a -Clang.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/48aed502cca5dfa35ccee123f4d661db -Clang.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/59a0bf98261e24a66bb899632261f639bb58b3129e31795ae181e6bfd21e6825a4bc18779a76da706b520e70b6fa719bc96966d3fa2785c2b95321430f986d78 -Clang.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/9b7568486b5bf931a109d491a5425e9c -Clang.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/7d0d8effef66ece3aaee119b563bc52f5224ecaf417930ffde312c7f781ee06d5f5dac3cd38da43fce0aa9b05342c6a7d5dafd16a642648d10015dc1978dbcb3 -Clang.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/463a505772082e1c0642dce32dc20f45 -Clang.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/47e2681e9cc79fa6007ee5675f5cd980fda3f2e2cdbb2b99ff23ceb6dc1e379056ccb85b4e8450527d09d77ea959b1bc53435c664a58d13f86830ec225e67c30 -Clang.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/30cb13a56111c5e9e6383eee7500f391 -Clang.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/1a3b794f38833a152bbf293caa867c60eb99efc31336333d83ae4ebaf9b72186befedf175c7061bc200fadd4b6910716bb792a68364baf3764aa79b4149b97ed -Clang.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/22a26b9d7a0b713c83845b41132a1158 -Clang.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/d89ed637970713b54a91112a536dedfdbd3276381f4fe210f67110a9d2ef51f6baefd954567b6389dddcfa753145c21d4ba4a3e27e211c22f898137317597ca2 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/4082236dc1e82a23ee8bc084ff648880 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/f544b692090d1c813ffc5f4fe0e354145b883660f089946d0a98fad1056d2720d6e73bf4052d92bdc717cf8c1f5bf724348c95e87fef789b3bad30cf318b0cd4 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/63a6ee473fe3e516c86a7d6a97ab5e26 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/4ba0f2cbc3e7f8a4bbbcd7b7221b29dc24fb02fafc93402f7408efe7c6b6e5f4782ed18ab466527f400723732cfa6be51995199174eefa2554f6dc290c184a15 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/aeaad6628f961ad8d74a51863155be1c -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/77a3fb05722045dff7b691b5709fef72fcc998497956a5e570d23d7c0975d7ce38a42dab7215158eb37be4cfb8c0eb4bc84be8338b53cc9843a5a3f164078406 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/1c0b0f913f5e70d1b5fafc7f9a4e8c86 -Clang.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/c7fd5a7efdcd2344c62b9ffb535296b70756884dd31d705ce712f3083f455ea72c403290a6e20d95012676580ab15161addb0a25e1fb70e725f7f1f569bf8387 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/6c34fbdbc8ae24ed6fed62ed2ae9f021 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/6e8242592602f2e147a38d61f76e31b1049ecce05e205b6e1a521061609914e0fd5a339d958d0effa114b3b654d13aa9ec903159a3fdd00192a78f21ee164ff3 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/5a6978a950e3100c637e226ad6a42ebb -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/1b4249ae27b4ed2ba6ddc77c395cd74fd7f55328214a854164c0ea8fc2f0d0d7314fa1755bb7c89b1da6aebaa38779478c9143a3c1206775bfc537949bdc6d9f -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/88e2412fe078d6b3036edb4465407bc0 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/1ad968ffa4e19b2df03a01732678c5b9711023896c2aa99d8ac6b40a614b706778409bac852563982e43216d1f7702941810825dc54da5e1a88c9c6fbe80425d -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/ee0977fa4e2b6d8e5c8114d90194d7a8 -Clang.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/ee190f8431736483b7beeb8acdc4136bd0b5a1d8d48d44c6f6ee297fb3102a89710e8e0799ab04a67a9e1f64092640325aa553f6b4cb7a0e7e53c3ce55a67e30 -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/76af69ba0d5c42db1c95d60ca4dbcc98 -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/eb442c489b0cb29e083461a099a544ac742d46326aa804a868d839744128bc8c2229d9a046ccd03816565f7b064ce347af0f72088eb6dbe4cb5731e22ebef77f -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/e0c1af7589a6b9961c75f6bc78df57ca -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/b436f7873eb9de3591149f2252fd5d171f05a8dbaab3a6f316e133700042450d267bdcc1340e25762712e227012688285282835937655a534dfe9d1fe7da8c6c -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/25fcac8f8e5b915b25ef0985ba5189da -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/6eb943416307ea668c044d66385dde58aaeed702226da0cc2c0fd09bdffee7b446a2f1f296ab0b95193853dfb88d13f21fd3e7c69171ad8f666606b176eb358d -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/e02980fb5c1d4bde16568f515c792f0d -Clang.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/dc8ff2276611e99644df9d9d98d1f24c2eca6ae5f1618293fee0f55263ad42928d18aea65d8441a70b99ff865dbd48c19a5727fd4a4e89c72666da85f7167aaf -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/0b48668e223f3dda099bfef99931c6a5 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/2b8a89b62f53c7fc32a16bdf9a34617774bb7c095958667187c0461d8b2c6bc66e1da5a238257f21d848c9a2bba9af6e27cc5f19d9baad336fcc9550ea1b7920 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/811735d7db6108ac0e75c6fac50d2cb7 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/88eeeb1d64b4c66958ce740ca682aed23157162e9de5e3ebb66d75d0dc98235e39aaf0c9ab2ecf9e7dd0bdfb0dde70265abe86571d39e5217bf2abd562a0f202 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/9964abfde7b35e01ae9acf2b801d2c1d -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/651f4876e66e2e54d46ccf436264f82ad28dbf082ea7d37d2c094727f25ef045f58ed9af04d00652fa2f606b8392186f53af39e6a763579c57a91332b6545a35 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/85346afdd02d674b3b3a0fe0e1c28110 -Clang.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/c89b54a68d35aa5e50f79198afb5e82e5d611fc06c8df143de4a9f7a9c70382d84db94cbac5eee05be06a83a4db1ebc913be21b6a6660674b350c51c77a74689 -Clang.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/cecfe0fb1e42b08e8e6f3d2cdc35fbee -Clang.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/069cfe48cfcbf8af2113bd2714005ce713f8e0e1f1d59821b2f3b83a15f58dba89b8c416debfc6deaf1752b315e51968eacf093f6e062f79925584c28e3d6415 -Clang.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/6cf9cfd6fda8b453ec5c4f26ad243bf3 -Clang.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/43a6cc58e5ba7e17c474e23cd72d13a1d947cb9ef63671d292b6eae74e131956cee57e420f413d70fadf41f55ef17d3206b8575bf51df98790ea02f7a7a29e67 -Clang.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/11b4b6c30f47d88ae6e145d4e32fa942 -Clang.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/c4fab2da5ca763fd0667babc5752b9b6b185da75518c79f2c3b8fee1a95e14d8b379d0d1d66492cce8200078d3ae796e670b0648b6d654f29a7e8ae54fc6c725 -Clang.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/38716e1b6856f825be45fdae522d8d65 -Clang.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/212599b9710a59a5a8816e20c9552f65373c687677253eb5596d48d30cf18a0f85094d821a7e23a0a5561ddac00cd29fe7f86e001bc60dc6340a6c283555a7c3 -Clang.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/31216157dcbeaa703bce9e4b9099df8e -Clang.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/83f2d4571aa5aa77d88afc33d96043bc599963f5d7843819c84ffbf9084aacbb0b34728e61d237eab66b8edebb7978824654a7f9a202ce1c202aa65d3001010f -Clang.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/05a2a02ad12e22661d922feb9121d8bf -Clang.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/f344bfce1cea72c19024736f8b35ef536bd443bbe946ceb61a79be81e86440bd8e52d17025924c74fe7c75a63ff5c3931b282501f3c65765967cbe54336d742b -Clang.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/e0b77baa8ade7e007b7af33ae8bc0097 -Clang.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/9e40df04bb5ba23be9866063e99ad3a93d00901fc31c07ce5f1b2d58d54fb8e16d47323c81492d30009a94fa0951688a893d750f90b938cbd800fee257cb278e -Clang.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/f0c68c6feea0845279fd576980d001f9 -Clang.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/8c9f07250a25a5a46898fe5caf06a6b9f5a5041fdb1edf1ee7b46ffdf1da90441967ceedf087c8295a22424c11e7acb4643c36b05252db0355ebd2e25e2af30c -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/d4a0231358cebe08f3e406fe3aeb060f -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/2a2d28319f57e23f9acca7d9438a58ce21c69d82cc8dde57920fc6262d92eafec068fa86be2120ff17a6720e7fbbb23623bf18937d2f5959932161989e62413b -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/f35ba6e11afc974917ad3f9b6dcf50c3 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/a928e5c3d6a87076a9006f2c2e2ba7ccf3c701d7ad016f356cd9baff84e6f3994387a318c4d76e3c176ff5b80d947ab1ab44100bf76acc1185924888637d1327 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/20075bf92d9d3131399d87d0b288d8f8 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/499b59f86661184cf4afc5da1031ad8173d1d36a8a48c6ac2d8b545477b21a8757e03586dca35acb591759e366bd1200b50651dc9aaeddced8a5f367d2f74a04 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/e4a9013e9f5c16e7d4dc8b8fe59b6967 -Clang.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/c1e6aa9a039e35bddcbdccbd951dfea5bf0fa1f647dea57668c679ab0a24db9bedd3965c36e7f49f113ebfd5332dcf7167cc7e8cb875c8e2fa3c77bad8882733 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/47ba8c618ae6000c62e95e31b61c113a -Clang.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/e53f815ee9e4a9b0e150bfc4a38582ed69d87e76d47612b4c1e6c91cfa2a2c41f0221094dcd1a77410691a63cbd01d5f82c305200d37b270f4490528f16ad88d -Clang.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/9bc07ba5ae5580ed126c5c099f41fa01 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/58c4c1a2eed1f5c824e845dc117534e8e9a2e2f91464e48741764e382a44372a854b6ef576a95a104073236d0f84300819eb5f43ce06e6e6298dc6825ff1a1f5 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/e20b9db2305a41e38f810e418ba23771 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/592913681cca072e3f8d5f2b893b8d81d7d210a2e3ecacce1040cbabc56d0f2bda19740a54d271dcf805805cc398a9aa13ba095b53d5e3cc3385cb398d8def43 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/d983df7963c49546970843ed9055fb04 -Clang.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/434535d306979f47053997510f1c5750be4fa45cb3939fff878ddfcf633fc79cc506258be5cb478a32ff88bc3fb643eae8a98c2e39636c60c3a70ade135211cd -Clang.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/337ac6885ff6686affbf2cd00973a288 -Clang.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/edb86fcb10c92016facceb3a5a0c1027a4a7b30c1936d3db15823f394c6a2c41fb6b1331df885152afa59cc28122040d702e76f00a5a680a22ce1f1804ce5450 -Clang.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/bf6b2209c091b552298e9381264e82f9 -Clang.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/2ad551a65004764ce3a135f50e58c5c1abefde69f8cfea84292e7f105a317a49a9609c04461693098be15d3a9479e9fb0803cd94c390d6024dc9b761347e3083 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/c9a517645db0b3a2e54eb1e02e22ca66 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/91dc8174e8c88944e54925364cbe1aea3b1349e2bab32d240d03ac99251ed7118a18138bb91432cf674744559d257db843ff3b264eb512281e606ed5b4fd2086 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/b1b59fe70d9027bcf293948202010012 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/56aaa7c6442cad375c92c506ce274a41a509b922b8c58665c2b4e34117531d408d6887419ee51c207102796d6f67b0073ef52af2cd6b5153fbe0a4dc69e005c5 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/7a4d68543169bc9e923c5cef8b033ec3 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/135bcf3f69e0be614eecdbdce51604d8a468c46a7ea209da3359cb6ad45dac3f0308718e5501e2cf981913bba37d105ac55ecd7e272687684f2c8e03a742d150 -Clang.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/2abc7a6d61ac3d943e237df602897c0b -Clang.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/8ac94e63399be1fa9f4de2cf5cf95e1976f1ab875e2266f5ee4c9bc238f8e2a6a8b9db9d841583da5b0f49329cbe9df45913319e9def6c1ed9666829afdca0bb -Clang.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/fb4e476d0c97e84a4e7c3681bf3d8eb4 -Clang.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/8ac633d7c661d8c9d3d2142ac320774623a1511747635a97e1b4818b76a425a72b32ad8f255b9f23a39dbad7b2cf7098892f4062a5d86702cfed55308f4a589f -Clang.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/610bc5117561824b6923ac7d3c87914e -Clang.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/e9dc69adf57a3af37d3ff88971fbcfb3c667fda6263e537491b80ab2ecf21892edfc52b4f531f3d29f11037383d1d7e0fb45417f9372995174026d5f2577b337 -Clang.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/c44c57244876f2bd809a5fe58995ed75 -Clang.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/a74e14b00d01bc387808fc55942e0af8a5cd1bc5e8e6d7517e0e13c9b11728faf8877bf596244f3f2f0847f1cccf3e96be0dafe6288975ddb212deeeae3c2c61 -Clang.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/14e163535fbf9df6a3fe3022a0d8dc00 -Clang.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/911e33e66c3b097c003057285920c20690ebecef6237cf47b4f0ce54f44c1d0a512d76c33b4e170aeb6c92a7e88f849eabaf5a137e3358d3a8b9091fd19c1883 -Clang.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/8841e6dd4825e16ceeccb8a623b99377 -Clang.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/3873039dcc7135a5d8b0d5165284fd16ff9fa27ea423443bd9dbb1d7d227835b53b4b85ad33f7b78c25bcbefef8afa7363467ccca3a3156d3131fed92f4d622a -Clang.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/5af0f418fa2c946bc1718304cced08e6 -Clang.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/de21210e81f3845c022b6d74d3316d51b6f4fd221d923c059cbf67af5edee3f40f6d61c13537ffb856316cf2576508cf80a88c348484fdae009b45ebd26c492f -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/4d42764ec888ed63603fdd5ddc2f9cac -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/a20eb88efb467ff2ec16368f443128fcaa1624b0ffc9deef191cc4364ac0407689674839aed3f2cd5b9db001732f17eaf9f2b15341dd1d4ae8015ff38a295447 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/ac566aa0848d55f53d2d7fe3d7724e8d -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/7bd3d606cf4ad6f2ec0af5e8857965564c14f7049afca3d975e4dd2817ac740694af489086f8e9c9a55039a2d1a0e1438ac8d08d6b19c391d9c3462bc251ee36 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/4db9ef315252cc72acb792c102058726 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/0e1bc63206529710dade1210bb29a759e8e264ad7405a87e870d09cc0cf229c51376e7a8014c42fe5fdc4a72e661c538a66dc3f06ec7b8f10ece330731ecb7f3 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/613278da489d7511f9b6633312e81945 -Clang.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/599b76f72e771ed6050975ffca0cbbcdd49000dce6804828ac4e95e28eb802788b47ba8f211f7078a93ffd316be2b9692dad08c5814b93c79fad235c5823196d -LLD.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/4bd306f4d6b381430b541600a4a28c19 -LLD.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/300fb7a3813128bee40922c276eead1975900887a1ba33fa196235f41e71c8c093df864a0e4ab4b5397e4340d04e61e165400a33dd5b7d2e6dec95b450589382 -LLD.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/2f1e6e33f03a27786b0a94700fbf175b -LLD.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/daa63ceb7afd2032d72d8731cc33bcf499b1168b629fb77a5905962832b5dd5f372fba81917525606e1fb5b861c18b96b183b8af43c9187d64a1d5288e28d8b2 -LLD.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/4fcc5f0fcacd6870e246dff5c5a800ed -LLD.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/f04f135bffb82b9ad6c13e72f8d6257af0fb3648afedd05e08c8b96f667c523a8614fc8143ebe3abd1043fd93b6295bd3b7f1cc19dadb6b6c29915401f3d8bbf -LLD.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/a2ef0ff32a8716b53e03e06406a5195b -LLD.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/b7d700be2b662619bb1f3695484d8027fa9eb2bf57342c4e19e14133ccc623d260a444f9364be8818e95c27a14b90f8c6c35cbcf422eddaab350325a557a68da -LLD.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/27e11239cd800b8ba423cedb915d035a -LLD.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/55ff701624fedf8befac528436385711e8ec7c7e0596ae23dc9727de4330d66417ef4a3055d47c58ab0bb68c60340b2c2dc4913340c04cea057def03451a2ff8 -LLD.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/d742fa681b4d9cb903c474d7e485f109 -LLD.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/b498591b431b8bdef3682e2f1366ef25913f93be7106ba86f6099750f01082d3ed8b34abc3957026abbe8ffa1941f8feede452d85caa45dea6b7a7cbecc1b072 -LLD.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/3bcbca15cb295ff426a61e9f6fea8852 -LLD.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/4a326f6f206d3fcb6a22caf9098954eaa8477bbd5b7dfc58372faa8120fd33746461342391c31946fe30cbb35a4f60e1f2fc9944cb23ea1f833d9179a865f52b -LLD.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/6ddad390e091228e45d30d0c7da18441 -LLD.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/640b9993974755791133902c3c54d96a730e4522165cacc90c352d7c97dd56a8723fcfb564cf560ea1ef0405d6e7fead9aed3c5eade081745b0ff51c8f7b1018 -LLD.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/f84d0e5e29b8d1cdec3692d46a2247f5 -LLD.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/1463bce08e5ab76a9d04f914ee0c0e56f9523d5dc45127345a09932a235b7adf26d33706837fa962089ef401d676ddee24cc7b2bec6031f31d3d3d78bbb03f68 -LLD.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/66d873d9b2213776eb35131d505a40cf -LLD.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/fd4e7f5ec55cd9b5381a5320b1b1642b7d7b0c4500bf85196167ceb7ab67d80fc01d0304bf12a18dabbc04460901c44c6a3e445e25c2c357690c9b3e7223c8cd -LLD.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/ac90f023cece0e8393e322734e7c42ae -LLD.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/c054a6ecf4c0199a144e222b24a72613f17af3e8ca33d3674c43582c3b6d0f6a4013d2a1ced263ac64a7f6fa9a10f0821523102c3c6e0fda41ff1fceb9ac2c76 -LLD.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/64acc7d428bbd83d566d3bab45b7d0f0 -LLD.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/fd67b402a64532c1c32f76bb070c547c9852e5cda2c0c0797d4f200610d7365b9e23a31b3da205d6621960e8753b28e337cb90bde5253444d25b16c25b015985 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/b3dee71f00faeb6d296c96a4966f785d -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/21003195e2060099c7b8343eca34fe73091e01734a53a1502be01e1afaf59f1192d58372c0d99e9a08b2a28e34c8054dd858b88744fc82ac5869ce923e9fd45e -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/bbd802a7be8d924fefd147aeb0e38238 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/2153df14d2d5792b4ce12f9c4f54f90de6bc13c08b8faf41eb0892b27e2d87eb9360eeff0131b69ecc64128ef48ef5c12f6e514e9207a96d910608ccdbb70c51 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/8e81e8f6e755bedb21b383fe138d1a76 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/3c91e58e26fe7dbe2e40070a8670951aaac79a1fdf7e569f0a144add283b59fb08ae9664559560d67c453412c063a9779af930c919dc2b70119de5947666e409 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/323dd4b5737b7ef970c4e8bd350b5912 -LLD.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/a4a06313d06ab90765d61b2795340a44c97afeb2e17cc0b2d01b08909d7239a300c97eac9d1c4e13ad4bafab76102eeeeff73f5d3674afbdd976baef8beb30cf -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/91833fdd701af8a9c697359ecf7d8aa2 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/1f292b47a88cf8a36e046df4350b97f026d5c3f6cf1f8d1da16a7a320e51934c4ae68be4253bab4c0953cee9849c289fbf5232a5eaa6358d6a07173ab66f0e94 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/0d1e06843e4d743ea3ce03841ed23598 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/13d046ea6a34d23fc47fec998067981fd1bff2d8d7e5077546f3b716d5c5634f0a7981502bba869847ce2159a5b3093d2dda93c6e0f3788733b6901dd5554088 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/87ca8fcac9b4b85c0e466fcd98b5d420 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/6146429da998203953297397f7c6546b96803f1f0c3962eb0ad5e30ae68555132fd88fbe0238fab5f5e13c4da43afc7cc6e0d3a1695f4d0bd5aa1c471f5c1a8c -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/670af9bc7d7c0323105de61f4720e238 -LLD.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/7bc4664b634495639d761e7186a4efe85dc3b485bf9f69b752b375077048326d8303343c23145b4d1840b12eae5dcb8727ee717d43a88c721ef63935b31f599f -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/a5c7e360cc1c7702c0bc3f14c5e3b09a -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/eb1754122ff8a435a91761386c70e9e1d00b9f5e76452f620d265c191f43d2606cb2623699ce68ee25951ecadaeefd3c548cc0862069d84ed8bf58618c04b1c8 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/0b8f88aa139ea1c76d7bf61b0b337673 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/b1cbbee4aa54617a683fc56ffdd5582e4260a5135d10a4c40cf0dc9dd65b5b0a86b78d4c22bbf40088248bb9676d7801c162149319fce6213c4eac045072bc26 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/2f9ff3a30f68f6a5bdf70e0600fb10f6 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/86f57613aff3b844d23956057b6c2ef1f84f58371cdd4375ca696d7058fa92691b5ee9925b3ecced7eb7d3593682531128b44eaa14d3c09588e0d14b13851bcd -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/4ec21d62b5c3eda8c99e6ce11db49a32 -LLD.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/f002feb859e326505c26c4970e027c0eba65575c5645a852e77ac8d01d8d445a227d2f85127204df8961142907dac492d07ea81f053394ce61ce1e02a73d00ad -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/3822c46d8db192fe84a526e0f8edb960 -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/974670d88fdeb11af6b1a296042e555f4978a4593d317f3057427442b5766d8266a9df5f70688e2e0cf55453438e381fe77ff6c3782aaab184d3766a896e790f -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/56ee0119e1fae3e2f4418f86c71c953d -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/27aabe86dfeb070675630854439c0ca2488bc32224476d5dcfd21ea481691edacebe20eecba0ffbe3e0fdf9719cec688fd3fd5d13d08acf551d4e0ee317adcdf -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/501429cff1d0b4374e474833668787cb -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/86c5f4920716c888ffeffa3f274581377f222449bc8a806898bb65e416cd1198063144098f8b9190da06fc3ffef290f33b229bddff4d059271de8f02f7197aa4 -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/805ce8c2df0c901d0b12971898f8e059 -LLD.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/c5b93d8689ea302d02a619ea2f36b4c82c69f1e90ff3900c50c3bec4ff45e307135ed4f1a7dc28966c18d9336fdc2d55f4d60dcd9b209adbc0822af972323da7 -LLD.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/31b3bdb0cd090d659e93930a8ea7fe27 -LLD.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/efa27f69a5a42f9b933ff28be872ac815be47bbf4626ccbc836ce9a9fdf98412afde4f9e1640235342b13dd4a7592763d39986d62038b55cf5528c5a7b4f5049 -LLD.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/095bcfc50bdc01fafd5f3e17d3cdb7cd -LLD.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/d1239fd0d0b75222030efc5a1604d1914de81a6eef93f8452d85870e4fcbe4e13864234dcc2d3a74f01812b9113a909c7985898091245c13f5bf94042c9a8cae -LLD.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/879ff2ea16a1fe12f324bd1963233f72 -LLD.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/571b20623a880a55dc46387619aa4cbb8ffb57d810108bd5c41f1299ade43a00a64b80b572d1b0caa49fbe458cfc1702a14b7954fca529031ec54485367b2e25 -LLD.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/4895c9717eb358de254a8fa4491dea6d -LLD.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/ca07bf2935d6f43dc75551ce5462259ea26eceb144bc31f53c56d0e3f4436bd26b72656e854837eb2e6c96ab69a3d2ebcc48db97f7607175315c58ae326d3488 -LLD.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/b33bd8c4c4814d4e893bd9e87afa68ec -LLD.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/0b6e18cea038ddc4a8070d04ac4e645dbdd34edcdf33dcbd2ee9cf03ccef42544d03824ce9a709d9e1a6556affda8955d1669a667bb275fa7867140dd5f5dc7d -LLD.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/b14c2f57273d416cb7a46da90563db12 -LLD.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/712d317e0048909b60d595c5dddb3e948e4945526629eeb9ba42c8d92d4eb2e5073169231368cbd73669b6ab31cb0311730480c4b150748864ddab052cabb33f -LLD.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/275dc909fee11673e80914c0d5ba958f -LLD.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/f6b5209090dec3a7d4ba9d39b215ee47c93bdfd3208563de6651b608aec99caa062de7e3d6cf744d4cacc0d4c89c4660541c9e65adc8a99de043457a2e882f3b -LLD.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/d62294a5252369c3c14c14402c3b60da -LLD.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/0dd65d3cf4eee36f3d63f6b2e5f59dd4b9fe3f40115aa83571697da2a4bfe3a381a8001770a32b0ee935de90c13b391357ec0dd6a3ea5421ce6379daa0ad1080 -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/078e1ee96626fd7c49c959d102cc4816 -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/a4b45ea18965d227fda92647a1a518dace7d68ca2b5ce6b2b378e46e9382ba56dcb55823cf6ba6ed224226987f45eb7d2c101acaf5c843be2f5d07922287647e -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/f5e6d729156c48a0d158f791a7aa0aa2 -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/6556e418cc07193b9de1999b8a8054745ba1b1ba1c1aec4723986d0d0338513259eb76bb84fe7421dc5123752c1cdbc1a292462d0490e414ed10f2a3a0d589da -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/ce065d4387ceac874a7704f47d169a3e -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/79a3d1ed6f90b64090a6719fdbcc4ed303276d319e9f9aeeb93819e08deed1433858a5c24844db308c8f873e3470ff228bbcc60f4bc141701a5ebfef18667745 -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/f3a6de4415f7460ff440274fe365b7ce -LLD.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/c6907493320c1bf1037400f9d88e428fb82d76d9767e4ef8f64aa7f1a94bab0dd3b464053ab3f5db78c8251477ef9dad92d9ed15f6423abc620c6d5219288496 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/8ff66a201ad3503b98170928303b4243 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/478555ccf774e3fb7bacd875aff108e46fd23e6301e9ae8c01116b60de6e1b08db7d1a86950df21bf056cdb4229f53efe68024682ff8a8f587390fc7b6feff57 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/2208e55d29b1d0854d5fe503381b9758 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/fc875886cac2504462bfc3b329451091fb47ec0b492ca64f5fa6b7106e34c4527e338f622134883500cdba501d21c9b72be7bf8520754487c541c05e276e3c4d -LLD.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/960dbe73abd59ec25d5cb29c6aa643d2 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/b8b06dab4c144866df3d207da01ce5e2c1287eb09e37e35b5f5208c46ca1771ebe5e4bc52798f494ea619b30cb7720326ea5418cd99944af591f89204a8ed5a2 -LLD.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/8d6ff4b3aee7ae73347cab3dfa13463b -LLD.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/9b19edbc489076c7435e51422956ad278bb51aa1150b84b1f540bbfcc2dff7aceb0d26f1e442d0a0f760d00c4dc433f2c3026595c1f1d862c590f473f7c6d508 -LLD.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/c0e97cfe35911a1331edce84498fb1a8 -LLD.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/4ef74c537a0f51c35915577c39053ca3f0b2625ae3f130dc4a605408424e49aef21aed00a894e942a25e57b58730aafd8c67c81784ef8f72885dc7412b5814d0 -LLD.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/8a212805b53c001a1b934149f5f07a27 -LLD.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/05eeae5ff8f52514e1a84220cb06b1d2dad5d70c023771b362864f9ea0b62138d7d8c80b58ea5c46cd51eae569b14b3ada209855fb36fe7648c81418962f2378 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/a95148da62df6df3fe8993b77ce4561d -LLD.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/c830d8774e2a36ac672b769f017832b41e38ea1b3d73b566dc099c1a63213fbcf2711904bddfe04e5a0e884e4f9ff48289697f07e2e84bfd665de55f640697d0 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/13b5e6eed527cfc0384f954b8ca2b11a -LLD.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/32d140225cd0410ea3eeb340679d17809f8f7e37c27ff63a7ae2ba4fc96afe71cf48caae39aa846369f5486518a6d184439cfa2c59bcbb93bb6dc0bc624c7138 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/21074f3a888ca8f5023b326e09c8170e -LLD.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/1ae17185c750a76b9015e6fecff2f68d39b4d85740368fa9c4d1a139cf0f5faf54dbf77d83cb3702393d76e59586022839671e97304e36a2607e2f3b3ad09800 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/a1a5b71aadadac7a1e90dc64b59467b4 -LLD.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/d7c5aa523ef24900cc2b9b1c3391f30be6caf817adda0d638d1fee26af20550dcee8440671f02a075c25de3232f42a8bde5b8f967bdbb1821bd2fda97c11277b -LLD.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/d9a77cf3b3eec63bd3d5d3b8f557b8ac -LLD.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/8ebddba3ad1bbbbec0d52bcf5e8163d34b560116dc99190ccc61a069446b176e0a1f2eb3caab8d39bc1e09e596ea684703b003acac977d0164a55deaf0678bec -LLD.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/4f1191e1c3b09fe73e579a6b38f76568 -LLD.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/d2990e316a1ffcab9310fffea4e51906fce058a0cd4d10cf3605e624b67bba0b08b52623313d493c29b82e1766fc9e3effab52c7580be2b38e9f7a8c85d0e798 -LLD.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/026dfe3080246343b37cb62c964fd5f8 -LLD.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/fb6c1454e53b1eea7d96ae48d74595d43cd17991059d870d8e80622effb322ccb3f441e8fc5dbbd376fd8837c7d2d3ab7135b9ebe1a261872cfa5d45a7a3b4dc -LLD.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/a5a3d048f996d78f26cc9058686db33a -LLD.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/0023dbad7d79c0d6798fc47520ff7c07072efa5dbe2a5cc7aef5e5ba1443e83c7c86d124d4ac08fa9150df6bb3d213272b783fb929df899127f1690f93eef35a -LLD.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/bf265929e24ccdc52d580cb7bdc3a3f8 -LLD.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/28b743402d04f2e5fd40e343ed48933ea7dd225aa4d35a8cdb097c547f39740708e50c5a9610e78aba4eb90f44c93b2530094adfeee8db0b72c71ebc0a846174 -LLD.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/3724e687de73b4d91f4f56119c750179 -LLD.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/99699b43d8de9d6b68da2908e15a83bbd6e583ae38ec489ed7be8894f4eeb282ffd8dd29d1204202c38e0f1178031ff42e61a1e7638c24f475f054352f953759 -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/4dc9c96eb08bca5484e8cda69ec34e6c -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/4ad4b93e60c06b375d07a13aadec231fe017b48aaaad345eaa76fc1cf7bb5d093bc776303f04c74b45eb4a1c2e818109aa14bc7a37170bd3061b57eea93abe9b -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/5ee059f4d562ccfede9cf0954da887dc -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/a8daac9e5181e1a3a5b11f3853b4fa3c6bac15fb6d0dab54ace9e3c482c95d1efbd150136f3981e06d9518a747234316427be245869c8476bbbefe7358aec732 -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/939490a7bff4ecb667da61755c7180aa -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/a89b327ef68665738c8d6af8d247da6362b425aee5e19770905d81e575bdd926a822e4f34c938dee985eb670ccdd5016b1cdf6046154a50cfdac328fcd58bf8b -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/8de3cba498a259ef2c51acbc7321f592 -LLD.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/5e28eadba7aec85b1150088054b7b1e7c420c43159a2b0ec75c7bb2a10a919415b099c2e2f886c53a5ccf91a0e1dfc6da2a98f39dcda51d5280dcabe78618b40 -LLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/e9a0ce56f669c371a2fbcd2fef1c8153 -LLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/79bb18db7f6ca80f72b52488905a9cc84f9173a48f788162030b7818830695d6136ed1298d8200c33b45a28023f8de5cc14c1eb5512b22e471022ffabb968520 -LLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/3abe35329a645e22be94d8ccc0bc0888 -LLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/5ee5c51d9a783aeb8f4cb479b14fefbb921ac08d2e964463d362b4851d694b6735d98f6e4a24af1766b441014a6bfb96665ffbe6705b1b05e7d6fd3e406f869c -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/667f436a793522c55a6592c615f330cd -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/ce25daabd9a61cacfd41be983e0bd756cdc53378f4380b1b69c8b841ee1a282f13bcbf793c1052635f52f9c09efec39ed2969cec7b8038c77f4d878589af8262 -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/e26f47fb96196d2390b285bd80f117a3 -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/05dacdd9d16c585a98d899ee68db36c615d4d5432d22a75ca0813a0cad0ef0ed16a38f6a696ac32f892f01adfcd3c9d5d3464f3ee5578d345875b85849937d0b -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/70c0542ce7d8d471cb234aeefacdb370 -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/5aa6e835d3d34f8535489e133a486dc8e6a6a6490095daa7056c00e0b90b801d6d2271a99e503fed2f2be3017e340c4e12f011dbfe89cbe4a0b810edb8ef5645 -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/f8e57dff4808081af9715d8ffd39967f -LLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/96b6186d30d4cddce4787da9326201f206c4cc9d6fe58b9d8d40cdcc78dde01965ef1e921d4c45f3cdc35d46220a3eff729e474bb0c80275a53d1f65dff90be2 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/8e415bf1c337a73433663396df4ad308 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/6a9d221abc26a9920e6f72ba3118df312f5291b49feb9a1d1671bb2818602e2ee46a3eebf7302b0d7c958bba674bff8641f1e34153e9fdd4b8cc0edc114573e8 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/40c9181205bffa4dfb78783d5b9f3729 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/bd2bce44797785f1438e42eff0cac3af927b501101baca6cd8e43683bd22ab85d01774fe2eaeaee1bec8089b61cc88ecc06568fed4448a48b5bda946823cf10a -LLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/c5a55408326f565be0e45066008703db -LLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/79136605ba3e35bbe14aed813b6a01be48dbd8ea0d20bd483183045d6f6aeab7b39f0702896796ad249d6a035bd97915c3401985ee1bee1b3cba2e9a89860f17 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/d1bc5ae2ebf454d45c7deef799394e78 -LLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/1781237f2e510d6737edfc0704b3f4875b79c0bdeab959eef2c2802370f0d0a13be712c7d3bb3f0eb892961e62e9b99b7a57b539ef3fc03541d96bac07c1bc20 -LLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/2f04b72c07dfce87916be643ab79e6a5 -LLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/52541787aaee8df1e4f206038769060f2c53efb99f3d31c7cc61fd03e1abd5e674837584a8f4f6e7d6f923d881db39f7e0a88cbe7c4942602b19ad7923b665b6 -LLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/3a3e235155d00ed229e8ec60ada8aac6 -LLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/bbb5d63390780ca76de3471c1326d4eaa7a41d65d97b3b56e0aae48ada8d0d2358c7362a09ad3bb7ec7d8cf9c8aa4894e5369b6315dace3d4ab059ee96d2f252 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/7ffffbc226902f689e8f5b68919be1cc -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/a89a792a3cfb19fcfc3183b565c629a1bbf1dc66d2bd7da5e377109f64df769ddfc336440423ec54fd6ba49112acf5c3eb452929fae1801aa0338d0219b11db9 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/e280360085eca0bf61246b384ca27569 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/bfb6ff7f9558de84371f09a4830c6c8cd3aeb13b077818966ee0e4e69d8bd648811696f2bf6d2c1259f2f8cef2ed2da903c6f0f81a7ca03c9473e330899024d3 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/110a384a11b95ccfd825fcda830234aa -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/90be37ce2aef1ee5754ecad693fe8a65ed2a1f8639f5e6ed73a598b30636cd846ecd4aa487716af8af71866f1359d594f27121c217c3e84ce1d686847e1006c5 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/806814307fae511da5e7b2a7b8bd71f4 -LLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/30c7d44e7fbad96812788bb29d2dbe3c537b852e72921952df5fe4d12800dc44cd499b7c551e7b14c3b605d5cca956634944cdacb5e90cbfcf4ce793f5a30910 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/2c7f8e8290b0bf857a6a510bd71913ac -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/6c0f02c1e9687031c692d2890fb19859d15b662f38ace1f4cc7502bc98c59982c39ed7931384bd993d5e5288be34ba75c0a1ad45fd8b96eea13e11a55e0a69b5 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/79ac969bd4fc54e8df9ad597c531c2e3 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/66ba7a12ce9fd3885a4842966a46d377990a12fc0cb0742e2a4bc45d5b1a6b25e5c1045cf9187267d91e228b3917fb604e3e87cd6e0f71fd63116164f9fa3561 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/1f93530df28dec64e65e8bafcac0396e -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/483194ff0b2d2b9b142f5ad804906f4e0429e1e037aea652ba99f98dc9f69baf4228efbe416690be1a7664f0c48663681fa639b86d7cac6a4e8e537bd1d0fff5 -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/d2f918e3293f504ab23f877b154f531b -LLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/9a7d9dc7e98cc27c51c8a1669bbc558921bc82626eccf5e1eb114210a5a16f326b2d3e817c4d531ba7075aa0ebf0f2a34a5c69bafff184320a9a2520ef34924c -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/d89c8d6a2d1dab82f9fd9db8ba5dfaad -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/cf80e352db737ec6ff85034e559e1adcff653837dcd8fe4462ba8416ece81ab8c7109914f79cb882e5603e9d2aab86640453ed2724e1559d787cc37bb13d1580 -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/7f09c2e06409525ea22dd8898d2efc7e -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/6f68f84c14e5ef1fa64b68dd6a74783be91706914a0bcd1b5a2ed51667c5ce37f151b15f621875d44ced56632d75c091bbdda12e246b9b5e5d944cc7101d7f4e -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/2a1765effead816a29428133ba21620f -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/b0c8414311625398441187d222611dddb77af83309eacf04a8a38d1b3fe30e73392cba4c68ca3f6be713cc4c25fb45d2e14557fabad5de9d3672bd270a00c756 -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/66f95c5aa1e2557398dd5e3947fb5f53 -LLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/79341f8e86a8796cc297242f455c87f8f984b1f0a37e72ba767276faabf8537d6827f96aab43192c5f92d4bb8528837be5509123614f86abf3b43478d7862970 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/26564072393a293f5b449340388ba097 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/b2d035eb429385d6d9bcca83d19ffec9a627ba93e0de9daf8b9d956321262aa43771e9f6cfb54b4582fd85563437e5bca475504a24ee077dfa7001423e4fcb4c -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/a4cbda097a9fa0a6382e968b4ab26eb9 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/3413fe7d5b4fe1fd4fdcbe9463bf377647c6022759d6deb0444ccffa4ee0ddd1a5339e0e3e7f8d0952ff90ee056c9fe35bf893c1d84646095110aa0ed54ae83f -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/295d07c1cd0b9e6b7841ffc9ef49f694 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/502916479d9ab19debbc1d4f5f6631cc1754bea79e8a90cb9528be2cdf3ca03ca45bd0efe7511edd1e5b2409305d3acadc0aaafc271d953669cf8a605da795a9 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/fdf675e16faca1de2f58549cff2c41c4 -LLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/809de084cc33f45f67c4d782e6a282af1a24c01a742f6ad432e75f2da720af2787489f92ff8dcd66769886325eba2f066a00038dc750b97830d7e5c1c38b219c -LLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/fca1feb1d920392a1c8683ef3241324a -LLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/baf865e9717fb0a2155dcd4cfde3154850c6cfba86e72e50d01e79ec534ce6ab51f2a7913e3133c0f78f081c7a8a90782e5e2634cc20be87a76de263f6ec1627 -LLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/1e9fa5da1d51e9a7e82920203dca7125 -LLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/58ec85403e16c4504bd92b270873f0473240ebae25b088119658574b740ac68e23a88191c5c8694723f76248393e8fb1a05d2879750e25000bdf666260545a9e -LLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/000ebf842bddd43b28d02e13108fa9ea -LLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/2b256013d51e1f2099357e56bdba8a2910f06b09aac6f12b325d72105f164024694900286507b43739cc4195048fb9be0996279d44ecaa3e46b4d075b4936fc0 -LLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/ecaea52eb5b8ebfe81ac96b02a4a6712 -LLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/636bb31231596c04f32d233e447fed46c8df39ec717a66eec94676014b0bcbab982fa34fbd824fef5796c7502db38ec809fde7589026638ff8cd09983f43145f -LLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/058bf6012bad71b802341fac19ffbab0 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/890cb969946d4ce2f55978d1ccbe4dd4eca6edbb9c03f385b3921f87e5a55df2273d342c14afcc31d31f251c58aa3e95b67a084b73380a4ecb304c2b00e4aa11 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/ec4c7d3a92e8ca0fea901e395f99aaa3 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/af83b9c1d404819c13fa2042541bd81556b434376700584efb603a07093bd9e14d90b42b4dd8278662be94ac6e856458b28b4acfd012ed30937f8a1bdac56271 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/9ab69e8ccd7f3bb52f3d334384f0b0e9 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/6a7899c208934e85cdd1513d34a6a222d6beda56bee3bbabbd8407dac74399f446e68329b95f38cac9d36e19545d6681a7416384db0d2cf51453703a4e567169 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/393592dbe8c83f7c9149ec59aa307999 -LLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/67b6867a6ad98abdc291d8e961d76f44e9531a6576e32ff973dfe7fe4a215f9943b5ccef86502968eb36ca92f0b287f681b4c0a466f1ee394399c96ccab95dc7 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/fce0b3598cd022481644eef5e0026877 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/6d98970d16317c9dde7fa98a04ff730b58f198afe0b06e0114883a5fce6ec773b56eb3ae6673192fd714677cf7b30a80d17b7cf5abfdb63de388ee849aade61f -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/ff8bcf145a1addd88fe5b08a38926329 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/b527235e9c20838b3dc0285cb249634d46cb289c5f8750c9a9ebfc22f27881a0c517cca702ab7d98d76eff9621d2b897cc8855392dfdbccc497d2b3cd49aa9c5 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/0d00a69c0baa5c7675420740437a5173 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/827afbe21e90c678f607090faeaf7ed892cfebe9cdc6281ca8a2da3b2cee4a5d86fc9120122827c52189b40f2747e985561d35b67f7c6171787a8a4ba9e6b4b6 -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/f2acc4d798b44797f8540f9a8e9a209a -LLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/2261f77ca2cbe317030693d873c8a71d0c7293038f104605aa170aa10098b3df87cd54f0ef10455a8e82341ac33d8460edaee57ca3754b7de394cd1df2bd37a3 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/c9ed2fc9fb644ea69263ce913cd37bc8 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/d44b21e04b48ee8b9bb7cbc0351517207d3d07dadf444a054a3ab1cab88dded8743be7b7500d63da84cf935e6be976a178e6d3700bd45e82f2a98ed69a1b1f22 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/f6a066937f6b47eb3c5cd8952605c2e6 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/f62a8999971ef84f1543154c4835332b2d9cb88a95116fe26af19a29eb918ce7a41218c15867d3b342384409f1ae793a8922c1db3693192e7a45246f8ce74111 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/c82b61f4c6c7279e5430bffc5cfc5099 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/c2fc473da360d25a15ae54b546937f9ae46093d44f063e2692500d8cf6bd7fad0f7f5ecb9d5e36f8bdea77eef75b189ee45dfc4d8b9baf7ebfa49ac23fe26d62 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/7e7a4225d0722fb37c2772ac90e033b8 -LLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/5d4ea609c99195eb2818b54cf7c642b146885a4856e1f97ba1e96d9fc73ed73f3d530bd0f332b973dbf773a39694f82880d86312dfd401256c053f026e0206ee -LLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/dfaafe326c11b9b5bf7cf708a333719e -LLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/faef0dc66f71956dfc65c476d2d70e56e929641b88738d01dfc71ddb79166ace2f47daf20a40a09bf53cf23c9053ac2c9eeb7e79e2ebc131e7abf0bb8eb5cdd2 -LLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/8d2ed4cbb60e8a12ce9b95159f08ff16 -LLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/8eaa10d1658b88c688312ab35588237c299914b500283c2c575a7e6c30b0eaace672e886d3a2b8e5d6c6e54e99612055ca226577b7001f99078bffe6c11739fe -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/aecc45c03661ea035c9a3306deda612d -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/0dc47901b7b774ebcd91ee318544345afbe7db36423fe206c463d0f0e50042c78706519f25056869677466ae15f57b05d6964c1d72d0639c6b204b84e515557c -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/d937dfbfe05cec30207b5f0aac5e110c -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/ff82a33f45c8a9aa72d72f85f7da05fafa733c86f4335455eb2ae2df42b64c2383579e594ef98fa1901f3d67db495a2e167eccd7160beedd56930fed6aa5beda -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/ca4e809b4192835d9816b68302f2439c -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/0cadc24b66ae3fbde1236ff3a056d0e6c8654fbeea0ce38c949bbdefdb226b9be77a7a5fdd498258c0ee2ba7c30cf6146d497213a83d1c642fe6691540716939 -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/a432129e867d9d18a0b897782b6123e4 -LLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/df90c56ad7db6594057fe9a1ca2891f6b6bda4d05c1d194de67e5f2543819cdf094985ce31be121363d8f5b49074a034100b8c9437f790ca727a33b2073622aa -LLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/0a31fa99b88139a1ac9e085af50897c8 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/c58092da7d99749ae4fce1c3a9a86fb69ec45160109c35ae9f3b3903dc7ada6e2a8875bd76795fd221816d3573187fba59192315587c8347aba1210b4dc34b48 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/0fad86259377c37fc6b55c175691556f -LLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/aa34c020ca6993af5e5c1197c1647e084fec8912191fe8e06704cbe1a3a13d983973fce9362736c6f78afb24ed7ee125b58c0e84d498ab25500a75b0c11bddc4 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/7cfb079ae1580be11b732fc13615c0aa -LLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/185730ccc5361d9b9abc29cdbf99c89e6023276e5df14c9c5beaff2afcaa0c151d82831ad047309fabf8a48637ec557fdb27ca0ec813ef055e94245ac2763d86 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/b8b4631f5d7ac0042f83ce63611abee4 -LLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/d98ccd1737a63dc57ef6ac9e8ceb305bf712ccb486b8627e0ad05f4615745d89786da1fa9cc6206fc613279b10fe45b18fb7a1bd669f98035048cfbad38c8da1 -LLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/b6c9dca4dd513a4a13a651e121bbdeca -LLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/e0e7ff4c6a7e0fd13278e61f21488c06903b87c9aec777bf565c8d7de3ad775e85f625f17684183f00e6e409d069b02167350c0925f287b83dd9751d2422b758 -LLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/dea961899e125ebff1008e1a546bf757 -LLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/0da3ac53678252f51778857672370abed0f48aa8905307c618554cabfc19845fcbec7359700f268c8577a5244e588fd6fe922c5dbfa870031cb7e464344ea7be -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/d280bf08e1fe086a23400faaa6faa333 -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/76da76f63663cd54cfcf0391949e8064096ad436c86efca0489fef19473e2a8b8c24aa5647657fcf15c0ce2dd4dc4421b8de88532a768ff48a31fa435861d39d -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/d2eaa8e38d7916ee3c41f0da782a891d -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/bf76104a2cd3c2ddbd9526f94c29ecba2c40481397aeccd84cc21e5cc19b8cf5d5637d787cf12ca4976ca9ecda5596c4a13ef8194f8bfe17098d40300d0957bb -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/6afa61b0fc4949ec91a0def4b2c2ef1d -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/ba0aaa45e8fbd8f3c3506f1ebb7c7d58963108d2be7a6f2991e6f834b06037d7915333f1a562b06b811ba9b38ae287f32f90b87019379a2a55e654e84d7267ca -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/6e5db8ede54690f4ce5fe58024e903a1 -LLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/5d1402963f9677dd9bf8364f20a832a1fe6466abbeb5837882c68bdbade3818798d3e4a5a2c1cb56efa7cfef1b57ff3d1a35fc3a19ba14b33026ecfc3d5e9597 -libLLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/58b1f2a32f0e3adc617c0027eb33142a -libLLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/4ad6eaa6166dfb47bc3b5e46c7e7f1c34666efe4adb20a917e7514a69f611e9b04dd20b40d709ec3a52e9744e187cd7f14d42db7bf96d4dd44d335f2f044f2ea -libLLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/md5/6b335d68cc47c2061bcffb5f36112ab2 -libLLVM.v19.1.7+1.aarch64-apple-darwin-llvm_version+19.tar.gz/sha512/7fe5aa51cbe352d0bfa52f3b87feefa615ed52b9fa57852f85cad5305aa8321864b18626de47d8f026d61579de05b230872204ee2482a25922927cac07a79c8e -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/8ad2caa455a2fcd642f3af38f5d0e939 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/b8ea397d73dc3be3634d15b601ecdf8ab87dc210f35728af6c78b6cc8db2b7406efe27a22540dce1184ba0928b5074e0a760b85e3834a41a5cc36f7e2c0be397 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/70897ebe7ebb83e58bc3aec3f8314b8f -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/29d571dea6ac9eb2e7bfc563d2789b77e67e65a955708413b51083ca9ee10759ac338999329346c08a556b5feb32e8cccac0a86533a25fd3c539542fee375401 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/582520a086a3412fb8412b63147e90d0 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/ae4c033c21aa0486bba755ed2600c17570d16b6cc9acbbf96490f6107d081077928fc716417ca4c55aa1301207cde78e06b3cf839abb80210c02a78af003d649 -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/e58868cfcebf4d8db2121d8fd5180d7b -libLLVM.v19.1.7+1.aarch64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/32caaa07e9305a0415ddbdbb931d48964282822097bcc6711d262f08f7c9230ffd2633a7aa859faf6e983b3384744f37b0d0e45775a0fae31438f10e615708ea -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/f8afc23c3a78cce871e909986de0eea0 -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/1910b76f06ad6c7756cf1fbb83f26252c2a3fc86e244727bfa8e0f34ca218e674e654bd8d1dcc87f9b1aef35d5c31390c4d0b58757be58cec3a3045ac5ec703e -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/58faa63cc3c61d3d98b222f661f37b3e -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/7ba4c020332cb2d64dbb37783ec9b482e08158c4ff9afcac23a8439aa3508779238df1c932c53a4118494632fda2b69a7ab4044f454cdc80ff11469cc17deaa1 -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/72c37072d8447d181f4ed593d4ac49d4 -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/cda3f4d2972c9bd71125f62b3df54abc44832d1364f8534f77891b21346bbd05605e3a646e9773789cfa7895e188dc46848a3271e7f0566783c53431bf3f423e -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/525202f68ce6dc1ab0165e582d3304f0 -libLLVM.v19.1.7+1.aarch64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/4e4c538b5b7ed504118bd659dca74111c3123317d0cb8fe1008a253ff6f4f4405121e49178a719d892320aea27540f9284957de65e39e0ddcc65d852ed29d37e -libLLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/28e5a013899637843d2c1dd09052af9f -libLLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/dd3b60bf1a97ae344a22d1abe4310ae6db096a82a6c88b4940843ed5223a508ce1689e03380e0a299afe400e953a5ca5cf0c1c4669c868aba37fd79d56194f2d -libLLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/md5/fb44bbbe4485665870045ec3d7bfed64 -libLLVM.v19.1.7+1.aarch64-unknown-freebsd-llvm_version+19.tar.gz/sha512/dacab6ed13aaf495a761ec3bd22aa959535141c33c20cdc6cd3119e2fe177454090b5acf7f8c61ef1aa24bfffe6afc583e17ec67258a5089fe3e8a8eb93b6b33 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/2397a9ff620a996a58242c61d8f32adf -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/d21f0d8a27c91593247aaf00a30d4ea7c4f4f80935ad7d885dd04b7b96233fa0981b3f0df37bfd8ec3a36c4ca86e03581eb91b2fd1349336b327c364c3613920 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/698fb604e9ad4e6c1cf21f5a367e3e42 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/029ba8e516b216f0631d280bce741fdc661c1ea34970c61ccd94397bafb88f0f382e363be9b63b4a15ae390c8bdb64685cd5bdea4bce0dbdd56c48cdd1ee4b67 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/c5ab553a5644b66701d68c7d8f6a3269 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/c371920abdd10633215451555faf4369f813295c51b822ef88b86de19e319c511f2bf718b29b70849d9ba03febbdffb7b5e6a43e819af27c04e8acd8bd165943 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/0900377096a24801c51908aaf90ad8c7 -libLLVM.v19.1.7+1.armv6l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/7de7b22352f0205619f6c67de031b93e9995d0ec39f043a8040b9a0380605300736ae9f4a7e6cb8d6005e30851223750767dbee83fdd40ab81ad668947ae12d7 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/63620db5e64a4d6e58567f082580d499 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/2eb8f33d1473c6e5a94f795d2889be3a91a67eb211c23cc96d33d30010a41628d2a3215928617c22ff964913db481ed4c656972317b84a8f92831a8a85d94596 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/f3e949271862c9e4635b354b2c8ef89b -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/e230bd57fdbe6b09978f99fc03dba34607d79be4941155b1680a1d07abeb4e6dd53276f47320b3d6017aca0bc0f7ee2da8750f957dc919327ef94f1fe4403ecf -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/152fc75ccdc0967a829f71e458aef79e -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/ce38103c75c41b151d87dbf18bfb6362706bbe7c2a992a9cf98d7255ce778b9029bc1c82ae638b7d6d3d1ac58596184dd04a7d712d7bbacfdc389bfb36a89ea6 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/c93b5ccf7d1cf11de0e4abb0473fb9b4 -libLLVM.v19.1.7+1.armv6l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/07d925c9d459adde3bc7830077de29f8e906d64a37560a4faf2ef11ae7e3c237ee65701d90765ef791c3d8e29e8f41bce160ed113855e66b1a2fb3d4a5bc678e -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/298bd876e6fd8283eb69ed8e625540b3 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/df219d36c96750e75118defbc9f8d717df7f8c1c8d3ed3f3147ef71eea346169146961147a9d73f0f47b702770d382e580e4ee56f741f8be353526a254baa31b -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/md5/0be962fa7752e35c8e9e2431fac55158 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx03-llvm_version+19.tar.gz/sha512/75fa76552e2dc4f3220da6d5c9fc0e8eb56c15b59bfba720f5a337997cdefecbd4c0bd21ce8b8a55317b652b8708317892c135bc090276f525b6bd5d98e3d3b5 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/c6d4f9e1ab5576d92f7e7246d1f6a062 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/02bd4af64e17c4713070b6b68dbb30ef7c3846a6bb6f014af5b7f044369720ea458cffb1e940c1f163fc3e7dd4fa9844b4b7c084477d496c2486b8d1770fa123 -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/md5/b20cb4b6a59b746954e3400fe27404ff -libLLVM.v19.1.7+1.armv7l-linux-gnueabihf-cxx11-llvm_version+19.tar.gz/sha512/383c758cb9d99018d55c0e185823711359c7b9f7dea9b847cb842943cb28df49f44d590972e4b06a0e826cb7341eeed8facaf67bd05cf8fdcb63e2ff8d21beb9 -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/md5/931705b532d694b8a45f619577039cfa -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.asserts.tar.gz/sha512/54652a871995517c73f1deb105d9ec3da788d162f2b44812cb05c97420069f93e080fe1f552dc14294fd189806bb2954f2a6723047ee6867a30a2744c492885e -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/md5/7dec94a6fe9dc615ea0ab95942370cb4 -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx03-llvm_version+19.tar.gz/sha512/625cfa1fbb0ab6ce0b0c48ff0e6739dee0a8809f41682793d5f76f6a6dd8f310fd7088b0dc18675d4de901af67f032abb3ffd93eb97b407f4fb337afeeb8fabd -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/md5/619798baa20185a9f9f4b526be6be262 -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.asserts.tar.gz/sha512/330e9b0dd923a95b91ab896dd6af1eff717f8606c9c25c00915e38336eff16880cae961480c83de7ce26236d1df675f5641cdf5b835c81f1cf6836a053f4d42e -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/md5/c8b5c3d899f357fa51dfe1faca315e09 -libLLVM.v19.1.7+1.armv7l-linux-musleabihf-cxx11-llvm_version+19.tar.gz/sha512/eab18a3b083145ac47bfe9364359e2ab0103cbe81d64599e344187b41afe20420811b17196bd481cdbfd572fd9f3205396b5d05d40618fdc650366c0c94ad519 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/6d8f7c937b779bc63616f55d29a614f9 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/736abc6019cd97c682ba7217148f7bfbabadd05f624d96a49bf497f69e74f5be713eda5a201339e24a9886d3b4d032a14f3deb0182a1dbbca260e852158d56bd -libLLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/32272e346edf8266a735511c56370be1 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/a8ec52808123319acc90a376ed1e2324a6760406d61f0b274af709ad7f79e2eaf3e7bb4146aca6c5f366dd29e2704ff2824f8dbbc044a6d563199662c8deacd4 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/793bd357314e062040f97d6540ac5ab9 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/6afc799e8d917fc7260ee8482a62fd433c40ad63c9285c68be08c55333fa03bfdf22aeac706ae7035ad70a9476278b46b5da804105019057db7aee87e6c94421 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/c8aeeaf25ec324f42f54e862373c40f6 -libLLVM.v19.1.7+1.i686-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/10fd104fc4cf4d840c7d1087bc11ae62c33b590d93d9ac33bf39d4d13e58708ff9c39ea085b4b08f8e3a6220dea59517ea73f90af18739a32a1c9f6ab2be0991 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/062fb449ba5bca7fdee92fd88111d67e -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/9c4933e45b2e5b6ba09b0817c317acdec7ac1e7a5d6dc465d272449ee5be964d1034c93c993015bd9192655a7a54dff4f8ccfcfc81a7a82c13c5c2e35d7c1099 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/ffef665d1e163a45cb9f8eb8942b7ac9 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/1d9e9aae1596fb2fe852881294254a500314b92a007897abcfde64a62b4990b52ab7afeae50ece62da10ac828964bfb4f3f51696d198b8987bd9eafe1a6cd9aa -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/98dfe522508910bb49daa96409a7a9a4 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/1b3186f215c35fdb44da2a4c6893bdbafd783f1be5cf68845a5626803bc4ff5ee9de1ea009c4780f8e95cf27412453855dfc17f399434fb6578e01b75a353916 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/a69aeeccfc58c037825bbb3dc0a279b3 -libLLVM.v19.1.7+1.i686-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/f8c1ae00dc45fb5c4d9047ba9ce2b43ffefd3420acfb5b72769554532bb819086ed60c2c043586a6cd524fe5ea30bfff213d90bcd03976083b1c04b80750b95d -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/904b518f2a7d00882159b8a62362e645 -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/f79a8f098f12a331866f10d2b767a41dcfdca0cc16808ffef22d4c6c62bfe294845d3bcaac93173909407c703e8ab406f555d69a9dfefb3b997afb0b6545becb -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/b06c7546c197affd4293394ee00a95a9 -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/8459c88f791b059e38f0f6a20bbd6d784f95d050f2dd029f8b83804b5b72bcf99f912224b47364adcfecb487b0f06f8acf98bbe0a0a99f89a97ae26413e4cb96 -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/294d4456537499b5388cbcca3b531fbe -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/74173b6a9f144bf0197afd88fe4dac24a1ed5481f4000092b9e9ed43502a412ed9d77ffd0d74dda8be36c4f10e8af1032ce0c7fd8e1f5a59317b44d8247a0fb3 -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/337ca0b2f64acd793fe5064dd5fad5da -libLLVM.v19.1.7+1.powerpc64le-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/76245773da9b24ebac526a473b73f3ca39f3fd6856f5e11bfced43a4f283601f572701207e27903549c269e36bd388194c168adbbb2579bbe39c3240eeb5e662 -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/9331f7f745d396cd0b2a25fe89cc2a59 -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/7a0e15ea5fa2fa680d00c42dd79ff62aef66ffc3cb5158129eb140c2fc07aa0bcc10089e30dfe84431036ab71f7e20144f9166540c9879107849463ff5267c19 -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/58d9896963896bf2b71eca47e66eef8a -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/c51d5330dcd68938b868a0750220d6ebc341b7d6a0a51bcd2664632dfa9cc691f1c20d41ce95ad8d91cf419f09ce2e62b4612b30508d6ea40f91bcaa9f0680ac -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/4672ea06acaa3216bf3a8f39e82026fa -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/af5173944499bba6543e82e1ad4bacff0b3606660abe4b38955899ecf723c5bfdd8eec3a7ea67e1f951ec7c5909f4e88f25a25d4f8c865fd1731fb20dfafdb8e -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/1db5d937eaceb816d5ea22971986a614 -libLLVM.v19.1.7+1.riscv64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/582d866bb1d4552cb3d90265661cbd65c08bcc73c1d8739855bf148bded253a137699865a913bb90121293b78382ec71747f3b2e100b87b63c91ae47cf21bb76 -libLLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/md5/d106579015b35e555b7eeb73af9cba76 -libLLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.asserts.tar.gz/sha512/51b369ee4e259f28b984f91526575b0b7e20dc9649151adafbd26a52ee32ed28b4f158234abca18b9c16e0dd885e942ad2fb107268292f4ce65f724d92610abb -libLLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/md5/c64188802f790ef1ce30b1be9b6d5e76 -libLLVM.v19.1.7+1.x86_64-apple-darwin-llvm_version+19.tar.gz/sha512/1a0e3623fc2b4ad49c3a466ec03738f74d720c0ec967f6c9b981a14d0f874cb5805918682e4c9aa415a6109bc8137bd8d4795a0ea5ef242ad5ec48339d407780 -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/md5/384e285954d7952d4cf7cb836dc6e01b -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.asserts.tar.gz/sha512/af84ddf41e64735872b7415c50361a62d1bef7cb2a3df1960201555b88112e78cb4dd8e267036b3322595e217fb6405e2b34f0bd3e4183b02dbea32d11ef2ccb -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/md5/58ff83045ba6cd9986e39bee947de9db -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx03-llvm_version+19.tar.gz/sha512/016881f83cbf58f35086d34170a266b3b14a7a71ad6e31840efd59e065792ca4f33d86fdffc69da9d41041df445161c29c2e20139753b1f16025088a857544be -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/md5/5c8aa1aee42bd8697c3d30fe9a0c0f77 -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.asserts.tar.gz/sha512/f7d77c8d3c598ea87c3860623fe32099f6d10875bb186482f32237dc265ef17ab903edde00683f04071b880b55f73768c64fadf196110f90b25471f66f54a059 -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/md5/8677709cc7e0a6e28512a8687cf632ea -libLLVM.v19.1.7+1.x86_64-linux-gnu-cxx11-llvm_version+19.tar.gz/sha512/6c7a334810bb09af3e4c8a93e8374cb02277bf355edfdb4d79caffaf5fdbf7c7c58a9c2e8d8ee569bb3683d32a7f62dd499fd1fdbd4c4096e3dfc51489b92797 -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/md5/c28519001f3331d87d0083e38ececdd6 -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.asserts.tar.gz/sha512/36959aa78954214840a41e58978fbf1a3b0b5fa386312fc1f51165587e0e7162051e653137366e43172431f73a9222a32fd9c7994b9fff4d6d0a783744244aac -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/md5/92d230bc3aaed389f1dbcf77eca3329f -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx03-llvm_version+19.tar.gz/sha512/25987d5c15f4b42b3a419d848b76cbec8204d24562e7043a9f9c16bc2317d66b6705fa7eda328843239a10d19922b093bd7c33fa68c996320300423387bebf96 -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/md5/5b53565b8d141db7795641b8eaae04ce -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.asserts.tar.gz/sha512/c7f1472bce0cefe3decd65453500aa7ac0972fe0fda85e7f5217ff756b99ec7fa2fac0d93487b649e1fa68cfbd158cae74cce6e521184a418ac409883f40be4c -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/md5/964eb68197504db676b74868043a3527 -libLLVM.v19.1.7+1.x86_64-linux-musl-cxx11-llvm_version+19.tar.gz/sha512/dd991f7cce7eeaf243b09caefafaecd510a523dc9b271ede3084d59edf49275f8542a46d052d855509822e776e870f8293491f5af092e367d13e2e2a87b65bb2 -libLLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/md5/0eded78842fb7d484b0d803e21418ae8 -libLLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.asserts.tar.gz/sha512/2b18a47acb21e8e2048b1ef28d40facea0f5dd9bd50c52c8a20eac492143bf1cd0a33f5c13b938c6afcb215c179b6a010e25c3215cfdda35ddff6c7f3f1482bd -libLLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/md5/40f497d34f06bb6df9b56012a6b52a4a -libLLVM.v19.1.7+1.x86_64-unknown-freebsd-llvm_version+19.tar.gz/sha512/29042d5d12be54af25a290d84e44315c35f7f53ed96ef17d50d21d6ef414d34e4c6d44d74777ef9d204f975f0fbbf9b8476876ba416796a790c506ea90d1b7a4 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/md5/2e4b98a1ba70d612d6f5d0f904642581 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.asserts.tar.gz/sha512/1a8918e3a3042e3c4ee45093d881052c10a654924012f12251ea4bdbbc7350fe6a8e83040bc37e9fcaf3e997bc8afb9a7beabdeeaee4029b038111c59ec2914f -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/md5/83473c82c8a2b13dee2f0d65a05c8fd3 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx03-llvm_version+19.tar.gz/sha512/8e3b3f8d2e9a50663b556d815af4db21141969cfa8c4bdc3c9c85313056747f9acae9ecab1b7ad3602f7e6944b368bf46d237333e79a94d3d9956a1eabffe812 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/md5/03b84dc6c792103fdf5068ecdc7f8107 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.asserts.tar.gz/sha512/173432f74fe546df74a4f0100f48ab3f0779802f30258273004842f31f575cd72250db7b95a967269ddd8d732e1e9fc06bb2f01ceab4d976444db346d34994a0 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/md5/f906192c572dbadbed09de707a7e9604 -libLLVM.v19.1.7+1.x86_64-w64-mingw32-cxx11-llvm_version+19.tar.gz/sha512/3be4cced3e630158b4b40fe337a5852eb5a080db0d562e1b0f038f3c36a0cddad4e1be1ec2b91ce2a85d67cdc97daac68ebdf00f993861c2cd283079cd8e80b6 -llvm-julia-19.1.7-1.tar.gz/md5/d8ce0cfe5e7ea8b52763a345c8410174 -llvm-julia-19.1.7-1.tar.gz/sha512/1299591bc245c0405866ab9d54783af2ff85d735d321decb0e38241991e5f13c3f7b4d8588664f1c054559d283ee19cdb99da858d76f9f3b1d6b68d5292bf83c +LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/038cfa3a9136493d533a122a0da0ba1c +LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/dd7921a4a056cdae21a7b04c862d5056776a42e91e29799c1daf48e12ddc049140469ae12d342a87931b7b83f139555675051ceb89bc9930e6683a08f82d4da8 +LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/26dcdd037bb2049b555f4b7da48b6ab2 +LLVM.v20.1.2+0.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/e39100ceabd0304eb6fe1b02292a9c25854c8e4229e36bf9831bb386bf08a2d2cc40719163111cfc624b6da0498e59b853624aa1916505d4db9fed7d799c9ac8 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/ce2172cfae9bc13e28f503ffed4c2bdb +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/b4a53a1a1ff9aab16fd7304772096d73d9716831ec5accab6016fcb9e51e5ea83f8f17d9342a0b03837e710c4f88f418c300366624924df1784faa35c9adcf92 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/6c0bd66796ef1982cfa3ac1cbbbf0e39 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/ed2d9753e7e885a6eb9d6bd5de186df8082280aeeb797e7c38dfa3821e1743ca9abc2c735b917720841c54a722ab44507bb125bb1c1e26f1bf2524b0ab0e5743 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/22fa2895daac28d10ad9403ca77d5693 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/29ce790dc2fc9e4c0079ecd5902c1cad16cfe2b176c9172656d14f9792f48ee3064bae94479fdf0f41c062842325a6c7ecf6f453c50304983ff48a2f5794e155 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/00aced1879f83691170582b689219827 +LLVM.v20.1.2+0.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/b0f61eb2a84b8a445a066abd54cb5c26dfeeafe02f2980bbb3216e8bcf562ac9890bd4dce89940b36937150e6f1d0be3986e3943b9240ac73df83e147dd1c2fa +LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/e4f0d850f1ddf3c3ded9f9c783749005 +LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/31761e04c462a7bda09e6c85239ee356318cfdbfb353350faaca6563e60698a9916e362b560c8f28b88c3bbfd556d7fa1631232307a9c67129b12184f6b34a10 +LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/51603be4b6e55ec5395624a1ad89461c +LLVM.v20.1.2+0.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/b219ff439fb028a1f5c360c3dc38c7332b6abe107f9fb13ba22cf756bd3ebef10d000737d8be1569d0d81294bbc9af6f6e5eb8aea98e90882775523baf20151b +LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/217b079b37b2038dff9905039ba12984 +LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/a5ffb0fca566384f014fd0ea0fb07977eea1f96c0e76a213864be472ad6021a8861055a1f8353dbe4cf85dba161d113240e732eb812ee1b821a3332e24d1147c +LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/7fdd788e1993ecede1e7e1cfa66ce1c1 +LLVM.v20.1.2+0.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/8483d21dbdbab20b40f453e776ebaed3e0ebde517646469e3848e066b6a69c39b9102e1a37e6d97725048423e6335313339b945f446cc0cb0f2b5c9cd336a742 +LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/7bf432f28eccb363d91411b2cb7e0dbf +LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/d374ee3a252a09dbcb5f9ce516e1dbe1c5fd1489d07312213f57b690d245fd06d9e3ebec3ee399fd24222fb6fb9bdee92e7aa58896d2954d7f6c821ccde2751d +LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/8be5b43865cdc03d55fe62166b10696d +LLVM.v20.1.2+0.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/6bf57f66b04d58643f0fbb2cee3c4138205e95f4978136b5b472450a5a6481fb69e968e84b63f2c7a4c6e4a4107e96742d04c30bfc7ca1fb3d64d0247c524993 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/c1e18ad4763fedad78b138c8361f18df +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/a7bc2154bf8d9ade8d346a3402995eeac3352dc7f3d058850d34df538fe004f8121fc91dbe1ceaf059b8215a5499bd03c71f2ac5db0a14325c3886e482c2a260 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/0720fb4c94a4d13d5eef479b99544311 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/c253aed3e4f7d0704083417631a9b64610db9a81c636c37050b194084631f1d918217dee69a78c517aa4972c22c68d96c64f1d3db067cd61ed7106f2e548cce4 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/df1648374d8feb6beb345bb3d048697e +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/74854e8f5fcdb068e13b6b3bf83369888d16e603a9cc4b524c7edeee9c95501e7bc39dfdb119696c0665914947b8363dae72949f76fd74942e15476cb5e5ed55 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/cd02804f5eb841693da41db5d6b61472 +LLVM.v20.1.2+0.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/af22ffb9becb1dfbe529dd30975b258fd1a7bf17f416ca326834f5a7ec534e15a8d74f08b98087de457492cda5c7f2bffb0ea57bae5025cd992324582f2cb3f6 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/4519bd2b723309739ac6b1b8ec9b40e6 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/a184299ad255e79b91232ee079fc8617bfd82b3b72f46c62e39a964b1a7bd664e134f0c565585c10cb7b78472417eb8fee0be833155bf2f2a42acc73f8f4f9be +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/bd8ff4a34fcc829c83ee9d634b2925fd +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/576771e02790b532f89b23a6c9df3434628d0bb48527e39a071bc458dc7483a040a6b0c136de183f3bf42fd95785418aac6cb85b8dc15c69245a8f6ed44effcc +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/4b9f611840f96c4dbc30771c72a93739 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/c10735cb77fbbcbf05e89811c87090fcfe19fd19f86cd2eebf8d09ff6d02f0b23c41ecc6a6c0205cf889047b3b5332aa0e362818369069e35212c2927a49a226 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/f5bd1acb396b3e8d87aacd01dc00a700 +LLVM.v20.1.2+0.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/0b0647af108b854e760ddd606ea9fcc0576793d71b6fe39e0f355fe03538e4ba2c8eb821a7468faf382de92adfeb2f12a07d47fa9f7b28cbe304d1e399bb0720 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/f3ff46e4cf9aea3fc8834d3b9e777048 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/3eb7d36b99e30c44cf189df640ae42316e664e6f4ad20dc83da4d1fe87a2a9375bdc4f54c0e751a6a84d97937c9b5add880b7413000d12e3ec34daf0830efbc9 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/d1644c95cd080315f31a7848f07f1866 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/22a438b6133ce7aa3aecad386e8e446b8373315bce6224fb41f183f2eca40f9c2a5d529221d3224f645526efc3f112b1dc6d3fec0c56cfa6b373d9f7667d217e +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/88e5f3943425417e11e137ecf45b7441 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/0d1b59de138a317a2e19860cdf4fbdef0e536cd3d22d1b8f3fe3e2737e08fde3521cc8e40562c5a304fe6e78ad4fade85af16bf7dae9665ce8b3e0b5c4212eec +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/7d57a0cd005d10393c27414e5294ae26 +LLVM.v20.1.2+0.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/eed2f929a73f4cb17c0e338f7e06b10f9ca0973a53c0b8459a3ec60330c25f4b582bcf0908bc3f193853e7a849ac3da973165c71032b0f99ca80997575fd2f0d +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/8947b6739ff8ba38c5afba85bbb3626b +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/303094d1223c7cadc9f3507f79df7c97c3ff53c40d93136b5ce622403d6625753329b718f23a74fe66d1d31c1517c7ae3574c86c88edefa15671cf236e6f7a00 +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/c72a78b88c7272823f5fda84ed05b379 +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/a19cc0c639b4724d00cefba060207db252794bb3e14dcf0ffaef6787d6a4c3e68248cb62197fe4c9177098c760024ff3271e3660599d25f6e336d714ff2318eb +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/07e0a1e3f46e3601216188222c585f8b +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/e56c24feaad72b16d80e63509449501d37ff5c31a84ebffbc4eb31608511a7ca6a30f6eeefefdb82558ec2378fc74d105acde60d5b930b4fd1bcff35c52ec210 +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/8831ec22c84c9737661aee6684b7951a +LLVM.v20.1.2+0.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/c1205e9fe613538bde8a90ffddd6e21ac849b601e55cf48cd257eb323f2fa52a9ecf0a1bced274efe5d78ea9fdf280b6bc3a21943d2ccf47475f474bef1f59bd +LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/1804610fc03ee2f414b68722b1cb4346 +LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6f36e726a78f02507c93cb75ca1112ec41e99b2fddef8ebf31090996d9b4a5992186062f1fc120b67b19c6da0c905ea1bbe404774eaf4811bf782d3150160554 +LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/bda9d895d9ee5ab01b12c830b31f7b65 +LLVM.v20.1.2+0.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/f72fc2e9d434b0de616a8decdd43145849220d230583284d9ac4497ecd4c06a77589db63290fcc3aa080364533d32c60cdd63f412795f8bf09a452f643447906 +LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/2d32e54cd77a9ebdc4e8ad09226139f1 +LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/2ba917662ee8a2a25827394207b25ec313d515ef4c942dba14d1ea8abcfc90c4adb472c557ff61c8f9f9655a0780d6addeef00a689852a5add01fb7e1e2f5b34 +LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/5d96535fd770ba7c782ee5ebb14a9341 +LLVM.v20.1.2+0.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/04127a1e3dfbfdeb1b7f644adc5832062f649068d161002dff376670dab7b1cef2d8ce91d2c7c1254b935f4ffda1699cbfa30f1b189eeec8c662dc6a00a351d3 +LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/5ab83d6a3b2a80a0ce26473403e9d6fd +LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/57541c1956b6db41d6da6ee4f9ed8e81946c6f50e6e0e9e2d88f5c2049c13ec2599794f92a0f87794ebe13ab7bbf345e1b5978920bd4b4a9419a0360944703fa +LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/1df5afd05dce8bfe48b4de3c83a1c169 +LLVM.v20.1.2+0.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/8f6711e77fea3aa28c6c752a93b676b062cbe95b682c7834a7d290a9f2d37c8abe1a44daca86c73f31dc8d1fad5314866c63e766e8cb581a63a390e49027479e +LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/9f7ac8c4df65e684f31a77c6458cbd8f +LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/f01b4df2f56b3a7fce3ddc2400a71a7795a5e6cc1392f63ee2d164f35acc678db4b13ec6ecd73b716e6405212f15362b751b5e4fc6ebadb037577906421d3728 +LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/b6c3ec59f73a15d6d884b75089752009 +LLVM.v20.1.2+0.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/c2d79fef639b06a24ffec333f7748298b19b40f3c1dbadadd609bc1c6a699c8ca3950a7f4d9715af66c01a74ea22addafa642509e61f71a13c7303ba97fa1f70 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/a47722f625d9ed01e4e8c89c7e5ff361 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/8a581faac501cfe8cff7b27502e8bf5108135eb60544ae2988790fe856caea26151cd00c430c12bac15f56d41357d862b97daf34b7630bf6f3883824fb1700c2 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/0deee3e8f23bb78aef9ab92901635db1 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/7c9cb3c1cd8b7b1f777d1ecc6bf3fa5b3536ef4bc74c2b7ac8dde9c82b52653890ea0c398ff4a53c9545381a4c1534ee6b1be3bbeeadd8b1d39d34ecbe66a432 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/4cdeeebe845a999b0af9ac967cba2c32 +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/c1cda095c549fe4f1e8a14aa9318b0a151653daa10f45598fd1e3f72441ae98f69f3bfc1ee1ad1ab15f88243081ba7a65315ce2996510d09f3541af4a243d63e +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/5ff186b5b66d3023298b02d3d1ba1ecb +LLVM.v20.1.2+0.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/733ce7757004f1dd06ee768389edb13dc86fcffcfd0678b90bf296205492ccb5206042f103ea65330aa291b0ab46208906d8fe513557bbf8da7b05c4e098a07c +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/07a298aa555905dc38733075e37bacfd +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/135deff2656e728dfacff4383e095876ce85da4f8de5ed178cc8919a949627f1d5ab61b2c4dab7fdc98e363b0fe630632d5a9bf9d6d074d33f0bc7ef62eb7ba1 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/0ad39270c0136606cde9bc37c6f84540 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/64e3d2ef2130e83eee773798c968e65dad02cb0313e48946e6f8e3cbff98100d246b0acdaf5e23cc4d2a61cffca2aa237a7909887dbefbf79ca48ad6f7a80572 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/3e5b8898f2c67fec7ad5eada36156686 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/860dff38be4614e21e2e12349d4f0a776cc96aee3870254b42a0f2851e0a3186188486d47b1f804b9c1670b6eef2501a66b758ad6c3f3823b10b65dc7128579d +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/b0c9231cceb48572296606fa1ec3b763 +LLVM.v20.1.2+0.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/db16e2d9dab7d88c27b33218f075442fe4627f00ce32af2ebe3f1b65bd5da852f91047e97abe99e1e3541fb42e592b638d314300ec870646af823e29afe3bcab +LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/16aefb3daade4fc7838c7f40b4a9717f +LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/54e8c25636ebb8a86edef578d039fd4d23859ed2c012025b872af5d05f6b485b3825d88cf6fc8027dc44b881c84bbd911753d6d70a963facf7708b92109707e0 +LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/4533027acd5733377fd24a971b1f0987 +LLVM.v20.1.2+0.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/8c4cfcf4f1961eccf3ecfa48ce55ae6d5b812afbb636dd59c1889a0cef60c59828e42b601528258584785fc0ca357e1b57ecd8cbcd2eef0aef9735b63ad7a778 +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/eb1c74229fb43d9f8f5b3917222b37ee +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6fef5953f3d03e9b1172feb297c8cec7d487cf38dd7d88085ef6e41c719920d2fbe4c62a1fea175600fc3e839c719385d7ae6ae62c606e0294d5df0555d5cec0 +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/86cf13ddeb512f99030957c9d223b411 +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/59b701fa8b2dbf3afddbc846f6e4953f1b18abd74ac263cfbad815c997d57048217a672a9b34cafa9aaf6f87a636d400d008762d80ce35de8ddf61978d1bafae +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/73c8986b390269b2f2a6e14e1b60c7dd +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/af8338185d68774d690ab0b9b3725d5752ff26a32139a0f596780dccd7b124661c38561958bbe5b3c26258a5587eb7a40ed67aebbf0273d238b90d12020f8c65 +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/4ca9c2962f62b45c82104dff43f7864b +LLVM.v20.1.2+0.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/63e580c4431d86925652374e25e00cb5d76fa72e48a2a1d6dcd99c2a6c7236382bf6ff696a12869f4d008ab2e9ffb10b84f7faab1b107816a8cf01dc1ef1b133 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/eba4632089f7cf8785e8aa122068eee2 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/adaf858e0362ac2813f555e46da82427e2366d90fa0e6b7b103cb57a98cda6586284923f58e91eb0d3f888fc0af17d0ccfa7a58c5432ba1f99cd90143febcac9 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/c12059240e4f2d78f99a0e0e912bad66 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/c14865f4088e4dd31c89738f21cafa6f34122a82bb7c20c9ca8f184d3b9c76c2bba6dbb3c9caa1b1bb7183517bc46e23f070741f0083c8bbf7cd4f3d5bfe89d1 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/4c2e4479005f4767ed04b216926031a8 +LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/fbfdc5451a652bcedfdd6a870b550a0c9dbcd8feae81a10792922f1972420408d5e3d69d675768d45cb7b893098b46493982669af55e6892b45654d2dd759caa +LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/1c78eb7776082c7555ec61975df84e6d +LLVM.v20.1.2+0.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/5fc325293747d8bc4508256f016f5ac62342e1dcb58f3d47bfe9027dd127564dbb7c321f1ee689ed2983d24abeaf79909156167774c3ae20ca7697647f7d6387 +LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/6850d32e14a28529daded9a0a74b2f23 +LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/9c837940c27307e67fab0a57b5619a44429ac85edd0e04c4ea5fc2991312f2534f606c2a7b263dc5e82b67b8e33447e3e29081862208e2fb19998d9bf59769a9 +LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/ffe2beee649f7d2624a04616f7bdba37 +LLVM.v20.1.2+0.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/e637f58309e61c568a2e3d28440330f0a8dca3a4005e171e9fe14808c1f25df498ee63d46de54a8827b99a78bf55b0323272bc1e82356227a26b2f86a5e8dcc2 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/fea1960f440c1f44161f7a2bee756073 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/97ab7fc9977fa3d4a7442554385218d56479f5a87420fc75a8827d9044db5ab52f3a46a9ad71769658ca08abac9e25df15f456d80c25d3c76afb98ab2855abf9 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/f71a246f402b1f4f5efe60e09e2f44af +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/568a39ccf791eb983b0d1a94a7188986477398308712425ac1007387a2fa0df99e00836046533d3891ff2b208ed4218289e0953ef0f1b4ec1f6127af76b3f066 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/b66193be08fbb0b07181e333a8857f08 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/17c9e77e907f40e799da4b2d21cb36e6e10bde6ceda6ff29c5941e5f671c8625401f6079f3be0de710b57475f4e7beccc1e7e64824c334286efad7b21919a7a8 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/122ab2567ca9d8824f94299b45e8b1f3 +LLVM.v20.1.2+0.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/78420eaba62e18541443f4b1d846fecaf95b58e93ee7cc08c4818339d3a0705b0dd82dadae2024f6be65b3b2e26011269e319eaa9739af599b640486031b60ce +libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/291723bdec338b312764cc6e33a76007 +libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/105282bd4f63d3c6fbf4e45f5a72a52f1a6154e46e6ba595c70aeb763afe27dea688884e6448d4ef502cc4598bb377f10f172747cfc05b42495ba7ab4222307c +libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.tar.gz/md5/289aede4d44649a0d265abba59041ec4 +libLLVM.v20.1.2+1.aarch64-apple-darwin-llvm_version+20.tar.gz/sha512/76f215d8e45bff747109a6aac5338ac09c5c89ab4af9b82c1575bc96908ba9a4675fc18a034bb222435a362f244b69abfb4282d0bc0c71f84ee9f715f281ec7f +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/267645d41e71d3f3a5a19cc70c63e573 +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/14d79aa893acca711baa5a20354bedec51e8c77a90f52639279d9b4453efb670b647dd0d1d60820b9c49033e848f90efdc25e87cb5aa65a9ac35aa8f6f71ac54 +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/2f03e683eb1631503e28117ffd6b0bfd +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/d0fafe8643cf0e1b702d4022fcc3c4a0c35389a65cca9490770d1a053a9ad8102dbb51e8a01fea46ef47d222074fefac66fdfdd4a34c2805a7761f43281acc0d +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/7729756ba2186da8578b5247e1ece99c +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/134058fecbdb9150435f1b3a9c6d208838562e0af8b3aac5c519502644b37b0d2093c2171a21ebd653ace9d51ca9bd4825343a60d80678496a5ccf876c951b71 +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/55e222b181a94df519477c3d9450ef1f +libLLVM.v20.1.2+1.aarch64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/6958f4972db0871b6005d24a8dc426b86504aa604a75a3cb18472dc4000477284e5c76c9477490252b521e40b17b9f587a7c6c3657e02e562858cbaae601260f +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/70deee22fe83f5f7234e5b2849579522 +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/24e956e1fc8431838d2f4d6b1d12bd90250e30acef64bda2330e542753ea7c60f7233fed0110a6166cd7510d65a0512b0115edead427924e5509f3443632716a +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/1082d39fd25ed971e1ed3f43882cdf77 +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/2fde64b294b1144af2754a543b5a9f63d3d161fe7de5c50061160ac845e4b80f8c447618a90768cc0a239546196a67e89f1a3295bc9cce6fd3ece36749cab871 +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/279ab853b812c2ae99aefbd301699e49 +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/f1f6ff89876bef3d2da28fcb230399378bab1df532a2ee31b83ae4cde8e5689e1bff2d44bb87b61c649055823fa8477fe8bb3445c8c8ed12a0b6cddffba5f26b +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/1ba2e81cc47c6600a25a507286bc74ae +libLLVM.v20.1.2+1.aarch64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/f8e88b9064b14e7565063d93630f67da9a927945e12b4234fce2335fcbc4c365528273058fde86cd046e2b63e0edc666641d5db6123a94865e1450f298468966 +libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/b722ae106708f785689f294f061105c8 +libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/82bda3b9668775da095c9078680042926e76c30ec3c18c893d16319cc740b7cc8e2f52f3a0c2f8f59ee84637be0c80bb11eefde531d5ae88d1d51d14f387d0b0 +libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.tar.gz/md5/ee1c39eb3fd4ab6b6cf81dec58cd0f1e +libLLVM.v20.1.2+1.aarch64-unknown-freebsd-llvm_version+20.tar.gz/sha512/4500d7b795dace2ab6c9279fd62c9a68376fc9c1cb000450dc63547d8c7f2b4a41f523130ab824923ad73efe890fe5497a486748d9efa6feb523b29640693b88 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/87a0be634e7f789b04aab50580b8c670 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/442fa7f7702c2c6e80682d943bef8df3fd4867e0f4d871a3bbb9ad015b53fce4c1f9d38f8fe80a7805d3955060031efe569c0e909ea326969923dcf923180a86 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/e50d255bfbec40a6d4e97cc810aa4497 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/9108628863586f4ff4099d04b45aea75ca51d6540191d4471910a32f2ccb89f4fda388968ee7011932affbfc1cf8d33a02e28e81d572ecdc6c96f2e2fe982a62 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/88260fa90d1a3f9bf0b3f7fe37af52ba +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/b2455517b2c5a698b465b04305b547352e641ce96079d42542278d3b4f92ca9e181585ab6fa42f2bb4b5407b4c72249af5a5fab131efa74fc79a043d975cc102 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/78cf5b861f6f159ca48335b6a927e759 +libLLVM.v20.1.2+1.armv6l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/b11274b7ae94edba3e12004df5a95d24004d9ef81e5af7037741caeaa44b3d27e300565ba82d120f4943ed1f401f5d306318cf20bef6cecba80344c71881456b +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/7f87487fce6b663644013178394c8194 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/6bceaa61a716c0950b176141eb838cd809e46a1ca79ec278f43b8d44bbe1f6cd22efbf2049ffda7bdcc521a5407cbe813aa07408e929a50bed5bfc0e19907391 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/e9fc6ba0213cc88f15282a946779dddb +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/ed893995aadc5cbf611bfd18e9e1a06d4a82da69414529f88ef1facffa3c1adeaf9952b9a75e1b9e40b15141b9fe1eee33f9fdf6e84145b51f5da1c0751d83fa +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/43a4ed1c5818dcf25f1b828f86ae2959 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/f834da31818afff29ff02a305569a3e9928789fd749e5c18a5df1d8173768ea9803c94ed73e0b362cb9111d0822794d084a4285a4ac4d8ce8bceea6a4c276e51 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/7aa67860d888970dcd8c441a8995e6a8 +libLLVM.v20.1.2+1.armv6l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/265f9ef484b09fb38929a9d713cf17ce295796da64eb9bb3c56fc5eb17159d730fcfcfca29b4555dbe39345216ecc12d22f0808b0d855e96fa42f46e4020978a +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/d59c841e3d0129e8e036282fcdb25a48 +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/88753f91f8c488ef851f2bf4548f3e4b25053ecbeaa950b24b334bf9f9ab78152d99cb4b1e88831e257990d1e8f750405f5b185fec61c81a87f6dab6e5b451da +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/md5/7f1dab964b8d008377f42db85e200ba1 +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx03-llvm_version+20.tar.gz/sha512/b407984e1b4d11d1bc3db3d094697972f34790466caa9fdc70b85ffd5b823e08b2335eb4cac1d7e67d1fe74cdc7ef29ead9a9e39f69d8cf55312d560ebd7c9e7 +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/29b4d1de6645ead45db6b9f17f4fc09b +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/509e2dcd9c5d2262fff70ceeae54f6d67995bb7cec316e199fc8c7032c74ea8896ffa3431245a33c28f3f846ef267a21c5f798b2ec8f761e6b3b4af4d4412899 +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/md5/9f079ae058a98362d0ee50d938a3125c +libLLVM.v20.1.2+1.armv7l-linux-gnueabihf-cxx11-llvm_version+20.tar.gz/sha512/7452b65b750ffdf596fd0cb0740b459e18b23586b833f296f4db23cc5b69a72f6744ae85a7c5e1bf77a07563addcb17480b6feab5d2314dc74feb6c80b398978 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/md5/2bebe715afc61928505ee08cd8ed3e9b +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.asserts.tar.gz/sha512/c5ae5c79ef4d73d8a9d8bc99d0d418d1f0e7503c7a9e90f6e40b51bece4a33a077acd1e91d1ea355361d9911089179f93e1a5068532646cede01fa6941100d65 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/md5/b6db0d56617db57ee07a90ea4f778b32 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx03-llvm_version+20.tar.gz/sha512/3aa62ee8df72ff360a660eb9b282f00c1690c181f739757423df618e224bfa8601e93eaf4690ab7527fa6097f1c0663cbfec7f0616ba897943a4a95bbc2d4abd +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/md5/8d945748dd65b28134943e6f8b101dd4 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.asserts.tar.gz/sha512/85371381137581a804b96c4864b6bda27aab0bad39626f2e33bc69bdf1ace5ccfac9a2e2255c1ddc09fcc67aa90b25be27fe8f88bca9e0e4a0bca284cb653473 +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/md5/2628e4c8be69d729c063a91a7db8493b +libLLVM.v20.1.2+1.armv7l-linux-musleabihf-cxx11-llvm_version+20.tar.gz/sha512/8dba6382ef1faee5fd80729e0754f288397a8dd20eb540536edbd952be6d43287aaebdb22576ffa4879a76577f275ebde9c87fc67af1064fcef815cc7b59f8f1 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/d8e406a6745deac26a671c9d4c567ee1 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/6e615de0bd6e3020d232af587aa09d380e7d0538daaade75ee06d99851e2d899c926ce2f7316bdad7b1e4de985a99cb0b7bc37cdea4c365156346896aa256197 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/f09676e4ee1c2c18b8906ccdc7fe1833 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/5aabe655e54b1045f94327f6358003ea13c435c313dd75c9c0a870a76fe47416d86d6f768fda3e4b69ff7c3040eebdfd8b7ed43edeaf45ec2e4adf438b10005f +libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/a7d246bcd0fa0d1425e1a405df00ef3c +libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/b27be72f5269096bfde5c26f15eed7285137dd83b802d8129b35557caf4e575619e66cef31a758323b20336d3003e11320b60e3188e4826c6802dc062b236fef +libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/7a4a7852ce434bdb89a202c8ea8079a1 +libLLVM.v20.1.2+1.i686-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/b3e6c6337fa388a095e51d03ec235964cd92de809db8269f14f346c7cdeed8e27423c36f72aaa09e6c6d97c552830791b842c317779e986c13d489600eaa71f4 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/15ca3900fc28ac7f59142b003411398c +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/5a25f51486a42ffcb62cd5904b09017bcbff9e2d2b1ae4460989dedeed736ac62c32e5d0c3168aa1340f541bb9558eb4c4ef70435cbcf08a92635efe4de37a43 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/bcd5a589ff854f7a0f51d4809a5c90fc +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/2bc56d8899993b7dd0243eebdba4d45d869d237cc8c8c89e2505de34d5752636c9e55bbfe60d7a91fbec8b37e14f8fe736c7f97859122285b90d6d399e38f582 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/9915cafbe3063f5d579d408385d8fc41 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/7a8aef9f82e6a4e7e0ab75a912841b4c02219f86cd5afba0aefb32f1060b343425b3e4ab54a89557686ffa0557bc4cbeede0219bda980e2f0d84a401892ed433 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/45a3430d5e980eaedf0e3d149a83f985 +libLLVM.v20.1.2+1.i686-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/ff849bce1450e67f9a94676999914f0a7c8aae60f0cf1037b70a727e6587cab77dce8f549dd6531d0a20c87682780c5bd6abdf301d3c70b7a12ca1db23a5a420 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/efa6087f7827c57d282c35125d5697e5 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/ca154f18d4d2861026f2d59c92a09da25794f30913efc764108427a8e33da6170c11bf4c229cba4c3ae87e05efa6f18af3a1a5b5bcbe94e96b1d52e036c65d75 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/cf88d74b55eab615c04412c1ea582fdf +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/58bce04a429a7fc893567c2c3ca5954d92183043fe5f71b7180a8bbc480dfc355bfc1fb1d5e71cb9de7806c37ad472c6fe4c7eb341fb48c856b93ab4d3d5ef97 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/25c8b00a4452cd9d66ef53edbb747276 +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/25716e63d7c28600324ee68664f0457167638fa46d91a5e4a3fa5001d08fb816e0b3e0402ffa5f45b7e7f1efc014976094fa090fc4f10d71ca753da3f2a73a3d +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/de3025ca30ed8df01e8f358fd868c3ec +libLLVM.v20.1.2+1.powerpc64le-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/517dc881378677bdd6011b349d88fbc51cf6e64cc67542f8cc5e4382deb5d0c16177990ed67890a797fd078c298a615b0ecac94788a2bf27ba6ea353361d4e95 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/85defcd1027bf22359db1c8dbbfa5f66 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/0d3995021935454a11d27da18ad41c2c7b3e86745d9f978ef321a16a312e73d0545482e34e6cb10ed9dd255bf0d5686c240aa4ea795d56c8d55b8207bd5b66c9 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/7a30a754501c41f5efe28c0a1d5dfff5 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/26cb6342efdba3e53305330355bf266e40b1300879011215ecf2311e4fe516d65a4793aa209e7eff96354d59ea962a0b58fb1b9544559786e569fba0fb393cc3 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/92d7c4e18a1ee1c32d7f25361b63d828 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/dd64730725a3ed2bc26262b661c565fee9c2a4a8660109707c773437615cb92a148c1d196700b254d6d7950de0de7177af7802963674021da9ee2d0650de95c6 +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/fbf6174730f401f14bfccdf34c97518c +libLLVM.v20.1.2+1.riscv64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/f3d78b63241bb142787d1dead601bc2b5f06d5b9dc2b2dd78da015df6d50d9f9a7cb478db2084157eba65a57f3328e9f728ddbfa3b7d760eda1aab0f97cb916b +libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/md5/fb1c596f521fc005e3626cee940fc4bd +libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.asserts.tar.gz/sha512/7b4ec22c2e882644aa8377e10c8ee18fa2f880e2afd1f49548c5251a74636a9933af104ab0d1cf114b1ec474342c2c296b9e5eacb681aec202868929283aea92 +libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.tar.gz/md5/ace5ea53262082e1962cd1e6068fdfd1 +libLLVM.v20.1.2+1.x86_64-apple-darwin-llvm_version+20.tar.gz/sha512/2391e569a172e200ec9b1384c5e6f2f04807c8b47dcf4938243872e23867ad2b53057f511ccf9e3fd2817696af322e6285764b75e2e72cef9da06cee2650be25 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/md5/9e207cccaf077025faf4ffbfbe911b26 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.asserts.tar.gz/sha512/b1fd95de38aa698e519316ed14e3ad4aff6e0e11a8415edfd85b2bf4e2504915d76cffcd631b69b3c949f10ae087c4fe43ecc7b44a390646e764f44100e3039b +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/md5/5d221a750e203aa06b6e4a6d22730acf +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx03-llvm_version+20.tar.gz/sha512/71b18114084096a3199d866f05df8741049d1f536b0ef156d90f18194a0f78262f04e8ac68e7e5e7f95bc2aef34dd524d06ab5c1f41524ff2ddad17e7002c3b3 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/md5/c3ea41e66f7d0749e1f45b7e9d2e32ae +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.asserts.tar.gz/sha512/a85cfc811ff0081bf565e922d337c56544ee3f7bd10d8db51f5bc26c45928b725bd1f3a57760c868b90b0ed1a7b949afd1087331dce995cbec73f627b1fe8d87 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/md5/a739244415d4ea0f2c0ae3a96ce9b3c3 +libLLVM.v20.1.2+1.x86_64-linux-gnu-cxx11-llvm_version+20.tar.gz/sha512/8cd8d5985172dfb3558d1b5d1658b3a01dfd75327badb0dea8582d51bdeeb77db0af6890a5dfb8ae25f3611ddfd734666ac08347c2d2d63f81bed6e9796b364f +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/md5/2d69f3b14ce0fe480dd6b3a8c030a27d +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.asserts.tar.gz/sha512/99f4c9e6fb8114b61aaa642fd85b945dbf47328b2eaa60cf5845b3883c2917e5b5e239a8fac13005271fd018d0f4b23d51934f5894ce3c994bc185d8eebfca17 +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/md5/5096e6347bb08fd53b0416b4d4338405 +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx03-llvm_version+20.tar.gz/sha512/bcc0848e319846f930c8b89008030f1ad124cd2caa7cae11a63d33d4ccde78f044e65857a4e11f7e25ffc0e785af8c9e7083d289273d79181aee366c1f3754fb +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/md5/022d407dbd191ba036f73bd986b067be +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.asserts.tar.gz/sha512/0c793bb350eb1540c09a70743801ba1b76053f07f64a57d133a1158fcbb057c52a62bc2d08403c605aef30a4dfbd7268fbcc83e18af8406523b114f939f7e830 +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/md5/644e82b0e190856d95d6af1fd1bc84ea +libLLVM.v20.1.2+1.x86_64-linux-musl-cxx11-llvm_version+20.tar.gz/sha512/bf736aa4b882ead66b02afb504cb23ad7f77e8674d9ab9fdd769e62c7328b0f9112a360967d495a208e597f9a2f20f9cf436acf2706e6d09847d65638a3cb482 +libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/md5/e90086cb8edaa9a4066b58b0bb8e2636 +libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.asserts.tar.gz/sha512/38908f3054bee0fc7b9f6c257cdae59dc641cef19d48793faf76a027aeba774fcf9ed9d820766bdc31d3301f4ddd5135c3c328f9db9a39c613ce925db0a166be +libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.tar.gz/md5/98be4b9ddd1d044cd7ad6250b38fccad +libLLVM.v20.1.2+1.x86_64-unknown-freebsd-llvm_version+20.tar.gz/sha512/18e881e0d752ce0334d3c37e4cd0a41db4172e376410c860fb29e31d37ad5d95f9598f1bf0280c81fabf88129a17535c4f22e7f80f519c58e3da521923667fb6 +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/md5/281234060bf9f417fbae340cafd20309 +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.asserts.tar.gz/sha512/e9e1c01dc849473f8885c2534ecb1391186cf283076402be9ba05bb5eb8633734e9b1f03af39218eb7cc40f59b727d8feae3a82fc60d593c86b5909f316d7c8b +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/md5/aab22266d8f08325697cfe2b8083895b +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx03-llvm_version+20.tar.gz/sha512/295a4d82a4671023df12f8faef8593863a2ca970f59a1946605add0f80650a220fa75bb250b7d3df05070b23bd10f11daba4594fc0d5a16db09a2a7b737f4bb8 +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/md5/da928189b76c03126b556d7141bb4adb +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.asserts.tar.gz/sha512/fe0cd8f8791bfba761ba43374c9618c06813828421e9b2a0093bb132b6dabf41b732413e0d59a6e2a97e42733d9cef8293086ec1d33bdac031bb0d528f424aa9 +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/md5/4f4c5a6edaf01f8d6b98fde4e27e73ac +libLLVM.v20.1.2+1.x86_64-w64-mingw32-cxx11-llvm_version+20.tar.gz/sha512/0f83c130e14afcfd3e25b3f6200b0dc909d3f8ad498e8580c480121e7ee313c9c33972cabda9d6b25f548a1ec42e19f7df9300b1717dcae352ebb5b8004b05ea +llvm-julia-20.1.2-0.tar.gz/md5/aebf84a1b2ea2384d8f7d89ae6e06a29 +llvm-julia-20.1.2-0.tar.gz/sha512/0023c0ba21df6f130b543e58c4934a4061c4fdba9525867da3b7f1dd7a9aa2189eca90c8600d93958da5694780cb4aa2c05683d19817d8298305ec81d88cdc4a +llvm-project-19.1.4.tar.xz/md5/1e13043b18558e4346ea3769094c9737 +llvm-project-19.1.4.tar.xz/sha512/a586f8a41dde5e0d9ca6d8c58e9ef2a2e59b70a86d2e2c46106dc31b5c096bb80af0cdbdb486179e9cc676a540099f49a1c2db9e5e84c50362db1f72e9af6906 diff --git a/deps/checksums/mpfr b/deps/checksums/mpfr index 7b3b57978bd01..7f0de6099713c 100644 --- a/deps/checksums/mpfr +++ b/deps/checksums/mpfr @@ -1,38 +1,38 @@ -MPFR.v4.2.1+2.aarch64-apple-darwin.tar.gz/md5/1f5bba3e8e540720e239da75e5ae79eb -MPFR.v4.2.1+2.aarch64-apple-darwin.tar.gz/sha512/7de26c625e540a5b88e280ec2cb8712d4514732d80a0c6342d2b2cabc6bc17c05f6c614b8e38800c93a4af5438c554733d3fa2002ef70072dfb44c08d3f03d26 -MPFR.v4.2.1+2.aarch64-linux-gnu.tar.gz/md5/112ddd4e5cddf36b005394f9cd81b8e5 -MPFR.v4.2.1+2.aarch64-linux-gnu.tar.gz/sha512/dc125f625e8c74ce18c052ef759ccbcfc2f3a932f2810a306bdddf70d5f37f3546200690fd08fb76742022322a7c1b9aa907b4aec6edb318060f0648ff426cbc -MPFR.v4.2.1+2.aarch64-linux-musl.tar.gz/md5/a0919ef7cc35bb663d05e27da2bcb9a7 -MPFR.v4.2.1+2.aarch64-linux-musl.tar.gz/sha512/8acbaaca766c2ce225ac8df88c103a57fc52119d1fd54e9fc7d1f9d725c4ca9f74a0090e86eea0c140482a1abaf5b6086c453824a7516e9aef3ede5058f1767c -MPFR.v4.2.1+2.aarch64-unknown-freebsd.tar.gz/md5/61e1dcc7e323b976854a4e8164316d37 -MPFR.v4.2.1+2.aarch64-unknown-freebsd.tar.gz/sha512/f3a5493f88b290d15aff9bf79b15158d19bea05af7210b2967368e0b2f98cd291f77e62f39ee0c7ad4e9d2ef6ebdba4bf2fea24c723791f71f7b9b1ef989a67d -MPFR.v4.2.1+2.armv6l-linux-gnueabihf.tar.gz/md5/629aad4ac45ba23becd8a26df188638c -MPFR.v4.2.1+2.armv6l-linux-gnueabihf.tar.gz/sha512/bb05a8bf127eb16608a82037546f48462cb6168e1adcdb2c60dc3bd08f62cff30cf603abcab87bb336305d37dbb7b0480ea8f6664191879bdcd487738a33dd99 -MPFR.v4.2.1+2.armv6l-linux-musleabihf.tar.gz/md5/0c3c026051b096d98c8d476dd44db334 -MPFR.v4.2.1+2.armv6l-linux-musleabihf.tar.gz/sha512/9e791fe9748c87068c167517883cc905fe51ea38d2db89562a7a0959cfd83b268eed2897e5eaaf90c0b0b08a4efd8039bdeece64e83b17bf1d676570d13c2b98 -MPFR.v4.2.1+2.armv7l-linux-gnueabihf.tar.gz/md5/a2433a717e49ad95c3e430a538d01134 -MPFR.v4.2.1+2.armv7l-linux-gnueabihf.tar.gz/sha512/abde21a943d4af312e0d44b1ff1d4aefa10b2f38c74ff0e04c0c2b8561750ef5d164679564ffe1b551821d83ebcafbe99467230b37fe4591c593a24dfb070c6a -MPFR.v4.2.1+2.armv7l-linux-musleabihf.tar.gz/md5/4c892b4cbf1926d5d2b6a88330015c8f -MPFR.v4.2.1+2.armv7l-linux-musleabihf.tar.gz/sha512/24825bb1268ef2ea42894ec9ff6589308abae430dd8e43a2ca0d368f1e718fd3cdf6d9bc4bc383346970ba845d2ef1721c4848ee0c783d09addc5505131db3e6 -MPFR.v4.2.1+2.i686-linux-gnu.tar.gz/md5/0b1e0268dcaeb3aa0f7f0a6451c6b841 -MPFR.v4.2.1+2.i686-linux-gnu.tar.gz/sha512/f0ef142c7b86e8f92b78a7ff0607da70bf8f3970b118fa77438cbb0acbea604dc0c7566b52ff1f85b179aac7661b31e4aee049f2c5ff799c95b385ba9cde2a25 -MPFR.v4.2.1+2.i686-linux-musl.tar.gz/md5/2fc9a938e76e7bdc0b73d7e8bfc8b8ee -MPFR.v4.2.1+2.i686-linux-musl.tar.gz/sha512/4aed3884ad569b7695b9383db9d9dbb279ffe5349f7757b867ff860fa600b47faa4c169f4a60409666ce45fc6e6f269c18cef2df6fa0585f056d7e07e55005b8 -MPFR.v4.2.1+2.i686-w64-mingw32.tar.gz/md5/d13c44bb28d721107639c8555db5e157 -MPFR.v4.2.1+2.i686-w64-mingw32.tar.gz/sha512/1b5562d2df322c28bd06bb4ba8c9039cf90ed62affcf7f2b0d7ae8925d503c76a0d3d2f9b65c8c55575f245a4df8fbc4c7c63e93e7b973188f203a7fbda4eac5 -MPFR.v4.2.1+2.powerpc64le-linux-gnu.tar.gz/md5/52b3912b2c5f59ab3dcd7c3e06ca41b5 -MPFR.v4.2.1+2.powerpc64le-linux-gnu.tar.gz/sha512/533cf1f93c4464b4bed1d56ea79946fc2d20f3a7825d6b0383ed98cec99f85713e7bca549fd8948adb69aedc14e5d14a54238b3e67ef103e1b049b0cfb6cc1c9 -MPFR.v4.2.1+2.riscv64-linux-gnu.tar.gz/md5/aef7709c8457ee2db2622c39f1da16b7 -MPFR.v4.2.1+2.riscv64-linux-gnu.tar.gz/sha512/7a9c88563e3e7ab22a3aaa45690ed89c3e7eb22333a3d45c5e04ad2660c91ad2c97f10cd6c1aa1ccfdbf97186f9fd7f92330a41ec0be026e2ff84c5ba91f2652 -MPFR.v4.2.1+2.x86_64-apple-darwin.tar.gz/md5/12afc9778e39a5b6d9ea0161e2c80a95 -MPFR.v4.2.1+2.x86_64-apple-darwin.tar.gz/sha512/a9070423a898fa865740753ae7513d3cc0b500bd9b6b5c6aa672833dcac429efd806eff48501b51afcba5db0d31e79dac243b11b2f8847a1551576c6131506f5 -MPFR.v4.2.1+2.x86_64-linux-gnu.tar.gz/md5/46c6a5f40243795bdff51bd68a89c82e -MPFR.v4.2.1+2.x86_64-linux-gnu.tar.gz/sha512/df8209d69ae55dd54491055078f113f4ac8be7bc68e1c0eb62944e6c9c04ed3e9a55c4a5f28ec68eb69f558d9f4d1b975f36de572fbd0ef7720568efc8042327 -MPFR.v4.2.1+2.x86_64-linux-musl.tar.gz/md5/045236ee0d558d2eda42df76c3397f69 -MPFR.v4.2.1+2.x86_64-linux-musl.tar.gz/sha512/52b68a673160af7cd09b191f3c28e17d5af7516b5baa86c0df9cb63a116772a15b5358f3db5f0b254b5752c652f8959454667cc1726ea4ff30946e3bbdb90ab4 -MPFR.v4.2.1+2.x86_64-unknown-freebsd.tar.gz/md5/da3da71bc7572eca5bc3d3895abf73c2 -MPFR.v4.2.1+2.x86_64-unknown-freebsd.tar.gz/sha512/4270b83ebe72d431f8fd9127b2b8d3bd75c2e52c563d390a4ca8d40c0514f5996fce57746d07b7d3bcbf93bfe78d420f815fde5eda4d84a5bcb7b7cf0e092504 -MPFR.v4.2.1+2.x86_64-w64-mingw32.tar.gz/md5/2a6f5ccb8d45591a845ad43916beb85a -MPFR.v4.2.1+2.x86_64-w64-mingw32.tar.gz/sha512/db9ecc9d8247fe4421c4cc9c6ab540e17a7445056b7a1062d4e334b353783a1c067062fd8e6f0517d8bd8782c9bb75abcce8ab8247be707ba066dc90b7fc12ff -mpfr-4.2.1.tar.bz2/md5/7765afa036e4ce7fb0e02bce0fef894b -mpfr-4.2.1.tar.bz2/sha512/c81842532ecc663348deb7400d911ad71933d3b525a2f9e5adcd04265c9c0fdd1f22eca229f482703ac7f222ef209fc9e339dd1fa47d72ae57f7f70b2336a76f +MPFR.v4.2.2+0.aarch64-apple-darwin.tar.gz/md5/01a13215fd646c761e469f36f693fdc8 +MPFR.v4.2.2+0.aarch64-apple-darwin.tar.gz/sha512/da473776ac8c687ab34792235ee5e1e08dc6a2e29b73620bd6dac93db32397037ae502b8ac3a35e020f722dae7da007a060e5e11e3287c4cdb846bf7e5168297 +MPFR.v4.2.2+0.aarch64-linux-gnu.tar.gz/md5/58ca9f3e08a388c3e40692e623f3884e +MPFR.v4.2.2+0.aarch64-linux-gnu.tar.gz/sha512/c6846d982ce1211791b466ed6fed2aad9e5f9a4866c48db99eb288dcbb1480660772010869fdea66d6453c8c140c92e367cfe55f6087fe41ea040fbd77eafe34 +MPFR.v4.2.2+0.aarch64-linux-musl.tar.gz/md5/2ff7e1400f27d049e3274a6277322860 +MPFR.v4.2.2+0.aarch64-linux-musl.tar.gz/sha512/388f7050288be9d30c4a2e772c0859e414b0cf6dbc845eec0eb6aeda53595df94a4e3001d02fa04c173fcf74e00c2552a8880b62ebf5adf443da2a95497be891 +MPFR.v4.2.2+0.aarch64-unknown-freebsd.tar.gz/md5/d1e6c477ab9678d1cd1dfa7e00366e69 +MPFR.v4.2.2+0.aarch64-unknown-freebsd.tar.gz/sha512/897174756651d01272d86bb147f5dda9f84f8f1bf1fe02b8505e141df3cc38523019f85cbe538fcc6ea8073d7743fc6428a06271107b059de80cd8f959c52daa +MPFR.v4.2.2+0.armv6l-linux-gnueabihf.tar.gz/md5/5213b0ef1b191c529e3335e05b918003 +MPFR.v4.2.2+0.armv6l-linux-gnueabihf.tar.gz/sha512/bbcdb90f80d8cb826cd055eb41f051890c7847fc0887389b61bd24c051d35873af36672e5f1956cc3fb23b8e3ee50ee069c185fc2faabe302787d70210bd5b07 +MPFR.v4.2.2+0.armv6l-linux-musleabihf.tar.gz/md5/9a9d9207a6b52b6e84b1b2b1c631e0f2 +MPFR.v4.2.2+0.armv6l-linux-musleabihf.tar.gz/sha512/fd40d16a40b1db2b441339e5c8cb3f8a1810d2889713b0504f9bfd5451f4f4c2dd0ca35a4b2922feca9cf50e4a9b3bf8cf2c088655dd85a23c33ee67c12e0a72 +MPFR.v4.2.2+0.armv7l-linux-gnueabihf.tar.gz/md5/44532dd5607ced01a8ba0856c3bfdbc3 +MPFR.v4.2.2+0.armv7l-linux-gnueabihf.tar.gz/sha512/469fc030f458bd52f6bdffc442ceaaf8659f0f1e40d581eb1303fd4753d2c665fcb75bc6c54d04eb53d77b1945d67f48a5ea5614f2ee82cc7fd27e89859b45f4 +MPFR.v4.2.2+0.armv7l-linux-musleabihf.tar.gz/md5/fbd13b054b8d27be6bc836283f7846bf +MPFR.v4.2.2+0.armv7l-linux-musleabihf.tar.gz/sha512/926dc03f99a6827c833614d17c5ef4f80fb862bdf4397db9aaf8ae9b3a66e8b9121cfa044b18db46f5774abbd7e9c129363183ccb2ae3192084711e7ff9d6382 +MPFR.v4.2.2+0.i686-linux-gnu.tar.gz/md5/da6fbb90dc20830af9325cfaf3544e4c +MPFR.v4.2.2+0.i686-linux-gnu.tar.gz/sha512/d235884e1d1bef406b1e5ceb9c34aab68c1a8040b2022964105238ef8cdfd4af7aebe474fef80849689ca88d9168697fd55e8d6ab92b6641a1f37c431d5e3ff3 +MPFR.v4.2.2+0.i686-linux-musl.tar.gz/md5/fc885092e1469a06aaaaf24168e8fafe +MPFR.v4.2.2+0.i686-linux-musl.tar.gz/sha512/5307926e1222b302e48e2f5c08479b920279d15b95937a245e16ac1dfd5c6206cb64fe4b6ca4cb7d6be847d8cc01a04d2661a630b978dc2dbd60605d222b8b21 +MPFR.v4.2.2+0.i686-w64-mingw32.tar.gz/md5/55f129d5b5b849b3bc018e68ccf14914 +MPFR.v4.2.2+0.i686-w64-mingw32.tar.gz/sha512/9a24e4616e05f5c1fb53e7a12167f7a55d05ec1895124d6ee23b2efd548f49e4c7995c16d240ec803f352d586ae4667027ee0bdeefa520e0c1f581fcc338dc44 +MPFR.v4.2.2+0.powerpc64le-linux-gnu.tar.gz/md5/6f47e4cde45ddf0cb2ea4f31ef9c9e04 +MPFR.v4.2.2+0.powerpc64le-linux-gnu.tar.gz/sha512/4fd8fbe166e719c636e430d4d5c938231fa9126b29eacbc678d2eb50d3d4b95cf6ccef155ce401c6d33b9730c2f89c0c77ec8fb39254483c2e4004639c503c1c +MPFR.v4.2.2+0.riscv64-linux-gnu.tar.gz/md5/c4736705ff2a55cf8206c3af84bfc417 +MPFR.v4.2.2+0.riscv64-linux-gnu.tar.gz/sha512/e1e77d64ee88de2990fbc791d7307afe859cfbdc1ac67e7bdfa633627b5542ce2e3ee0cd9fe4036abfaf60509277a43f263e2155665ac2c5e38b8627e470f399 +MPFR.v4.2.2+0.x86_64-apple-darwin.tar.gz/md5/c3e983178a1e9600f42714d4cc1ecdf6 +MPFR.v4.2.2+0.x86_64-apple-darwin.tar.gz/sha512/c5c6cebcdfc5b7b84e9e217a81d99e5af78d163949745d570af5689210b3eedeb9de3c11991b1b36d8fdbee17b550a4072af951d19c3f863cf24cda7d9c12950 +MPFR.v4.2.2+0.x86_64-linux-gnu.tar.gz/md5/61fc7c7aa676d0a07e1709b433a8e423 +MPFR.v4.2.2+0.x86_64-linux-gnu.tar.gz/sha512/74bdefa72c51c82ca709e3494cd664a6593173bbfbe0198f18f4c0add06ce4c1217e4dd49e99cb151d71c85cd696ae2147aed29ed2cf3f1ca0e5b40582abb571 +MPFR.v4.2.2+0.x86_64-linux-musl.tar.gz/md5/207ee8ad2293ba36d3d7bb845ab346e0 +MPFR.v4.2.2+0.x86_64-linux-musl.tar.gz/sha512/63325e6595861a324f3c299d8c51b1d665197217c8fc9a5ae627b624037394f050bb08a9acd14e9809f982942c066f1185dded0fa493f360bcd3baae17a05f92 +MPFR.v4.2.2+0.x86_64-unknown-freebsd.tar.gz/md5/74e5a5ce0ea84959ccec7b7f7ab22c66 +MPFR.v4.2.2+0.x86_64-unknown-freebsd.tar.gz/sha512/411dbb339218669af6181fdf1e17f926abb9830ae54a8f9ef1b7df53021e8da01a41fda13067731afaf9b803324d5f82c060ef5b5b91045625188458b99dcc75 +MPFR.v4.2.2+0.x86_64-w64-mingw32.tar.gz/md5/2de84b494ea832147be4f9bfa786cd19 +MPFR.v4.2.2+0.x86_64-w64-mingw32.tar.gz/sha512/5f86aef6ab4fd7517cb23ad9a32ae21954a3ce1f27f5cbd28abe038271e20197b7c241055092a4aa6d5391f012bdee10465c58b53acd64bb5b99fd754c75ad29 +mpfr-4.2.2.tar.bz2/md5/afe8268360bc8702fbc8297d351c8b5e +mpfr-4.2.2.tar.bz2/sha512/0176e50808dcc07afbf5bc3e38bf9b7b21918e5f194aa0bfd860d99b00c470630aef149776c4be814a61c44269c3a5b9a4b0b1c0fcd4c9feb1459d8466452da8 diff --git a/deps/checksums/openssl b/deps/checksums/openssl index 134ad867cbd3f..cca21ccd8c5a5 100644 --- a/deps/checksums/openssl +++ b/deps/checksums/openssl @@ -1,38 +1,38 @@ -OpenSSL.v3.0.16+0.aarch64-apple-darwin.tar.gz/md5/72c29fd0048b0fee44410cfb82ed6235 -OpenSSL.v3.0.16+0.aarch64-apple-darwin.tar.gz/sha512/a1d23c15c16d577e7f350004c3e3e8c9f14375ca31256f4e42dcd828b610eeea269eecac58e1c3449c99dcc80b7a8885bbbd39beb5d5ff85167d953c26c10d00 -OpenSSL.v3.0.16+0.aarch64-linux-gnu.tar.gz/md5/29bf1097dbe8ac0c42ffb0c1ff9234cf -OpenSSL.v3.0.16+0.aarch64-linux-gnu.tar.gz/sha512/b388a13fbfd416feb95bac4f2f4f47605ac8ad971551ec218a8822618cd6e127ad538782524fed5c5f75ab3caf5b86107598967993ae03516706efa6a3af1010 -OpenSSL.v3.0.16+0.aarch64-linux-musl.tar.gz/md5/971ba30a9e0be025433afe3b0aae6260 -OpenSSL.v3.0.16+0.aarch64-linux-musl.tar.gz/sha512/43b6bea8d3e0ab783ed2ec1140fb9054ef0cdd0ddd34e5e95fb36c7b1b72d7e988b2bb17c878c57b0721c44b7783b2db9d4fc614a63bb557b1c32088dd01d506 -OpenSSL.v3.0.16+0.aarch64-unknown-freebsd.tar.gz/md5/3a3d963e16c7efbacdaea9754db640e3 -OpenSSL.v3.0.16+0.aarch64-unknown-freebsd.tar.gz/sha512/7c3d0ed3a7f37e879e3b8f4c5c67cf2766b5e421fab273806a0412ba12ab4de421bce094713aeb3f6c3915260cb8d7fcb35e214344131e9a5b0081cb7bf0d5dc -OpenSSL.v3.0.16+0.armv6l-linux-gnueabihf.tar.gz/md5/c94d4882e57cb9d3688127f5f82331a5 -OpenSSL.v3.0.16+0.armv6l-linux-gnueabihf.tar.gz/sha512/b8133e873a960b125d0ab8ddd5b4071a6ab269e2b2f5b36e0d723e759875d21f734ac44f4baa4122bc6b94e23f0d82d401f16c88d9c70dac4031f07dcd20597a -OpenSSL.v3.0.16+0.armv6l-linux-musleabihf.tar.gz/md5/149aacf601e86ed15cd4305d035fb7b2 -OpenSSL.v3.0.16+0.armv6l-linux-musleabihf.tar.gz/sha512/dad7bade8fdf62c642ba1a8553f8a02218dbddd15040388b0a0eaac5391fb3c606860378c5dc40b16978c2f8f3837a1698788788d83006a4b312b1c6e73b2a53 -OpenSSL.v3.0.16+0.armv7l-linux-gnueabihf.tar.gz/md5/a1d45f34e463df42c2ea77d9084a4360 -OpenSSL.v3.0.16+0.armv7l-linux-gnueabihf.tar.gz/sha512/5f5ba285564b1ff1c5db3a2fe4d2051b9b17a77e6c6da37bc739f64051d64a5bff3968d5326760c102f9ddbc3509bf3eeb3ae267acfecbd0a55ff86ec90b5cb0 -OpenSSL.v3.0.16+0.armv7l-linux-musleabihf.tar.gz/md5/81ed6b1188a7ccda1768b67fdfc6e094 -OpenSSL.v3.0.16+0.armv7l-linux-musleabihf.tar.gz/sha512/2424620b462b5596e317e77ebc294377811ec1210c65baf4cd072b62f8d303aa77b2b4f799611ca91202dc371ea4575c2c13cb222e6edd809a036b639c1595c0 -OpenSSL.v3.0.16+0.i686-linux-gnu.tar.gz/md5/714bab6e53849d0bf6d9922be51871e2 -OpenSSL.v3.0.16+0.i686-linux-gnu.tar.gz/sha512/141102d20810986b75ba3b2b4446e4a260f4ae38583b7f8dd8a59b8e0ec8b2bac03ee83127b8f4cdb67bd8fd5ebbb102cb581ed182735283109ff85d049cc55b -OpenSSL.v3.0.16+0.i686-linux-musl.tar.gz/md5/e4f5f918c011d87626a0f830243324f9 -OpenSSL.v3.0.16+0.i686-linux-musl.tar.gz/sha512/128e6c29f0537818a68cbbd262a3735ee99ccca2377874c22978782abd43c3d0f9bd49d68c05d09f37293df52c238aa33d87244276eef080959aefe42fe8c92f -OpenSSL.v3.0.16+0.i686-w64-mingw32.tar.gz/md5/a68b80b7725887ef33ea36e7d19f7cd1 -OpenSSL.v3.0.16+0.i686-w64-mingw32.tar.gz/sha512/91aeb66c49f73eaa00114a61fc2b644e278bc39948f408806c66f61529ad98d3bc1f885152e1cc275a8374dde2d5ace3695e6f70eec718bacb0269560bce83b0 -OpenSSL.v3.0.16+0.powerpc64le-linux-gnu.tar.gz/md5/ec36f9b42c64ab4b1ce862229bc06924 -OpenSSL.v3.0.16+0.powerpc64le-linux-gnu.tar.gz/sha512/71470814b096ca9127fc2055b4ecc6e6acb30f03fe16754010c5c860f8d82cb37c2e723dca3f92f2aa2e9604fd1f4d141eca333d2c7524274e33d98da34326df -OpenSSL.v3.0.16+0.riscv64-linux-gnu.tar.gz/md5/5aecf142b6849b8c2cca957830d49f4e -OpenSSL.v3.0.16+0.riscv64-linux-gnu.tar.gz/sha512/d25879a4d6b8cc76c8fc4ed49b38d48938c80ba163e7ffe276bb86fdc4bb76cd596f150564bfa3bd88fa90451916a086ca04d31ba884bd978f40f57f0e4332f7 -OpenSSL.v3.0.16+0.x86_64-apple-darwin.tar.gz/md5/41c847ee490a5935fac8c4b663d8e325 -OpenSSL.v3.0.16+0.x86_64-apple-darwin.tar.gz/sha512/04979622fae5b24b0f0d79b0853ea311e872a7ca465c6e6700e713a34fa90a182ad78db8babacf322fb288cb62caed1a73f8d47e55fd2b074238191649e0141f -OpenSSL.v3.0.16+0.x86_64-linux-gnu.tar.gz/md5/b3c143deff5a311740ccc592a1a433e9 -OpenSSL.v3.0.16+0.x86_64-linux-gnu.tar.gz/sha512/31819e2e78dbd7aeb95b84664eac9ee29b0ec4e1b0bc9e06a4ea8f7cb65c21929414d9e4e3fa6ce15944bfd7fc68d699d4016cbe5ace16e98394a60fe369541f -OpenSSL.v3.0.16+0.x86_64-linux-musl.tar.gz/md5/bed866803232e6ada4e22eadfd2b98d1 -OpenSSL.v3.0.16+0.x86_64-linux-musl.tar.gz/sha512/ee6f26d150c81e30c93f08de5428d2f92e02e12565e6dcef09bbd1029ff5477bb2176b6cc7616a7cc898dfec8c5d8263108c3558460397c71ca90af8b230e0c1 -OpenSSL.v3.0.16+0.x86_64-unknown-freebsd.tar.gz/md5/ee6f11020b0ce6eac8016877d1635a04 -OpenSSL.v3.0.16+0.x86_64-unknown-freebsd.tar.gz/sha512/dd154fe1e8c537d42919c0889036ac79e181b765c87130edfeffd2ef8414da902995469167812664e6f77f3c89379c799d4311336f4233b05796b6823838e01c -OpenSSL.v3.0.16+0.x86_64-w64-mingw32.tar.gz/md5/d08f7d27c775eec9c7e4709d0898d60e -OpenSSL.v3.0.16+0.x86_64-w64-mingw32.tar.gz/sha512/4806c67ff8f629ee6b3a7c358146675310f0812772773aad7e30d0ccee0cc085741c06d252ed16bf8f874fe794d1b8291e0cbcd65c8eacb9e87c0a5f65742c5f -openssl-3.0.16.tar.gz/md5/7b6a9cded21b9fa51877444f5defebd4 -openssl-3.0.16.tar.gz/sha512/5eea2b0c60d870549fc2b8755f1220a57f870d95fbc8d5cc5abb9589f212d10945f355c3e88ff48540a7ee1c4db774b936023ca33d7c799ea82d91eef9c1c16d +OpenSSL.v3.5.1+0.aarch64-apple-darwin.tar.gz/md5/b19522093c25c50685002ad48933a835 +OpenSSL.v3.5.1+0.aarch64-apple-darwin.tar.gz/sha512/afa6363f8396deac5131f5efbe92d5b60f4d6982d279d63b5847e80ac4717d89e32edcc9bc7a5fbaab95e03908a6e3e9b386a3931effb0a7163b947b38ed2cd5 +OpenSSL.v3.5.1+0.aarch64-linux-gnu.tar.gz/md5/60af2cb22b7d5f4fddd94bd196f86ad2 +OpenSSL.v3.5.1+0.aarch64-linux-gnu.tar.gz/sha512/3d384f5da4be3af848b47f48f2438dbda8cdb228b8569d01bd4fbd6feea9f494ecafd3cab6e7b0bbed596746aa2614826971133a2b6dea02836c0904ce870760 +OpenSSL.v3.5.1+0.aarch64-linux-musl.tar.gz/md5/5f96641ec5256a547e03cd6028892a50 +OpenSSL.v3.5.1+0.aarch64-linux-musl.tar.gz/sha512/668c08f2a08f9d65b2e5c1ca4db8f74932995d0fa97c4737a2d9cedb3548482f85fddd478fad37325e2d48f76259fd8f7e003d31fc2a9ecfdb88c4748f90e1d6 +OpenSSL.v3.5.1+0.aarch64-unknown-freebsd.tar.gz/md5/377bd17ae448f4394b3100b290602f35 +OpenSSL.v3.5.1+0.aarch64-unknown-freebsd.tar.gz/sha512/f989a15062b47f059086d4dc8fd53af00717ca622ef8c475a11f6e62a29d8ec4a80159d93a683e8729da66c4bda4c46b7647754dc996ed2ff5635cbbdaf702aa +OpenSSL.v3.5.1+0.armv6l-linux-gnueabihf.tar.gz/md5/83bcc0b545bea092a0a5de9e64cbcbf1 +OpenSSL.v3.5.1+0.armv6l-linux-gnueabihf.tar.gz/sha512/c589119945ff6c1341bc229a2e61096c630288f7d483ea9538202915f8ee1a9e26cd53efc479f1e92a83a75aa6c7453ceba446832678ffed340a4bec13fefbfc +OpenSSL.v3.5.1+0.armv6l-linux-musleabihf.tar.gz/md5/3b2e34e506960259dbb40a36fed26ffe +OpenSSL.v3.5.1+0.armv6l-linux-musleabihf.tar.gz/sha512/735f22fe1202818f64f369471292bb8fdf8cf1f3395d01e70ddf8f65efc5430aec54a63fe950e52625f2c8a5dbd59ed0175f088144af8d99c7db1967ed0e5aeb +OpenSSL.v3.5.1+0.armv7l-linux-gnueabihf.tar.gz/md5/aedb37bde1b3fad428987745dc1dd095 +OpenSSL.v3.5.1+0.armv7l-linux-gnueabihf.tar.gz/sha512/1919823df3c0de01c59a5f9cf42b912a1d56fe7de903c4e7cbcd54603760783a99fe34cd1c108e954d5fe41502c1791b803d67742d70abae64d316c3656b7280 +OpenSSL.v3.5.1+0.armv7l-linux-musleabihf.tar.gz/md5/573a752ca28fd62644208a4c0b32eaa4 +OpenSSL.v3.5.1+0.armv7l-linux-musleabihf.tar.gz/sha512/3385b170973a9e50919981e66e591077479ae7561368941f018aca6f42c86b3d01aa1d9896d4d5f6deb69760fa42f535f6aaa076b75a15f207328ba6f0a32660 +OpenSSL.v3.5.1+0.i686-linux-gnu.tar.gz/md5/fcdb2ab108900c412abf154a6cbd46e7 +OpenSSL.v3.5.1+0.i686-linux-gnu.tar.gz/sha512/b558e6c23809f702a7388dba7031a9df577e1a2eb1ca86b7cf0dcd9809973dff1c9b56d4a09c315b17dcc9860e7f89604513a2d022117d9145f2bc81befa094b +OpenSSL.v3.5.1+0.i686-linux-musl.tar.gz/md5/0192e44a52d9518d712db58019ace62c +OpenSSL.v3.5.1+0.i686-linux-musl.tar.gz/sha512/fe9740850e6eb32eb539d16303b39d9ad1d3e8cc2e5a742304013890a0e1e8af93e131a5393c3c817b5723680425947d6954455dd715cc83589fd097c517b5c2 +OpenSSL.v3.5.1+0.i686-w64-mingw32.tar.gz/md5/51b5546301f8c474bcc9c97b625df2c1 +OpenSSL.v3.5.1+0.i686-w64-mingw32.tar.gz/sha512/47874ce005e6944f3a4d482f3edf44bcaa3724338230d68fff22c484c0620fe857a11bdc515ef9154522a2884f64bacadfd1fddb1430a45c7722a6a4799107f6 +OpenSSL.v3.5.1+0.powerpc64le-linux-gnu.tar.gz/md5/1aeaa0660006b4b8c13cd1cb45b2acfc +OpenSSL.v3.5.1+0.powerpc64le-linux-gnu.tar.gz/sha512/dff025feb0d1ae96a7c33f1beff5e6f76d5a85833314470f59d75bf791e90363145ae79f3ed82c5c40e36416b75fa9deb5807911c8133fe11f31b4705737f0bc +OpenSSL.v3.5.1+0.riscv64-linux-gnu.tar.gz/md5/160065eb12650c504fd40a25e4bae2ba +OpenSSL.v3.5.1+0.riscv64-linux-gnu.tar.gz/sha512/68951cf98c4eb778d372e239d14497405e6161461a623135a5254c3fd65bc3a12fe3df1ecce88909cb05dc29104b5b18caafea115799c5abf2505afe75be3207 +OpenSSL.v3.5.1+0.x86_64-apple-darwin.tar.gz/md5/7e5903d1d051de70a93a9b801ce274db +OpenSSL.v3.5.1+0.x86_64-apple-darwin.tar.gz/sha512/729b33cc208b8646394bcf0029a276ad41cf2e9d44737dbc1e15dca794cc55a52e2b782b0c72ef57b5580b84a93b25133788424f1b16ef2b784d409bca598150 +OpenSSL.v3.5.1+0.x86_64-linux-gnu.tar.gz/md5/9cee745524f41dc21af2f460ac2f1293 +OpenSSL.v3.5.1+0.x86_64-linux-gnu.tar.gz/sha512/6949c7f19b7919073542af903019ec0d8fd5172450141a3f69f0c325f0c5cc19618f1959b96380719538c5a1a5a388165a0db8e6eab041d0a5874a627820212b +OpenSSL.v3.5.1+0.x86_64-linux-musl.tar.gz/md5/e55565c84e5cff597ea490e02c559d1a +OpenSSL.v3.5.1+0.x86_64-linux-musl.tar.gz/sha512/c27930401c72b6be94ba7717f7b3be0025b09138e146f3d2a761e805308ee51f4ca871459941448e102f64b0e3c1aa479f39ee77f3234321890fa7105418ed44 +OpenSSL.v3.5.1+0.x86_64-unknown-freebsd.tar.gz/md5/276d97e2d573977727ca8d2113335fac +OpenSSL.v3.5.1+0.x86_64-unknown-freebsd.tar.gz/sha512/71f72a82c590542928660f004f38f84ea335b303e7da53578d712994fff9e84a3b69fc836c08d03dd8310d17c9edc63e5f6975e6d26e67124124763633ab1b59 +OpenSSL.v3.5.1+0.x86_64-w64-mingw32.tar.gz/md5/cebdbbf8a8a301e332d75c46dcdb1af0 +OpenSSL.v3.5.1+0.x86_64-w64-mingw32.tar.gz/sha512/0e141d7317ac8f5c43d1ecc9d161b00345f99145af785d7554f750b5787ea69969f785e7a1305059137271d26450835c25dd126bf9e5aef2cdf7dcbbdebb6911 +openssl-3.5.1.tar.gz/md5/562a4e8d14ee5272f677a754b9c1ca5c +openssl-3.5.1.tar.gz/sha512/0fa152ae59ab5ea066319de039dfb1d24cbb247172d7512feb5dd920db3740f219d76b0195ea562f84fe5eae36c23772302eddfbb3509df13761452b4dafb9d3 diff --git a/deps/checksums/pcre b/deps/checksums/pcre index 9e290c914baec..dd22c2b91576c 100644 --- a/deps/checksums/pcre +++ b/deps/checksums/pcre @@ -1,38 +1,38 @@ -PCRE2.v10.44.0+1.aarch64-apple-darwin.tar.gz/md5/14de26cfc0f6ff7635fac39e81e81a27 -PCRE2.v10.44.0+1.aarch64-apple-darwin.tar.gz/sha512/45079ecca5f4966a32895fcc63585f1dd60f306dc1cb5c098d42452fcff67f7f6b405c200a15747af4680151bb6a6374832a0119b8ddd743d2ed13d0beaef7c9 -PCRE2.v10.44.0+1.aarch64-linux-gnu.tar.gz/md5/3cf179ed36d37bff698ab81cf3d5797b -PCRE2.v10.44.0+1.aarch64-linux-gnu.tar.gz/sha512/db93e5a5c0c46b5536ed49515682d9bfe1d23f6ba8ae2468289ec8f2160140f39f5606a3c7095f45251f3663d8ccf2d6d7e5e8b1efb21c39bbf9a13b6ec60ef9 -PCRE2.v10.44.0+1.aarch64-linux-musl.tar.gz/md5/02baa415218f581a5ceeb7bf7fc0a090 -PCRE2.v10.44.0+1.aarch64-linux-musl.tar.gz/sha512/1685f37ed8f465ecc2f738fdf65d20bb1806934ff2c50194882282fb6c3900121c61c39210e4c0b89847493bfc3e15bb7b9136b0d968103b47c8662a78b412fe -PCRE2.v10.44.0+1.aarch64-unknown-freebsd.tar.gz/md5/4de065ea59ab4f622b46079df1d9d941 -PCRE2.v10.44.0+1.aarch64-unknown-freebsd.tar.gz/sha512/aa6df9edfb690d155a8b5a9390db7ca11622ac0020174cf070a33a075801bfe43bd4c80b8e28017989a8b7374d39897cdcf72ab0e1962e3e234239975f7ac0b4 -PCRE2.v10.44.0+1.armv6l-linux-gnueabihf.tar.gz/md5/f8a0907fbb20a06507fce849db098c4f -PCRE2.v10.44.0+1.armv6l-linux-gnueabihf.tar.gz/sha512/3f5bcc1742380a31683a81740d55e198d7ec8d8ea5a13d6d0556d6603e4fadbf0dc648093c44e36dd6d3793c52a5e3dae6f2f459c73e3d3b5a005f3395d26772 -PCRE2.v10.44.0+1.armv6l-linux-musleabihf.tar.gz/md5/8854c24183441aa6fd21989c00888904 -PCRE2.v10.44.0+1.armv6l-linux-musleabihf.tar.gz/sha512/a74d9378f071dc4cb021e5171d66cd4ac5de3b348e993fc90d824ce5d2f554f7c8af7af55ec31d874d302aaba7d542b6505cc5963e53656c28026a06a53ed48b -PCRE2.v10.44.0+1.armv7l-linux-gnueabihf.tar.gz/md5/04960309ee7cf69a53e280878d5880ef -PCRE2.v10.44.0+1.armv7l-linux-gnueabihf.tar.gz/sha512/a1644daf036daa3799368598427c87c23bcfdddac55a0d06adca08a2e9d617c893285855af562101b05129d0ed0d84d22f5a8a1703316ecd09aa1752b8330eef -PCRE2.v10.44.0+1.armv7l-linux-musleabihf.tar.gz/md5/1335defc6090be76c509840633f7cdfb -PCRE2.v10.44.0+1.armv7l-linux-musleabihf.tar.gz/sha512/9595052eeae4da413b930b14d7e89359a29220cd9e908325e0b7788c8f4a2feb2134e78a0d8f56007787f0fefadc9de31750db6104bbdd048fa50e1d785c2a8c -PCRE2.v10.44.0+1.i686-linux-gnu.tar.gz/md5/e2d6be1d19566c965c2afeb995aba52f -PCRE2.v10.44.0+1.i686-linux-gnu.tar.gz/sha512/4a9d981bb6aa9150b670db7c5d4d188c8391fcb2a16bc710ede7a84bf7ec546fc5fd9096a339720579d25b6dcb5674b2b5b28e9664e5ef589b1a5044ce38b6a7 -PCRE2.v10.44.0+1.i686-linux-musl.tar.gz/md5/23cf857bd3daea4f094fcec48a7712dc -PCRE2.v10.44.0+1.i686-linux-musl.tar.gz/sha512/534f0cfab0cd60db9498eff387f7280a8baaf893a98dd2e7a737e68ba6473ed8236e9da85116eefb9812ec5323c705a00fcaff010b1900f752de8bdff65ef3ad -PCRE2.v10.44.0+1.i686-w64-mingw32.tar.gz/md5/3d05764df2305f16e4ffab60031ad40c -PCRE2.v10.44.0+1.i686-w64-mingw32.tar.gz/sha512/3e21cc6b71849c1a361373de30567990dba13dfd8812e7a7b5e2734b572bf1d45aeb730289d329975e76932c4c40e476824be2ab8e80a40fb7a7e2f46159235a -PCRE2.v10.44.0+1.powerpc64le-linux-gnu.tar.gz/md5/596d7c29d1417ed8959ea3ae3b4df453 -PCRE2.v10.44.0+1.powerpc64le-linux-gnu.tar.gz/sha512/89e03bfd6890150e2c8dddc4e7d024f2e09421c25a3d0fef3b5cd7f6bab7d6402ec1e82b02ecb5d26d01dfa2fb6068d050513894c374b7f2244c8fcbf00d69e2 -PCRE2.v10.44.0+1.riscv64-linux-gnu.tar.gz/md5/8330a431f4da1d20cffdb64d2c270dfb -PCRE2.v10.44.0+1.riscv64-linux-gnu.tar.gz/sha512/a836d0b9feefd9ffd50cf29db72ab704e6ae442939322526e2a5613973eabc8e543c5546ce507b0c5f9e6f1ce324978aeb6e99f8833eb60fc90e74139e47c6d2 -PCRE2.v10.44.0+1.x86_64-apple-darwin.tar.gz/md5/18f13c78ff6388c601bd36788e526b31 -PCRE2.v10.44.0+1.x86_64-apple-darwin.tar.gz/sha512/7b43a289f54064fc3c292de98173ec91cde2e49402c99c7848cbdc0e6d90a23a86d41f521e3986fcc8d941ee070d09e29ddc89a4e23009b8e9333e577ae4a09c -PCRE2.v10.44.0+1.x86_64-linux-gnu.tar.gz/md5/9f45feca0955f81ceb898208b9c74e15 -PCRE2.v10.44.0+1.x86_64-linux-gnu.tar.gz/sha512/eac215838306f7b5adb2166c3f620a69ed52fbd752ef3673a887507963a826c305d9b078dbb5236dc9a45eaca0d34f77325aab41703745701a077c84822ec0d0 -PCRE2.v10.44.0+1.x86_64-linux-musl.tar.gz/md5/79f092c6e8e971027ac6c1f0987376fb -PCRE2.v10.44.0+1.x86_64-linux-musl.tar.gz/sha512/2c5655b0f719a7d442c89f1040f2973b03f8becd855a0cfd6c0a985a07b25de351a84e3b9daaebd952b62628db0d937de08a8d05ee4bcace7e72d6b5ce6b8435 -PCRE2.v10.44.0+1.x86_64-unknown-freebsd.tar.gz/md5/a0bc32a099a584d453458a76c892fe47 -PCRE2.v10.44.0+1.x86_64-unknown-freebsd.tar.gz/sha512/6649c1b9e9569a9decccf6ebaa61d44acdb9069208ec796777d8e70a908210f775be2142053f6a5762ebaa321e297f6d8b51db99629766bc702c498b5f772492 -PCRE2.v10.44.0+1.x86_64-w64-mingw32.tar.gz/md5/eeffb6164fba08b0d5c7f50afa081475 -PCRE2.v10.44.0+1.x86_64-w64-mingw32.tar.gz/sha512/f06db992a2070a88559c15224972aeb098d4291a4325970fc0fbbb7cdd539f4a2fd4f90c0de90a34fe454da6c38290f9e0c7fdf2fe8c441f687fe4491d652adc -pcre2-10.44.tar.bz2/md5/9d1fe11e2e919c7b395e3e8f0a5c3eec -pcre2-10.44.tar.bz2/sha512/ee91cc10a2962bc7818b03d368df3dd31f42ea9a7260ae51483ea8cd331b7431e36e63256b0adc213cc6d6741e7c90414fd420622308c0ae3fcb5dd878591be2 +PCRE2.v10.45.0+0.aarch64-apple-darwin.tar.gz/md5/896991347ac3198a2cc84a1c39b8bf2e +PCRE2.v10.45.0+0.aarch64-apple-darwin.tar.gz/sha512/92ad56e63f7efe6511092826a994cb5d1ac3cfb20f86db557042bb734df8786d9f2f592c7cfea695955494696274916bea5af90116fd549ece49dd142d2de5f5 +PCRE2.v10.45.0+0.aarch64-linux-gnu.tar.gz/md5/dceb9622c7845494635226627c5d9809 +PCRE2.v10.45.0+0.aarch64-linux-gnu.tar.gz/sha512/d2688bf19ccc7f0567e9d9926be1f95b3e5aa8c85800fc40aae6feda81eda4e59bbff03f8a8f08cd21ac14956c44e7b6077ef5b8eba61fb0fa3541144ba23f47 +PCRE2.v10.45.0+0.aarch64-linux-musl.tar.gz/md5/5caf1b5a65e01ee25272b0125168c6aa +PCRE2.v10.45.0+0.aarch64-linux-musl.tar.gz/sha512/e0bf9422cce467373df64cba82d411768a2bb7e1b03b28c16267f49a56cf259450b34a47ecfa18051ba12a6b1bf2126a28df18508c25e0e09a35040502bb95c8 +PCRE2.v10.45.0+0.aarch64-unknown-freebsd.tar.gz/md5/5e1fc216d3051103911b7958aeb9ca8f +PCRE2.v10.45.0+0.aarch64-unknown-freebsd.tar.gz/sha512/de892eae2c3f815f676cc05771a54e84c2f6fc9b8cf6994b6953ee19eec7160950c80a6a3ff2d9974c4d42a23fccd0d24cd0fc569c87263f3b9dd05e86a66527 +PCRE2.v10.45.0+0.armv6l-linux-gnueabihf.tar.gz/md5/d00c7585e21c9c35288c9ad1ab86f6db +PCRE2.v10.45.0+0.armv6l-linux-gnueabihf.tar.gz/sha512/4fba211d5cb526fc313d62b6eeaa49d5daa471f507a4649cb6331686b762c951217e54017ebc745214811ca7a26a621e850d0092f9e88f1bbf9e1dbb451776c9 +PCRE2.v10.45.0+0.armv6l-linux-musleabihf.tar.gz/md5/67e2cf530ea4f54065e8218da5735de5 +PCRE2.v10.45.0+0.armv6l-linux-musleabihf.tar.gz/sha512/5097fcf75be76e62f95f066c65a2e87f33bf8897cde9951889349dd2f0b62b609dafbe5f4b0039d3acfb79eb5a18606bc5f10e591ff0d8520c37e32b487bf7cf +PCRE2.v10.45.0+0.armv7l-linux-gnueabihf.tar.gz/md5/87b587f70b5fea9a1623f836d810c73c +PCRE2.v10.45.0+0.armv7l-linux-gnueabihf.tar.gz/sha512/07f00365f81d6e27dc3aab0ee9f5c47ec612a09e271baa9f703e40d6a5394e631c01cc171ced248f24dcf561d9ee95e54d879d4458db9809755887a799e0b97b +PCRE2.v10.45.0+0.armv7l-linux-musleabihf.tar.gz/md5/efdb80dd89b059f50e425e5788873422 +PCRE2.v10.45.0+0.armv7l-linux-musleabihf.tar.gz/sha512/9fcd05831e9581b8ef5c85bbf13d90ec11268a60b474237b3679c71d804176d7816750aa65b1d02652ab599814f61f3023fa4c57e4e0bc29a9667b928f906b0a +PCRE2.v10.45.0+0.i686-linux-gnu.tar.gz/md5/6d79e55c8e3461400eee5a280d1a7bf2 +PCRE2.v10.45.0+0.i686-linux-gnu.tar.gz/sha512/a500f6fbf3952497539b6dd1d3c01fe4ca713eec838de3162854183b559100077f43537021224121abbdd72836dd0ed599a7ef7d2728848bf32c5af344d8deaf +PCRE2.v10.45.0+0.i686-linux-musl.tar.gz/md5/f820b790aba3c4dcbf0dd43c3570f94f +PCRE2.v10.45.0+0.i686-linux-musl.tar.gz/sha512/b9daa0df01701b1c4b97bdeaab113226ee085f4af49298ca93f4bab29e94781c2cc10442342a0205f52f089937efe01900f4ed75360cb8174f70aeec48e864e9 +PCRE2.v10.45.0+0.i686-w64-mingw32.tar.gz/md5/fd5bd53baab8ce75026722fc88a5f804 +PCRE2.v10.45.0+0.i686-w64-mingw32.tar.gz/sha512/117bf87ced28c2c85e5a90e83af14bd75a314bcb0628a71a86d7103a8a098f43be168b6093bd6c0439eeec940ff7327e20a5cfcc57060b9a6383090b03e947b5 +PCRE2.v10.45.0+0.powerpc64le-linux-gnu.tar.gz/md5/b6d67f0a9710b6531ac5c5aa32559a24 +PCRE2.v10.45.0+0.powerpc64le-linux-gnu.tar.gz/sha512/a287b1ced53d8fac0f6970d140dcbac7677cde241382f4d33617d4b0370ab5b65d498218962891a46b6a9e80ed7b2bd361675d263b4606fdac336606cc80d36f +PCRE2.v10.45.0+0.riscv64-linux-gnu.tar.gz/md5/bd06447be24fe3b9eaa8258664684d6d +PCRE2.v10.45.0+0.riscv64-linux-gnu.tar.gz/sha512/a8c5aa1011daf811c9431122a7221e1ed98e1a593861f21f8ce06593428d6b024633b7fc1b31c89b06a69f03e0da7c99ea8c6abc4ab8c9c47ffef786fcab0de4 +PCRE2.v10.45.0+0.x86_64-apple-darwin.tar.gz/md5/f4cafdaaa4cbb224d35337fce37a5fa0 +PCRE2.v10.45.0+0.x86_64-apple-darwin.tar.gz/sha512/6f891e3fbcc0a3e9c906d645cf98cc15eb224addb31d1a31770e41aa5fd1d84b8979f8f8f76c9ed9b2a34e97e137f5404a7d3d49a1750c1e633cf9f03415b580 +PCRE2.v10.45.0+0.x86_64-linux-gnu.tar.gz/md5/26a732774abff1d4c301640f513f4e8c +PCRE2.v10.45.0+0.x86_64-linux-gnu.tar.gz/sha512/ee89bfedd7a79dbe85c83460bb0b3834606af3f950d26223676733417ee4028d9b5a9fe3f2851f42c55b3a4c800ee48e25afdf7ae7bdbe7df021c38cffa7c7fc +PCRE2.v10.45.0+0.x86_64-linux-musl.tar.gz/md5/e4b0e47adc80128317697edab7312130 +PCRE2.v10.45.0+0.x86_64-linux-musl.tar.gz/sha512/521cfe714c1ff1604e86b24935060b12fbb3ab62af9b0c59121f0c26627bc24f5d612f4758739e068f3f7974c3890043c55c6f28e500c9de6e5550705c45c982 +PCRE2.v10.45.0+0.x86_64-unknown-freebsd.tar.gz/md5/c6655a1f349c51ac898878836b72fa1a +PCRE2.v10.45.0+0.x86_64-unknown-freebsd.tar.gz/sha512/69b48b9b6872f0c6b61a8a3ae81ba103fb49887c392497e5a387de888b35fc45c716b99c972670ee5edf0255db361051f265e30ea8f509b790ab9edd50bcb6db +PCRE2.v10.45.0+0.x86_64-w64-mingw32.tar.gz/md5/cb5c7a9a56e1f2209b2e227c598336c4 +PCRE2.v10.45.0+0.x86_64-w64-mingw32.tar.gz/sha512/ff5a982d90089316ccc1e6a9dc2c17cc61b348df0db16aaa4bb7cf1787701acf31a65ffcf51be48abc8cb4436afe94984086fead84580b3de3509d111ba71992 +pcre2-10.45.tar.bz2/md5/f71abbe1b5adf25cd9af5d26ef223b66 +pcre2-10.45.tar.bz2/sha512/4c1f0cf793624516d7eeb15745d6c07c9f678dd2c2b349062c6b614e88bf42262972d133576e85140dee2a882984aaf2d688953fc9c69ec7105b2daaeae89845 diff --git a/deps/checksums/unwind b/deps/checksums/unwind index 5d4967cb0cf22..c7dbc28dd3976 100644 --- a/deps/checksums/unwind +++ b/deps/checksums/unwind @@ -1,28 +1,30 @@ -LibUnwind.v1.8.1+2.aarch64-linux-gnu.tar.gz/md5/de3690f3a8ecf0aa5d2525813bdab3c8 -LibUnwind.v1.8.1+2.aarch64-linux-gnu.tar.gz/sha512/366090b4291623603e54d3c73437efcbc3c7f52ce0c64a63e8439eff8a3ddeb4efc1ab6b2513e0a60e2714239bf259cd667159a24207f0c9ce3134530e539155 -LibUnwind.v1.8.1+2.aarch64-linux-musl.tar.gz/md5/e8adf4e842e998b6806653964e721a47 -LibUnwind.v1.8.1+2.aarch64-linux-musl.tar.gz/sha512/77411646767f5f13e2f45d32bfa48d6864b712d46d339e3fd4d62d12f4a26b6ffb8293636209ee5645d8e5552bdf70db5a848736ef0df75db74c8c878553cd40 -LibUnwind.v1.8.1+2.aarch64-unknown-freebsd.tar.gz/md5/ee8fc39c934cf1c640ae4ae41addcc30 -LibUnwind.v1.8.1+2.aarch64-unknown-freebsd.tar.gz/sha512/6245fc3003ef24fce0f84007c0fa1390658e71dc64da6a2f5d296d3928351096ed2c0c83808890413332883abe5fcee7615eb40b2baeddfc56d3484315f3dacf -LibUnwind.v1.8.1+2.armv6l-linux-gnueabihf.tar.gz/md5/4c454e174be7b5f220f4cb8f659722d8 -LibUnwind.v1.8.1+2.armv6l-linux-gnueabihf.tar.gz/sha512/f6e3d83576ae963f400972250c8558b0b15bdd9657aac6eacbd0c3f59af6a3574d0cc475c6e606ad8f2e0b178ba33f297aec0aeac8a5970d93b2c36d9ffae59d -LibUnwind.v1.8.1+2.armv6l-linux-musleabihf.tar.gz/md5/dbec8675d2b73807c9d9e3afc2ce2260 -LibUnwind.v1.8.1+2.armv6l-linux-musleabihf.tar.gz/sha512/45d9ac63282c21bdc6488b65fae8f03bbaa55d18b346ac3fc3d40f38ebd05b2a0db539f23dc6c6f88bbbad8f2ec2cdcf677db1acff83a99d9875bee93555ad1e -LibUnwind.v1.8.1+2.armv7l-linux-gnueabihf.tar.gz/md5/98517b7a4ae874099ef0aafb46e740c9 -LibUnwind.v1.8.1+2.armv7l-linux-gnueabihf.tar.gz/sha512/3a00792415a15fe45c3454f9bf480222862217178a61db0738837537c7e2c50f71b53063facd591680b14e7b3bde218c34cee9b2854ad94897b306388749af1b -LibUnwind.v1.8.1+2.armv7l-linux-musleabihf.tar.gz/md5/f276569278383f7711f40e623670620d -LibUnwind.v1.8.1+2.armv7l-linux-musleabihf.tar.gz/sha512/48160616ac1ed4b3e343556517e3cbb4959e80e9be237fc820e33e06f6668e95d9365dd7c86e68dc898fee1141cd825495bbbc27d685913a2f2808d974b54c19 -LibUnwind.v1.8.1+2.i686-linux-gnu.tar.gz/md5/2cd0203f2b70436ac2323077dad1d5d1 -LibUnwind.v1.8.1+2.i686-linux-gnu.tar.gz/sha512/fa42b3306d9b67011468b2c07bdb6cca6847f0f1632ee4aca3212c5944e991f9a1ae8f881fb4ce86e641e977695942d873a39fc212bdcf6acdf3e12c24b31d8e -LibUnwind.v1.8.1+2.i686-linux-musl.tar.gz/md5/3c456a1b3da2f5d785e02e1b6cb4cd74 -LibUnwind.v1.8.1+2.i686-linux-musl.tar.gz/sha512/fce8368ee670109b681c9d442ad89fee8fdf8eac1e115407784d1e8b82cfb98acd9d2edb4dbea29f8c63c83054da2a4d34149fe231655e2535834a4ef7319666 -LibUnwind.v1.8.1+2.powerpc64le-linux-gnu.tar.gz/md5/73b04ae80ca9fdbe06b3eeaae40d5dc5 -LibUnwind.v1.8.1+2.powerpc64le-linux-gnu.tar.gz/sha512/d4083a696a3492ced38b05fb573d44c4cc2b5332a351b65be2c3992d9e932bb6ea71f48260c643fa54219adb800b5da41160e1d56b0d9145061edf2e5dfc0ef6 -LibUnwind.v1.8.1+2.x86_64-linux-gnu.tar.gz/md5/f9d6132f4166c5ede15b2303280a1066 -LibUnwind.v1.8.1+2.x86_64-linux-gnu.tar.gz/sha512/124159e7d13ce1caee5e2527746ec98b10a776f57e5f9c99053b7ab76e7d9447b998cbc044da7671fd39356445a983f16f2c7bbefc076b29e45d2c2bb4d0364e -LibUnwind.v1.8.1+2.x86_64-linux-musl.tar.gz/md5/665d9215ef915269e009f7dde1f827b3 -LibUnwind.v1.8.1+2.x86_64-linux-musl.tar.gz/sha512/2d8754bbfa7a4b576fb58a2d22b08940bb9f615988bfc388e9ea2cc96e3a573e6c31a4023b2509a3424a0ce3d946584c09ac5d18e4bca6f0f47e52597e193944 -LibUnwind.v1.8.1+2.x86_64-unknown-freebsd.tar.gz/md5/cc8149747db86524da0c9749ed538f3d -LibUnwind.v1.8.1+2.x86_64-unknown-freebsd.tar.gz/sha512/4d416999616fbf08103553aa43603ce62109c21e9a97d6a391fb267c04d382834da380f459c96412773f19d93b8e996ddd405831623ce118d239ad1a0d9025fd -libunwind-1.8.1.tar.gz/md5/10c96118ff30b88c9eeb6eac8e75599d -libunwind-1.8.1.tar.gz/sha512/aba7b578c1b8cbe78f05b64e154f3530525f8a34668b2a9f1ee6acb4b22c857befe34ad4e9e8cca99dbb66689d41bc72060a8f191bd8be232725d342809431b3 +LibUnwind.v1.8.2+1.aarch64-linux-gnu.tar.gz/md5/f23ea74855dd2db466170f170a8eea7a +LibUnwind.v1.8.2+1.aarch64-linux-gnu.tar.gz/sha512/27b4c37d225c1632e1d8814989d768b0fda2675552c611cb9fad1eea158e4b3815e0a8127c2eea97bcbfd23685d970cf1c478d24f6b5a8abe8357e12d8762ce7 +LibUnwind.v1.8.2+1.aarch64-linux-musl.tar.gz/md5/5073e00eeacd4f5921164090966f5ae8 +LibUnwind.v1.8.2+1.aarch64-linux-musl.tar.gz/sha512/0499cdd22510745e3e1f7da5c4c937a79b2eb07e2bf97afa5db9ce7e91aa2bab85bef2dc3ad8b859ac147ca8da908047bf12daf6e3d3fcb2c7a865402dd90beb +LibUnwind.v1.8.2+1.aarch64-unknown-freebsd.tar.gz/md5/b4aa1f850db793ffe9c1e0bf0524da65 +LibUnwind.v1.8.2+1.aarch64-unknown-freebsd.tar.gz/sha512/92bbb42a742427d4a4bfb67530191093ee55a210a99abe0145bdf4ccdc3e583a9852a8045dfdd20002a8fd8eaf9feb6245f84a4995fee4a097a33a2b64998dcd +LibUnwind.v1.8.2+1.armv6l-linux-gnueabihf.tar.gz/md5/a6eff40a5bef97c13fed1546dd77a503 +LibUnwind.v1.8.2+1.armv6l-linux-gnueabihf.tar.gz/sha512/b0e687036c080cf5a421a924e070e57d9f52cee1e67e6c9558d586edc58296d15c9e10e2e46c27e6d290c866b84edeee11023077fb2481dfc9f08fd08540326a +LibUnwind.v1.8.2+1.armv6l-linux-musleabihf.tar.gz/md5/19f8f17923dc1b87b3e89480b1eeaedb +LibUnwind.v1.8.2+1.armv6l-linux-musleabihf.tar.gz/sha512/23251a9fe459242589797a364ff69e38711ebf93d4f047acc7d302c8bc669d3b0546a318568787f0f9bf4d1e633670e0941887cbac1c34e34d86e217040533af +LibUnwind.v1.8.2+1.armv7l-linux-gnueabihf.tar.gz/md5/74bd5a25811e4977386ba3aefe98bd8c +LibUnwind.v1.8.2+1.armv7l-linux-gnueabihf.tar.gz/sha512/2d41f653640ffaebb7295d49a533efebf2ffca7314a181e6e669c399e65b49d176915afeb60b4702632d65aa20aad772052373efbd3c6b9fdc0c9fd772986b2a +LibUnwind.v1.8.2+1.armv7l-linux-musleabihf.tar.gz/md5/7d882db1897c88c7e2eb7cf94e3b3f9e +LibUnwind.v1.8.2+1.armv7l-linux-musleabihf.tar.gz/sha512/dad5bdbccd630eb27af69a32f01245865fc9cf494f1e2ab4cf9e06d78985d997accbac7db6c947bb1964f0aa98f7c977cdb9288e4453fa22d47eb99d73144536 +LibUnwind.v1.8.2+1.i686-linux-gnu.tar.gz/md5/74076851e0da8ebf94fefa35718b3ec8 +LibUnwind.v1.8.2+1.i686-linux-gnu.tar.gz/sha512/4549cbe8da9b0ff5cb62c7cdc24534e4fd0f48354902082212d9c559efbdb02017884b41554432e4f16886bafe27b737f4264c05ddba4b14166212290d97f37c +LibUnwind.v1.8.2+1.i686-linux-musl.tar.gz/md5/9a91cf5da60391e2bbae8ec77ef158c6 +LibUnwind.v1.8.2+1.i686-linux-musl.tar.gz/sha512/5dd44a67c259cbd629d8b17375e00a1e18f9fcf2d9bb86f26eefc31c52cef0ae7ebfd279e37c6b2ef428df98af30348c65a0760e27409ba90614f3cbb80a8c08 +LibUnwind.v1.8.2+1.powerpc64le-linux-gnu.tar.gz/md5/052e192212dbf6412a4ea12ca5a8b40f +LibUnwind.v1.8.2+1.powerpc64le-linux-gnu.tar.gz/sha512/2d985c65453935393de449133dbbfabab68c832ecb6243100e1514ba7a613716dfa2bf690bf1df9a86d9c45eb683e7c61c04ff187347d61b17055964ee7944af +LibUnwind.v1.8.2+1.riscv64-linux-gnu.tar.gz/md5/bfd355c19743b5b8121005f41a7a1eaa +LibUnwind.v1.8.2+1.riscv64-linux-gnu.tar.gz/sha512/4659d1ddf80cf41c99ee374d98a56264706fd81836ce6cc5e71bfa547bf181cf7775d017d9c3b925a678e33446ca09bb8cda057ed48fdb702e338c9c1adc5aa4 +LibUnwind.v1.8.2+1.x86_64-linux-gnu.tar.gz/md5/9c0d4c4c8d96a3b64d1edeba61df80d9 +LibUnwind.v1.8.2+1.x86_64-linux-gnu.tar.gz/sha512/2290ca45c950dc6c0ef2f3f915386879e179f98e08b29a5f3e55e8c5fb646389e8dc09dbd1f8f91c382e623a167996322624dfe035142aea11f8e30fa643ec2e +LibUnwind.v1.8.2+1.x86_64-linux-musl.tar.gz/md5/19a6b5638f1c12732a4ea29dc2ba45ca +LibUnwind.v1.8.2+1.x86_64-linux-musl.tar.gz/sha512/834f2e5a2a98e11af20e8d0c3ec91a6e72a60573e8cd556d6c5d2321ecda94afe90911f1b9b6b313140f13433a5c3b1b30d4951ee40ffc862500eea88a44c8b8 +LibUnwind.v1.8.2+1.x86_64-unknown-freebsd.tar.gz/md5/cc97cb986489fb1aaa157490986da567 +LibUnwind.v1.8.2+1.x86_64-unknown-freebsd.tar.gz/sha512/7ca93ec2eaec8f1c22104459ad2db49d5f1303686e8f686312296287b37f51cbde3fb605be84297848fc0997836735e59914f2f8c86fe72bb9ec7c44c35ee8f9 +libunwind-1.8.2.tar.gz/md5/0124a38fb752aa5492635f35d089f6b7 +libunwind-1.8.2.tar.gz/sha512/f1ff26763c1b2e68948413c4aec22303b6c886425a8264eb65fbd58fc202f79c7b04bd4784bd8499850d08933f0e363cfa3a7d177efdadc223ed0254bc381345 diff --git a/deps/checksums/zstd b/deps/checksums/zstd new file mode 100644 index 0000000000000..aea151b266966 --- /dev/null +++ b/deps/checksums/zstd @@ -0,0 +1,38 @@ +Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/md5/d6b2fb32d705078dbc369986ac8b056b +Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/sha512/5dfcf36087ce8540b1f6a04181adee962e2164a763e758ac5cc256c332756774b381ca58e26641a15ce555d59641690a6da72a67bf935d8611734f2006bde504 +Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/md5/0c627ec83e426383c25eb4bc297f3548 +Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/sha512/1fdcf77e877f0676fc26a05e0cc20a1d6e1df731d81e0bba9a5657131116bbea75da4d38953969d8d07dce0bf2d7654075dbb285ebe5f4588c446e88774336c8 +Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/md5/cc9ada74a19db50d7dd6edd05866c902 +Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/sha512/0b33c0df144bb1e95290685f01695b26da834a70a365c0362314cb001ba611962a0876bc5baac31f19c80bcb110e030fb9840a56761b4d29a7893ca65f95b111 +Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/md5/5daa5b2bf2b856c448feaa8329d0de1b +Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/sha512/b39d025463b4bf21295fd5bbff91ba501506b3480363cdcfe6dd2f11d2e0afaf130f6c74d962e503fccb7a55bfcad0504ebb19f18b6b5c8b8103e7b9919df536 +Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/md5/f4218e8b4f8d415df49aeba9d43f0ba0 +Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/sha512/878d4f90160c6b0c341c61ecafbf5f5cb89c73db3175f272adc666bc25c88b127145d78946bc0fcb992489b54fbb48089bfcacf768397fc5d54d7cae4aeae9f9 +Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/md5/3c2e132ca47e6d1d23c149fdde9d8bd5 +Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/sha512/3745d99c9ca0ce9f98ff9393e405e8b382d05573a972067d57e800e282a9544fff7bc3d49b91eccc98d7736acdc3faa4c637911d79fab10f5a691d33ae775574 +Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/md5/926d765281bef388ecc25d04cbb66102 +Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/sha512/2d2c14587e2e7b2b147cb6423720cc30ed6aa57ed07372a1aa54e7f2e6badb5aa640b116e83371561d6f8f3a1b3f7fff7f6df137f8c7be788ee889bb30273eae +Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/md5/c25420561ce254e57d74e30c88fc53dd +Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/sha512/2f924e2089589057e8713d04db9a1cb2f2d571ad9e7eeda3b7f898c9a75f8fecf0647f2185d3c01fc3b399d3662ff3b1acb13429c8a953f0394a3ed9ca30b877 +Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/md5/3314bf1b52f2295555fb4ae44b1d9331 +Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/sha512/91502910a0c9b786d91499477fee2445b8f6de6bcb71af7d79c738ea2430c67cb1957866383ee3921ed1a23c53a80be19aea6abcf0e76056ffee69583728c3ed +Zstd.v1.5.7+1.i686-linux-musl.tar.gz/md5/845eddc06527a4c4b196666f7ac64ba3 +Zstd.v1.5.7+1.i686-linux-musl.tar.gz/sha512/bb15b4327cef32be38c2fd68afedb3245c7db881ad66d3ece2198ff3034be9c12efa3d62bcba2b8e6056e7d8cb5f1b3e33726f7d1e1bead235c38f8fa985b557 +Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/md5/9bc0b3c951f5e66393fd5433bf60a2c8 +Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/sha512/550b0189097e569f98404aa836b76a5cbdc36428292214c4af8916dea2713440cf3ba94125b3e5fa0c65b2bcb916733094fdef906ad19f923d90dabfc961c75a +Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/md5/468d930de7a27af961996e7c6ed35298 +Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/sha512/d680715b1ac9ff07d5662c499fbab67757509599335f861158b9dba32fe9b22da6e52d0db6b402dd4542799621ad3dccf254dfd9d3c8748bbd22f7446681539a +Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/md5/b93fef8db2b0b4417f7836d73c5fbe86 +Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/sha512/9f3ee42c7952aba2d2c26252f058bb7ab96828fafc978c9273b500ef15ccd271c51399d4b93eebd4c832b087ab5ed8a4847104ce9c83c9483aaa13c22df681bb +Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/md5/29a260789fae6f6b6df0e5cebdafd615 +Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/sha512/015045a1b7a477504057cb4c87428d42386218e48af38f83739dbe6b93961ca2c8dd4d794377a2d54b8cc284f5a467e3358d4f534cf8bcbcad886ef8cea038e9 +Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/md5/06656befb6ef9a8cc7f56e7152c2acc5 +Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/sha512/16aea0d95432a87d21d9a6f55d84e45df85caf1fda77c75b7e9a8bba519605168585f21a812773ddf1075d9bad68412e63b8cad1a143420e25ae4405bb41842e +Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/md5/da13dd1cc0d20ba9a06e9e79a588cda4 +Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/sha512/cd4218fa92dcf8772390788d5654ca12132af7829fb0ada016f3c663e2045e29e7d7587f2f5a4f057020cacca17c188c8537f284b1456100d57e84bb47c40e77 +Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/md5/bce5f37e53e330bfe4df4a28cf5c223b +Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/sha512/8f6bd7664efea537ac7815db0604ca1a07bcfb71b5152c22dc7f0a11b57643f059c341fa71d315407e2333e4c97e43e214471c73eed8b977680785302c7c2b3e +Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/md5/7cf3a740fa174004b94125e8754f4a19 +Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/sha512/faac37ad4dacb0f083364c593cd3bd1c0b592947341a631bd2fbc4081361d97ef89482f4459c46ad37ae030aa900c62305a8525e64a2ad8e91204d76dda89db1 +zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/md5/a679d9aa86549b5851100ac5d4044c68 +zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/sha512/27c6fff165abea694d91311a6657a939433ba1d707147ed9072b5e4ecce259b929970306788e0c3e95db38ce85e894e5025936b1faa81cf67741b8464e24fc4e diff --git a/deps/clang.version b/deps/clang.version index 48130a21d68a4..68dafab189f7e 100644 --- a/deps/clang.version +++ b/deps/clang.version @@ -3,4 +3,4 @@ ## jll artifact # Clang (paired with LLVM, only here as a JLL download) CLANG_JLL_NAME := Clang -CLANG_JLL_VER := 19.1.7+1 +CLANG_JLL_VER := 20.1.2+0 diff --git a/deps/csl.mk b/deps/csl.mk index fef950aa41621..51368187c55fc 100644 --- a/deps/csl.mk +++ b/deps/csl.mk @@ -2,10 +2,10 @@ STD_LIB_PATH := $(shell LANG=C $(FC) -print-search-dirs 2>/dev/null | grep '^programs: =' | sed -e "s/^programs: =//") STD_LIB_PATH += $(PATHSEP)$(shell LANG=C $(FC) -print-search-dirs 2>/dev/null | grep '^libraries: =' | sed -e "s/^libraries: =//") ifeq ($(BUILD_OS),WINNT) # the mingw compiler lies about it search directory paths -STD_LIB_PATH := $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g") +STD_LIB_PATH += $(shell echo '$(STD_LIB_PATH)' | sed -e "s!/lib/!/bin/!g") endif -# Given a colon-separated list of paths in $(2), find the location of the library given in $(1) +# Given a $(PATHSEP)-separated list of paths in $(2), find the location of the library given in $(1) define pathsearch $(firstword $(wildcard $(addsuffix /$(1),$(subst $(PATHSEP), ,$(2))))) endef @@ -54,6 +54,13 @@ $$(build_shlibdir)/$(1): | $$(build_shlibdir) [ -n "$$$${SRC_LIB}" ] && cp "$$$${SRC_LIB}" '$$(build_shlibdir)' endef +define copy_csl_static_lib +install-csl: | $$(build_private_libdir) $$(build_private_libdir)/$(1) +$$(build_private_libdir)/$(1): | $$(build_private_libdir) + -@SRC_LIB='$$(call pathsearch,$(1),$$(STD_LIB_PATH))'; \ + [ -n "$$$${SRC_LIB}" ] && cp "$$$${SRC_LIB}" '$$(build_private_libdir)' +endef + # libgfortran has multiple names; we're just going to copy any version we can find # Since we're only looking in the location given by `$(FC)` this should only succeed for one. $(eval $(call copy_csl,$(call versioned_libname,libgfortran,3))) @@ -63,11 +70,20 @@ $(eval $(call copy_csl,$(call versioned_libname,libgfortran,5))) # These are all libraries that we should always have $(eval $(call copy_csl,$(call versioned_libname,libquadmath,0))) $(eval $(call copy_csl,$(call versioned_libname,libstdc++,6))) -$(eval $(call copy_csl,$(call versioned_libname,libssp,0))) $(eval $(call copy_csl,$(call versioned_libname,libatomic,1))) $(eval $(call copy_csl,$(call versioned_libname,libgomp,1))) +# Configurable either a static or dynamic library depending on the system +$(eval $(call copy_csl,$(call versioned_libname,libssp,0))) +$(eval $(call copy_csl_static_lib,libssp.a)) + ifeq ($(OS),WINNT) +# On windows we need the static gcc runtime libraries for linking pkgimages +$(eval $(call copy_csl_static_lib,libgcc.a)) +$(eval $(call copy_csl_static_lib,libgcc_s.a)) +$(eval $(call copy_csl_static_lib,libmsvcrt.a)) +$(eval $(call copy_csl_static_lib,libmingwex.a)) +$(eval $(call copy_csl_static_lib,libkernel32.a)) # Windows has special gcc_s names ifeq ($(ARCH),i686) $(eval $(call copy_csl,$(call versioned_libname,libgcc_s_sjlj,1))) @@ -120,6 +136,7 @@ ifeq ($(OS),WINNT) GCC_VERSION = 14 install-csl: mkdir -p $(build_private_libdir)/ + cp -a $(build_shlibdir)/$(call versioned_libname,libstdc++,6) $(build_shlibdir)/libstdc++.$(SHLIB_EXT) cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libgcc_s.a $(build_private_libdir)/ cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libgcc.a $(build_private_libdir)/ cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libmsvcrt.a $(build_private_libdir)/ @@ -127,12 +144,15 @@ install-csl: cp -a $(build_libdir)/gcc/$(BB_TRIPLET)/$(GCC_VERSION)/libssp.dll.a $(build_libdir)/ endif endif + ifeq ($(OS),WINNT) uninstall-csl: uninstall-gcc-libraries uninstall-gcc-libraries: + -rm -f $(build_shlibdir)/libstdc++.$(SHLIB_EXT) -rm -f $(build_private_libdir)/libgcc_s.a -rm -f $(build_private_libdir)/libgcc.a -rm -f $(build_private_libdir)/libmsvcrt.a -rm -f $(build_private_libdir)/libssp.dll.a -rm -f $(build_libdir)/libssp.dll.a +.PHONY: uninstall-gcc-libraries endif diff --git a/deps/curl.mk b/deps/curl.mk index 6232d56e5e333..1f650dce9370e 100644 --- a/deps/curl.mk +++ b/deps/curl.mk @@ -31,7 +31,7 @@ $(SRCCACHE)/curl-$(CURL_VER).tar.bz2: | $(SRCCACHE) $(SRCCACHE)/curl-$(CURL_VER)/source-extracted: $(SRCCACHE)/curl-$(CURL_VER).tar.bz2 $(JLCHECKSUM) $< - cd $(dir $<) && $(TAR) jxf $(notdir $<) + cd $(dir $<) && $(TAR) -jxf $(notdir $<) echo 1 > $@ checksum-curl: $(SRCCACHE)/curl-$(CURL_VER).tar.bz2 @@ -49,17 +49,14 @@ CURL_CONFIGURE_FLAGS := $(CONFIGURE_COMMON) \ --without-brotli # A few things we actually enable CURL_CONFIGURE_FLAGS += \ - --with-libssh2=${build_prefix} --with-zlib=${build_prefix} --with-nghttp2=${build_prefix} \ + --with-libssh2=${build_prefix} --with-zlib=${build_prefix} --with-zstd=${build_prefix} --with-nghttp2=${build_prefix} \ --enable-versioned-symbols # We use different TLS libraries on different platforms. # On Windows, we use schannel -# On MacOS, we use SecureTransport -# On Linux, we use OpenSSL +# On other platforms, we use OpenSSL ifeq ($(OS), WINNT) CURL_TLS_CONFIGURE_FLAGS := --with-schannel -else ifeq ($(OS), Darwin) -CURL_TLS_CONFIGURE_FLAGS := --with-secure-transport else CURL_TLS_CONFIGURE_FLAGS := --with-openssl endif diff --git a/deps/curl.version b/deps/curl.version index 0a435fca5b7da..ecd8089d9fd1f 100644 --- a/deps/curl.version +++ b/deps/curl.version @@ -3,4 +3,4 @@ CURL_JLL_NAME := LibCURL ## source build -CURL_VER := 8.12.1 +CURL_VER := 8.15.0 diff --git a/deps/ittapi.mk b/deps/ittapi.mk index b62b981a34ddb..f27ae2d6f77d5 100644 --- a/deps/ittapi.mk +++ b/deps/ittapi.mk @@ -10,7 +10,7 @@ ITTAPI_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DITT_API_IPT_SUPPORT= $(BUILDDIR)/$(ITTAPI_SRC_DIR)/build-configured: $(SRCCACHE)/$(ITTAPI_SRC_DIR)/source-extracted mkdir -p $(dir $@) cd $(dir $@) && \ - $(CMAKE) $(dir $<) $(ITTAPI_OPTS) + $(CMAKE) -G"Unix Makefiles" $(dir $<) $(ITTAPI_OPTS) echo 1 > $@ $(BUILDDIR)/$(ITTAPI_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ITTAPI_SRC_DIR)/build-configured diff --git a/doc/Manifest.toml b/deps/jlutilities/documenter/Manifest.toml similarity index 80% rename from doc/Manifest.toml rename to deps/jlutilities/documenter/Manifest.toml index 8dadbc0ecde2c..55ff511a39daa 100644 --- a/doc/Manifest.toml +++ b/deps/jlutilities/documenter/Manifest.toml @@ -1,6 +1,6 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.12.0-DEV" +julia_version = "1.13.0-DEV" manifest_format = "2.0" project_hash = "1e9ffa7d4739f7d125a5e2c66af8747a8effd889" @@ -28,9 +28,9 @@ version = "1.11.0" [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.6" +version = "0.7.8" [[deps.Dates]] deps = ["Printf"] @@ -38,43 +38,42 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" [[deps.DocStringExtensions]] -deps = ["LibGit2"] -git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +git-tree-sha1 = "e7b7e6f178525d17c720ab9c081e4ef04429f860" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.9.3" +version = "0.9.4" [[deps.Documenter]] -deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] -git-tree-sha1 = "182a9a3fe886587ba230a417f1651a4cbc2b92d4" +deps = ["ANSIColoredPrinters", "AbstractTrees", "Base64", "CodecZlib", "Dates", "DocStringExtensions", "Downloads", "Git", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "MarkdownAST", "Pkg", "PrecompileTools", "REPL", "RegistryInstances", "SHA", "TOML", "Test", "Unicode"] +git-tree-sha1 = "6c182d0bd94142d7cbc3ae8a1e74668f15d0dd65" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "1.8.1" +version = "1.11.4" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" +version = "1.7.0" [[deps.Expat_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "1c6317308b9dc757616f0b5cb379db10494443a7" +git-tree-sha1 = "d55dffd9ae73ff72f1c0482454dcf2ec6c6c4a63" uuid = "2e619515-83b5-522b-bb60-26c02a35a201" -version = "2.6.2+0" +version = "2.6.5+0" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" version = "1.11.0" [[deps.Git]] -deps = ["Git_jll"] -git-tree-sha1 = "04eff47b1354d702c3a85e8ab23d539bb7d5957e" +deps = ["Git_jll", "JLLWrappers", "OpenSSH_jll"] +git-tree-sha1 = "2230a9cc32394b11a3b3aa807a382e3bbab1198c" uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" -version = "1.3.1" +version = "1.4.0" [[deps.Git_jll]] deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] -git-tree-sha1 = "ea372033d09e4552a04fd38361cd019f9003f4f4" +git-tree-sha1 = "2f6d6f7e6d6de361865d4394b802c02fc944fc7c" uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" -version = "2.46.2+0" +version = "2.49.0+0" [[deps.IOCapture]] deps = ["Logging", "Random"] @@ -89,9 +88,9 @@ version = "1.11.0" [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" +git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.6.1" +version = "1.7.0" [[deps.JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] @@ -117,7 +116,7 @@ version = "0.6.4" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.12.1+1" +version = "8.14.1+1" [[deps.LibGit2]] deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] @@ -125,12 +124,12 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" version = "1.11.0" [[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.0+0" +version = "1.9.1+0" [[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "Libdl", "OpenSSL_jll", "Zlib_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" version = "1.11.3+1" @@ -140,9 +139,9 @@ version = "1.11.0" [[deps.Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "61dfdba58e585066d8bce214c5a51eaa0539f269" +git-tree-sha1 = "be484f5c92fad0bd8acfef35fe017900b0b73809" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" -version = "1.17.0+1" +version = "1.18.0+0" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -165,32 +164,38 @@ version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2024.12.31" +version = "2025.5.20" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.3.0" +[[deps.OpenSSH_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "OpenSSL_jll", "Zlib_jll"] +git-tree-sha1 = "cb7acd5d10aff809b4d0191dfe1956c2edf35800" +uuid = "9bd350c2-7e96-507f-8002-3f2e150b4e1b" +version = "10.0.1+0" + [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.15+2" +version = "3.5.0+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.44.0+1" +version = "10.45.0+0" [[deps.Parsers]] deps = ["Dates", "PrecompileTools", "UUIDs"] -git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +git-tree-sha1 = "7d2f8f21da5db6a806faf7b9b292296da42b2810" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.8.1" +version = "2.8.3" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.12.0" +version = "1.13.0" weakdeps = ["REPL"] [deps.Pkg.extensions] @@ -198,9 +203,9 @@ weakdeps = ["REPL"] [[deps.PrecompileTools]] deps = ["Preferences"] -git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f" +git-tree-sha1 = "516f18f048a195409d6e072acf879a9f017d3900" uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -version = "1.2.1" +version = "1.3.2" [[deps.Preferences]] deps = ["TOML"] @@ -214,7 +219,7 @@ uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" version = "1.11.0" [[deps.REPL]] -deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +deps = ["FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" version = "1.11.0" diff --git a/doc/Project.toml b/deps/jlutilities/documenter/Project.toml similarity index 100% rename from doc/Project.toml rename to deps/jlutilities/documenter/Project.toml diff --git a/deps/jlutilities/revise/Manifest.toml b/deps/jlutilities/revise/Manifest.toml new file mode 100644 index 0000000000000..143f20b08e9c1 --- /dev/null +++ b/deps/jlutilities/revise/Manifest.toml @@ -0,0 +1,159 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.13.0-DEV" +manifest_format = "2.0" +project_hash = "6de5e8b1c4d9b467a5c126490dbc755dc0575a9c" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.CodeTracking]] +deps = ["InteractiveUtils", "UUIDs"] +git-tree-sha1 = "062c5e1a5bf6ada13db96a4ae4749a4c2234f521" +uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2" +version = "1.3.9" + +[[deps.Compiler]] +git-tree-sha1 = "382d79bfe72a406294faca39ef0c3cef6e6ce1f1" +uuid = "807dbc54-b67e-4c79-8afb-eafe4df6f2e1" +version = "0.1.1" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.3.0+1" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.JuliaInterpreter]] +deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"] +git-tree-sha1 = "6ac9e4acc417a5b534ace12690bc6973c25b862f" +uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a" +version = "0.10.3" + +[[deps.JuliaSyntaxHighlighting]] +deps = ["StyledStrings"] +uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" +version = "1.12.0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.1+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "OpenSSL_jll", "Zlib_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.LoweredCodeUtils]] +deps = ["Compiler", "JuliaInterpreter"] +git-tree-sha1 = "b882a7dd7ef37643066ae8f9380beea8fdd89cae" +uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b" +version = "3.4.2" + +[[deps.Markdown]] +deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.3.0" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.5.1+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.8.1" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.45.0+0" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.REPL]] +deps = ["FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.1" + +[[deps.Revise]] +deps = ["CodeTracking", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "REPL", "Requires", "UUIDs", "Unicode"] +git-tree-sha1 = "82dc140c7f52e4daeeec3675a411d48167a85a87" +repo-rev = "master" +repo-url = "https://github.com/timholy/Revise.jl.git" +uuid = "295af30f-e4ad-537b-8983-00126c2a3abe" +version = "3.8.0" + + [deps.Revise.extensions] + DistributedExt = "Distributed" + + [deps.Revise.weakdeps] + Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.3.1+2" diff --git a/deps/jlutilities/revise/Project.toml b/deps/jlutilities/revise/Project.toml new file mode 100644 index 0000000000000..7f7ec8b06162c --- /dev/null +++ b/deps/jlutilities/revise/Project.toml @@ -0,0 +1,5 @@ +[deps] +Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" + +[sources] +Revise = {rev = "master", url = "https://github.com/timholy/Revise.jl.git"} diff --git a/deps/libgit2.mk b/deps/libgit2.mk index 8b17ae6d70424..85bc0629f6f28 100644 --- a/deps/libgit2.mk +++ b/deps/libgit2.mk @@ -4,6 +4,7 @@ ifneq ($(USE_BINARYBUILDER_LIBGIT2),1) LIBGIT2_GIT_URL := https://github.com/libgit2/libgit2.git LIBGIT2_TAR_URL = https://api.github.com/repos/libgit2/libgit2/tarball/$1 $(eval $(call git-external,libgit2,LIBGIT2,CMakeLists.txt,,$(SRCCACHE))) +$(SRCCACHE)/$(LIBGIT2_SRC_DIR)/source-extracted: export MSYS=$(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) ifeq ($(USE_SYSTEM_LIBSSH2), 0) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/libssh2 @@ -13,7 +14,15 @@ ifeq ($(USE_SYSTEM_OPENSSL), 0) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/openssl endif -LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DUSE_THREADS=ON -DUSE_BUNDLED_ZLIB=ON -DUSE_SSH=ON -DBUILD_CLI=OFF +ifeq ($(USE_SYSTEM_PCRE), 0) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/pcre +endif + +ifeq ($(USE_SYSTEM_ZLIB), 0) +$(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: | $(build_prefix)/manifest/zlib +endif + +LIBGIT2_OPTS := $(CMAKE_COMMON) -DCMAKE_BUILD_TYPE=Release -DUSE_THREADS=ON -DUSE_BUNDLED_ZLIB=OFF -DUSE_SSH=ON -DREGEX_BACKEND=pcre2 -DBUILD_CLI=OFF -DBUILD_TESTS=OFF ifeq ($(OS),WINNT) LIBGIT2_OPTS += -DWIN32=ON -DMINGW=ON ifeq ($(USE_SYSTEM_LIBSSH2), 0) @@ -42,16 +51,12 @@ ifneq (,$(findstring $(OS),Linux FreeBSD OpenBSD)) LIBGIT2_OPTS += -DUSE_HTTPS="OpenSSL" -DUSE_SHA1="CollisionDetection" -DCMAKE_INSTALL_RPATH="\$$ORIGIN" endif -# use the bundled distribution of libpcre. we should consider linking against the -# pcre2 library we're building anyway, but this is currently how Yggdrasil does it. -LIBGIT2_OPTS += -DREGEX_BACKEND="builtin" - LIBGIT2_SRC_PATH := $(SRCCACHE)/$(LIBGIT2_SRC_DIR) $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured: $(LIBGIT2_SRC_PATH)/source-extracted mkdir -p $(dir $@) cd $(dir $@) && \ - $(CMAKE) $(dir $<) $(LIBGIT2_OPTS) + $(CMAKE) -G"Unix Makefiles" $(dir $<) $(LIBGIT2_OPTS) echo 1 > $@ $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-compiled: $(BUILDDIR)/$(LIBGIT2_SRC_DIR)/build-configured diff --git a/deps/libgit2.version b/deps/libgit2.version index 6bfb6106e67d2..90cdd89b83a29 100644 --- a/deps/libgit2.version +++ b/deps/libgit2.version @@ -3,12 +3,12 @@ LIBGIT2_JLL_NAME := LibGit2 ## source build -LIBGIT2_BRANCH=v1.9.0 -LIBGIT2_SHA1=338e6fb681369ff0537719095e22ce9dc602dbf0 +LIBGIT2_BRANCH=v1.9.1 +LIBGIT2_SHA1=0060d9cf5666f015b1067129bd874c6cc4c9c7ac ## Other deps # Specify the version of the Mozilla CA Certificate Store to obtain. # The versions of cacert.pem are identified by the date (YYYY-MM-DD) of their changes. # See https://curl.haxx.se/docs/caextract.html for more details. # Keep in sync with `stdlib/MozillaCACerts_jll/Project.toml`. -MOZILLA_CACERT_VERSION := 2024-12-31 +MOZILLA_CACERT_VERSION := 2025-07-15 diff --git a/deps/libssh2.mk b/deps/libssh2.mk index 3f802db15be6d..05cc12b6e159b 100644 --- a/deps/libssh2.mk +++ b/deps/libssh2.mk @@ -17,11 +17,9 @@ endif ifeq ($(OS),WINNT) LIBSSH2_OPTS += -DCRYPTO_BACKEND=WinCNG -DENABLE_ZLIB_COMPRESSION=OFF -ifeq ($(BUILD_OS),WINNT) -LIBSSH2_OPTS += -G"MSYS Makefiles" -endif else LIBSSH2_OPTS += -DCRYPTO_BACKEND=OpenSSL -DENABLE_ZLIB_COMPRESSION=OFF +LIBSSH2_OPTS += -DOPENSSL_ROOT_DIR=$(build_prefix) endif ifneq (,$(findstring $(OS),Linux FreeBSD OpenBSD)) @@ -37,7 +35,7 @@ LIBSSH2_SRC_PATH := $(SRCCACHE)/$(LIBSSH2_SRC_DIR) $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured: $(LIBSSH2_SRC_PATH)/source-extracted mkdir -p $(dir $@) cd $(dir $@) && \ - $(CMAKE) $(dir $<) $(LIBSSH2_OPTS) + $(CMAKE) $(CMAKE_GENERATOR_COMMAND) $(dir $<) $(LIBSSH2_OPTS) echo 1 > $@ $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-compiled: $(BUILDDIR)/$(LIBSSH2_SRC_DIR)/build-configured diff --git a/deps/libsuitesparse.mk b/deps/libsuitesparse.mk index 85b2c23473a18..4bfeb0742fb7e 100644 --- a/deps/libsuitesparse.mk +++ b/deps/libsuitesparse.mk @@ -58,7 +58,7 @@ $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/source-patched: $(BUILDDIR)/SuiteS $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/build-compiled: | $(build_prefix)/manifest/blastrampoline $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/build-compiled: $(BUILDDIR)/SuiteSparse-$(LIBSUITESPARSE_VER)/source-patched - cd $(dir $<) && $(CMAKE) . $(LIBSUITESPARSE_CMAKE_FLAGS) + cd $(dir $<) && $(CMAKE) -G"Unix Makefiles" . $(LIBSUITESPARSE_CMAKE_FLAGS) $(MAKE) -C $(dir $<) $(MAKE) -C $(dir $<) install echo 1 > $@ diff --git a/deps/libuv.mk b/deps/libuv.mk index eacabac55e34f..993aa4fc144da 100644 --- a/deps/libuv.mk +++ b/deps/libuv.mk @@ -4,7 +4,7 @@ LIBUV_GIT_URL:=https://github.com/JuliaLang/libuv.git LIBUV_TAR_URL=https://api.github.com/repos/JuliaLang/libuv/tarball/$1 $(eval $(call git-external,libuv,LIBUV,configure,,$(SRCCACHE))) -UV_CFLAGS := -O2 +UV_CFLAGS := -O2 -DBUILDING_UV_SHARED=1 UV_FLAGS := LDFLAGS="$(LDFLAGS) $(CLDFLAGS) -v" UV_FLAGS += CFLAGS="$(CFLAGS) $(UV_CFLAGS) $(SANITIZE_OPTS)" diff --git a/deps/lld.version b/deps/lld.version index 1b76dd46f5b46..023122efc4596 100644 --- a/deps/lld.version +++ b/deps/lld.version @@ -2,4 +2,4 @@ ## jll artifact LLD_JLL_NAME := LLD -LLD_JLL_VER := 19.1.7+1 +LLD_JLL_VER := 20.1.2+0 diff --git a/deps/llvm-tools.version b/deps/llvm-tools.version index e6740380b599a..285079ca412e5 100644 --- a/deps/llvm-tools.version +++ b/deps/llvm-tools.version @@ -3,5 +3,5 @@ ## jll artifact # LLVM_tools (downloads LLVM_jll to get things like `lit` and `opt`) LLVM_TOOLS_JLL_NAME := LLVM -LLVM_TOOLS_JLL_VER := 19.1.7+1 -LLVM_TOOLS_ASSERT_JLL_VER := 19.1.7+1 +LLVM_TOOLS_JLL_VER := 20.1.2+0 +LLVM_TOOLS_ASSERT_JLL_VER := 20.1.2+0 diff --git a/deps/llvm.mk b/deps/llvm.mk index 09dd4f187d611..e3303aba55afd 100644 --- a/deps/llvm.mk +++ b/deps/llvm.mk @@ -7,6 +7,14 @@ ifneq ($(USE_BINARYBUILDER_LLVM), 1) LLVM_GIT_URL:=https://github.com/JuliaLang/llvm-project.git LLVM_TAR_URL=https://api.github.com/repos/JuliaLang/llvm-project/tarball/$1 $(eval $(call git-external,llvm,LLVM,CMakeLists.txt,,$(SRCCACHE))) +# LLVM's tarball contains symlinks to non-existent targets. This breaks the +# the default msys strategy `deepcopy` symlink strategy. To workaround this, +# switch to `native` which tries native windows symlinks (possible if the +# machine is in developer mode) - or if not, falls back to cygwin-style +# symlinks. We don't particularly care either way - we just need to symlinks +# to succeed. We could guard this by a uname check, but it's harmless elsewhere, +# so let's not incur the additional overhead. +$(SRCCACHE)/$(LLVM_SRC_DIR)/source-extracted: export MSYS=$(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) LLVM_BUILDDIR := $(BUILDDIR)/$(LLVM_SRC_DIR) LLVM_BUILDDIR_withtype := $(LLVM_BUILDDIR)/build_$(LLVM_BUILDTYPE) @@ -60,6 +68,10 @@ ifeq ($(BUILD_LLD), 1) LLVM_ENABLE_PROJECTS := $(LLVM_ENABLE_PROJECTS);lld endif +# Remove ; if it's the first character +ifneq ($(LLVM_ENABLE_RUNTIMES),) + LLVM_ENABLE_RUNTIMES := $(patsubst ;%,%,$(LLVM_ENABLE_RUNTIMES)) +endif LLVM_LIB_FILE := libLLVMCodeGen.a @@ -70,7 +82,7 @@ LLVM_EXPERIMENTAL_TARGETS := LLVM_CFLAGS := LLVM_CXXFLAGS := LLVM_CPPFLAGS := -LLVM_LDFLAGS := +LLVM_LDFLAGS := "-L$(build_shlibdir)" # hacky way to force zlib to be found when linking against libLLVM and sysroot is set LLVM_CMAKE := LLVM_CMAKE += -DLLVM_ENABLE_PROJECTS="$(LLVM_ENABLE_PROJECTS)" @@ -95,7 +107,7 @@ LLVM_CMAKE += -DLLVM_TARGETS_TO_BUILD:STRING="$(LLVM_TARGETS)" -DCMAKE_BUILD_TYP LLVM_CMAKE += -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="$(LLVM_EXPERIMENTAL_TARGETS)" LLVM_CMAKE += -DLLVM_ENABLE_LIBXML2=OFF -DLLVM_HOST_TRIPLE="$(or $(XC_HOST),$(BUILD_MACHINE))" LLVM_CMAKE += -DLLVM_ENABLE_ZLIB=FORCE_ON -DZLIB_ROOT="$(build_prefix)" -LLVM_CMAKE += -DLLVM_ENABLE_ZSTD=OFF +LLVM_CMAKE += -DLLVM_ENABLE_ZSTD=FORCE_ON -DZSTD_ROOT="$(build_prefix)" ifeq ($(USE_POLLY_ACC),1) LLVM_CMAKE += -DPOLLY_ENABLE_GPGPU_CODEGEN=ON endif @@ -183,6 +195,11 @@ endif ifeq ($(fPIC),) LLVM_CMAKE += -DLLVM_ENABLE_PIC=OFF +else +ifeq ($(OS),FreeBSD) + # On FreeBSD, we must force even statically-linked code to have -fPIC + LLVM_CMAKE += -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE +endif endif LLVM_CMAKE += -DCMAKE_C_FLAGS="$(LLVM_CPPFLAGS) $(LLVM_CFLAGS)" \ @@ -242,6 +259,11 @@ ifeq ($(USE_SYSTEM_ZLIB), 0) $(LLVM_BUILDDIR_withtype)/build-configured: | $(build_prefix)/manifest/zlib endif +ifeq ($(USE_SYSTEM_ZSTD), 0) +$(LLVM_BUILDDIR_withtype)/build-configured: | $(build_prefix)/manifest/zstd +endif + + # NOTE: LLVM 12 and 13 have their patches applied to JuliaLang/llvm-project # declare that all patches must be applied before running ./configure @@ -270,6 +292,12 @@ $(LLVM_BUILDDIR_withtype)/build-configured: $(SRCCACHE)/$(LLVM_SRC_DIR)/source-e echo 1 > $@ $(LLVM_BUILDDIR_withtype)/build-compiled: $(LLVM_BUILDDIR_withtype)/build-configured +ifeq ($(OS),WINNT) +ifeq ($(USEGCC),1) + echo "LLVM source build is currently known to fail using GCC due to exceeded export table limits. Try clang." + exit 1 +endif +endif cd $(LLVM_BUILDDIR_withtype) && \ $(if $(filter $(CMAKE_GENERATOR),make), \ $(MAKE), \ @@ -284,8 +312,10 @@ endif echo 1 > $@ LLVM_INSTALL = \ - cd $1 && mkdir -p $2$$(build_depsbindir) && \ - cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit $2$$(build_depsbindir)/ && \ + cd $1 && mkdir -p $2$$(build_depsbindir)/lit && \ + cp $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/*.py $2$$(build_depsbindir)/lit/ && \ + cp $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/*.toml $2$$(build_depsbindir)/lit/ && \ + cp -r $$(SRCCACHE)/$$(LLVM_SRC_DIR)/llvm/utils/lit/lit $2$$(build_depsbindir)/lit/ && \ $$(CMAKE) -DCMAKE_INSTALL_PREFIX="$2$$(build_prefix)" -P cmake_install.cmake ifeq ($(OS), WINNT) LLVM_INSTALL += && cp $2$$(build_shlibdir)/$(LLVM_SHARED_LIB_NAME).dll $2$$(build_depsbindir) @@ -343,6 +373,10 @@ $(eval $(call bb-install,lld,LLD,false,true)) $(eval $(call bb-install,clang,CLANG,false,true)) $(eval $(call bb-install,llvm-tools,LLVM_TOOLS,false,true)) +# work-around for Yggdrasil packaging bug (https://github.com/JuliaPackaging/Yggdrasil/pull/11231) +$(build_prefix)/manifest/llvm-tools uninstall-llvm-tools: \ + TAR:=$(TAR) --exclude=llvm-config.exe + endif # USE_BINARYBUILDER_LLVM get-lld: get-llvm diff --git a/deps/llvm.version b/deps/llvm.version index 194528a2643df..c68198b6d1729 100644 --- a/deps/llvm.version +++ b/deps/llvm.version @@ -2,14 +2,14 @@ ## jll artifact LLVM_JLL_NAME := libLLVM -LLVM_ASSERT_JLL_VER := 19.1.7+1 +LLVM_ASSERT_JLL_VER := 20.1.2+1 ## source build # Version number of LLVM -LLVM_VER := 19.1.7 +LLVM_VER := 20.1.2 # Git branch name in `LLVM_GIT_URL` repository -LLVM_BRANCH=julia-19.1.7-1 +LLVM_BRANCH=julia-20.1.2-0 # Git ref in `LLVM_GIT_URL` repository -LLVM_SHA1=julia-19.1.7-1 +LLVM_SHA1=julia-20.1.2-0 ## Following options are used to automatically fetch patchset from Julia's fork. This is ## useful if you want to build an external LLVM while still applying Julia's patches. @@ -18,6 +18,6 @@ LLVM_APPLY_JULIA_PATCHES := 0 # GitHub repository to use for fetching the Julia patches to apply to LLVM source code. LLVM_JULIA_DIFF_GITHUB_REPO := https://github.com/llvm/llvm-project # Base GitHub ref for generating the diff. -LLVM_BASE_REF := llvm:llvmorg-19.1.7 +LLVM_BASE_REF := llvm:llvmorg-20.1.2 # Julia fork's GitHub ref for generating the diff. -LLVM_JULIA_REF := JuliaLang:julia-19.1.7-1 +LLVM_JULIA_REF := JuliaLang:julia-20.1.2-0 diff --git a/deps/mpfr.version b/deps/mpfr.version index ec109e181ecdc..70585f95a6385 100644 --- a/deps/mpfr.version +++ b/deps/mpfr.version @@ -1,5 +1,6 @@ +# -*- makefile -*- ## jll artifact MPFR_JLL_NAME := MPFR ## source build -MPFR_VER := 4.2.1 +MPFR_VER := 4.2.2 diff --git a/deps/openssl.mk b/deps/openssl.mk index 6f96717b2fb74..ab6bd94921562 100644 --- a/deps/openssl.mk +++ b/deps/openssl.mk @@ -59,7 +59,7 @@ $(BUILDDIR)/openssl-$(OPENSSL_VER)/build-configured: $(SRCCACHE)/openssl-$(OPENS mkdir -p $(dir $@) cd $(dir $@) && \ CC="$(CC) $(SANITIZE_OPTS)" CXX="$(CXX) $(SANITIZE_OPTS)" LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN) $(SANITIZE_LDFLAGS)" \ - $(dir $<)/Configure shared --prefix=$(abspath $(build_prefix)) $(OPENSSL_TARGET) + $(dir $<)/Configure shared --prefix=$(abspath $(build_prefix)) --libdir=$(abspath $(build_libdir)) $(OPENSSL_TARGET) echo 1 > $@ $(BUILDDIR)/openssl-$(OPENSSL_VER)/build-compiled: $(BUILDDIR)/openssl-$(OPENSSL_VER)/build-configured @@ -72,14 +72,21 @@ ifeq ($(OS),$(BUILD_OS)) endif echo 1 > $@ +# Override bindir and only install runtime libraries, otherwise they'll go into build_depsbindir. +OPENSSL_INSTALL = \ + mkdir -p $2$$(build_shlibdir) && \ + $$(MAKE) -C $1 install_dev $$(MAKE_COMMON) bindir=$$(build_shlibdir) $3 DESTDIR="$2" + +OPENSSL_POST_INSTALL := \ + $(WIN_MAKE_HARD_LINK) $(build_bindir)/libcrypto-*.dll $(build_bindir)/libcrypto.dll && \ + $(WIN_MAKE_HARD_LINK) $(build_bindir)/libssl-*.dll $(build_bindir)/libssl.dll && \ + $(INSTALL_NAME_CMD)libcrypto.$(SHLIB_EXT) $(build_shlibdir)/libcrypto.$(SHLIB_EXT) && \ + $(INSTALL_NAME_CMD)libssl.$(SHLIB_EXT) $(build_shlibdir)/libssl.$(SHLIB_EXT) && \ + $(INSTALL_NAME_CHANGE_CMD) $(build_shlibdir)/libcrypto.3.dylib @rpath/libcrypto.$(SHLIB_EXT) $(build_shlibdir)/libssl.$(SHLIB_EXT) + $(eval $(call staged-install, \ openssl,openssl-$(OPENSSL_VER), \ - MAKE_INSTALL,,, \ - $$(WIN_MAKE_HARD_LINK) $(build_bindir)/libcrypto-*.dll $(build_bindir)/libcrypto.dll && \ - $$(WIN_MAKE_HARD_LINK) $(build_bindir)/libssl-*.dll $(build_bindir)/libssl.dll && \ - $$(INSTALL_NAME_CMD)libcrypto.$$(SHLIB_EXT) $$(build_shlibdir)/libcrypto.$$(SHLIB_EXT) && \ - $$(INSTALL_NAME_CMD)libssl.$$(SHLIB_EXT) $$(build_shlibdir)/libssl.$$(SHLIB_EXT) && \ - $$(INSTALL_NAME_CHANGE_CMD) $$(build_shlibdir)/libcrypto.3.dylib @rpath/libcrypto.$$(SHLIB_EXT) $$(build_shlibdir)/libssl.$$(SHLIB_EXT))) + OPENSSL_INSTALL,,,$(OPENSSL_POST_INSTALL))) clean-openssl: -rm -f $(BUILDDIR)/-openssl-$(OPENSSL_VER)/build-configured $(BUILDDIR)/-openssl-$(OPENSSL_VER)/build-compiled diff --git a/deps/openssl.version b/deps/openssl.version index 48831039a313e..2313ae5ffe116 100644 --- a/deps/openssl.version +++ b/deps/openssl.version @@ -3,4 +3,4 @@ OPENSSL_JLL_NAME := OpenSSL ## source build -OPENSSL_VER := 3.0.16 +OPENSSL_VER := 3.5.1 diff --git a/deps/p7zip.mk b/deps/p7zip.mk index c7c2874d49a5e..cbc850a1d5280 100644 --- a/deps/p7zip.mk +++ b/deps/p7zip.mk @@ -3,6 +3,8 @@ include $(SRCDIR)/p7zip.version ifneq ($(USE_BINARYBUILDER_P7ZIP),1) +P7ZIP_BUILD_OPTS := bindir=$(build_private_libexecdir) CC="$(CC)" CXX="$(CXX)" + $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz: | $(SRCCACHE) $(JLDOWNLOAD) $@ https://github.com/p7zip-project/p7zip/archive/refs/tags/v$(P7ZIP_VER).tar.gz @@ -17,12 +19,12 @@ checksum-p7zip: $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/source-extracted $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled: $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured - $(MAKE) -C $(dir $<) $(MAKE_COMMON) CC="$(CC)" CXX="$(CXX)" 7za + $(MAKE) -C $(dir $<) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) 7za echo 1 > $@ define P7ZIP_INSTALL - mkdir -p $2/$$(build_bindir) - cp -a $1/bin/7za $2/$$(build_bindir)/7z + mkdir -p $2/$$(build_private_libexecdir)/ + cp -a $1/bin/7za$(EXE) $2/$$(build_private_libexecdir)/7z$(EXE) endef $(eval $(call staged-install, \ p7zip,p7zip-$(P7ZIP_VER), \ @@ -30,8 +32,8 @@ $(eval $(call staged-install, \ clean-p7zip: -rm -f $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-configured $(BUILDDIR)/p7zip-$(P7ZIP_VER)/build-compiled - -rm -f $(build_bindir)/7za - -$(MAKE) -C $(BUILDDIR)/p7zip-$(P7ZIP_VER) clean + -rm -f $(build_bindir)/7z$(EXE) $(build_bindir)/7z$(EXE) $(build_private_libexecdir)/7z$(EXE) + -$(MAKE) -C $(BUILDDIR)/p7zip-$(P7ZIP_VER) $(MAKE_COMMON) $(P7ZIP_BUILD_OPTS) clean distclean-p7zip: rm -rf $(SRCCACHE)/p7zip-$(P7ZIP_VER).tar.gz $(SRCCACHE)/p7zip-$(P7ZIP_VER) $(BUILDDIR)/p7zip-$(P7ZIP_VER) @@ -48,5 +50,23 @@ check-p7zip: compile-p7zip else # USE_BINARYBUILDER_P7ZIP $(eval $(call bb-install,p7zip,P7ZIP,false)) +# move from bindir to shlibdir, where we expect to install it +install-p7zip: post-install-p7zip +uninstall-p7zip: pre-uninstall-p7zip +post-install-p7zip: $(build_prefix)/manifest/p7zip + mkdir -p $(build_private_libexecdir)/ + [ ! -e $(build_bindir)/7z$(EXE) ] || mv $(build_bindir)/7z$(EXE) $(build_private_libexecdir)/7z$(EXE) + [ -e $(build_private_libexecdir)/7z$(EXE) ] +ifeq ($(OS),WINNT) + [ ! -e $(build_bindir)/7z.dll ] || mv $(build_bindir)/7z.dll $(build_private_libexecdir)/7z.dll + [ -e $(build_private_libexecdir)/7z.dll ] +endif +pre-uninstall-p7zip: + -rm -f $(build_private_libexecdir)/7z$(EXE) +ifeq ($(OS),WINNT) + -rm -f $(build_private_libexecdir)/7z.dll +endif + +.PHONY: post-install-p7zip pre-uninstall-p7zip endif diff --git a/deps/patchelf.mk b/deps/patchelf.mk index c019892058d0e..aaf0ecb313b80 100644 --- a/deps/patchelf.mk +++ b/deps/patchelf.mk @@ -7,7 +7,7 @@ $(SRCCACHE)/patchelf-$(PATCHELF_VER).tar.bz2: | $(SRCCACHE) $(SRCCACHE)/patchelf-$(PATCHELF_VER)/source-extracted: $(SRCCACHE)/patchelf-$(PATCHELF_VER).tar.bz2 $(JLCHECKSUM) $< mkdir $(dir $@) - cd $(dir $@) && $(TAR) jxf $< --strip-components=1 + cd $(dir $@) && $(TAR) -jxf $< --strip-components=1 touch -c $(SRCCACHE)/patchelf-$(PATCHELF_VER)/configure # old target echo 1 > $@ diff --git a/deps/patches/libunwind-aarch64-inline-asm.patch b/deps/patches/libunwind-aarch64-inline-asm.patch deleted file mode 100644 index 123643e30cdeb..0000000000000 --- a/deps/patches/libunwind-aarch64-inline-asm.patch +++ /dev/null @@ -1,157 +0,0 @@ -From 6ae71b3ea71bff0f38c7a6a05beda30b7dce1ef6 Mon Sep 17 00:00:00 2001 -From: Stephen Webb -Date: Mon, 22 Apr 2024 15:56:54 -0400 -Subject: [PATCH] Rework inline aarch64 as for setcontext - -Modern GC and clang were barfing on the inline asm constraints for the -aarch64-linux setcontext() replacement. Reformulated the asm code to -reduce the required constraints. ---- - src/aarch64/Gos-linux.c | 115 +++++++++++++++++++++------------------- - 1 file changed, 61 insertions(+), 54 deletions(-) - -diff --git a/src/aarch64/Gos-linux.c b/src/aarch64/Gos-linux.c -index 7cd8c879f..1e4949623 100644 ---- a/src/aarch64/Gos-linux.c -+++ b/src/aarch64/Gos-linux.c -@@ -2,6 +2,7 @@ - Copyright (C) 2008 CodeSourcery - Copyright (C) 2011-2013 Linaro Limited - Copyright (C) 2012 Tommi Rantala -+ Copyright 2024 Stephen M. Webb - - This file is part of libunwind. - -@@ -28,6 +29,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ - - #ifndef UNW_REMOTE_ONLY - -+/* Magic constants generated from gen-offsets.c */ -+#define SC_R0_OFF "8" -+#define SC_R2_OFF "24" -+#define SC_R18_OFF "152" -+#define SC_R20_OFF "168" -+#define SC_R22_OFF "184" -+#define SC_R24_OFF "200" -+#define SC_R26_OFF "216" -+#define SC_R28_OFF "232" -+#define SC_R30_OFF "248" -+ -+#define FP_R08_OFF "80" -+#define FP_R09_OFF "88" -+#define FP_R10_OFF "96" -+#define FP_R11_OFF "104" -+#define FP_R12_OFF "112" -+#define FP_R13_OFF "120" -+#define FP_R14_OFF "128" -+#define FP_R15_OFF "136" -+ -+#define SC_SP_OFF "0x100" -+ - HIDDEN int - aarch64_local_resume (unw_addr_space_t as, unw_cursor_t *cursor, void *arg) - { -@@ -36,65 +59,49 @@ aarch64_local_resume (unw_addr_space_t as, unw_cursor_t *cursor, void *arg) - - if (c->sigcontext_format == AARCH64_SCF_NONE) - { -+ -+ /* -+ * This is effectively the old POSIX setcontext(). -+ * -+ * This inline asm is broken up to use local scratch registers for the -+ * uc_mcontext.regs and FPCTX base addresses because newer versions of GCC -+ * and clang barf on too many constraints (gh-702) when the C array -+ * elements are used directly. -+ * -+ * Clobbers aren't required for the inline asm because they just convince -+ * the compiler to save those registers and they never get restored -+ * becauise the asm ends with a plain ol' ret. -+ */ -+ register void* uc_mcontext __asm__ ("x5") = (void*) &uc->uc_mcontext; -+ register void* fpctx __asm__ ("x4") = (void*) GET_FPCTX(uc); -+ - /* Since there are no signals involved here we restore EH and non scratch - registers only. */ - __asm__ __volatile__ ( -- "ldr x0, %[x0]\n\t" -- "ldr x1, %[x1]\n\t" -- "ldr x2, %[x2]\n\t" -- "ldr x3, %[x3]\n\t" -- "ldr x19, %[x19]\n\t" -- "ldr x20, %[x20]\n\t" -- "ldr x21, %[x21]\n\t" -- "ldr x22, %[x22]\n\t" -- "ldr x23, %[x23]\n\t" -- "ldr x24, %[x24]\n\t" -- "ldr x25, %[x25]\n\t" -- "ldr x26, %[x26]\n\t" -- "ldr x27, %[x27]\n\t" -- "ldr x28, %[x28]\n\t" -- "ldr x29, %[x29]\n\t" -- "ldr x30, %[x30]\n\t" -- "ldr d8, %[d8]\n\t" -- "ldr d9, %[d9]\n\t" -- "ldr d10, %[d10]\n\t" -- "ldr d11, %[d11]\n\t" -- "ldr d12, %[d12]\n\t" -- "ldr d13, %[d13]\n\t" -- "ldr d14, %[d14]\n\t" -- "ldr d15, %[d15]\n\t" -- "ldr x5, %[sp]\n\t" -+ "ldp x0, x1, [x5, " SC_R0_OFF "]\n\t" -+ "ldp x2, x3, [x5, " SC_R2_OFF "]\n\t" -+ "ldp x18, x19, [x5, " SC_R18_OFF "]\n\t" -+ "ldp x20, x21, [x5, " SC_R20_OFF "]\n\t" -+ "ldp x22, x23, [x5, " SC_R22_OFF "]\n\t" -+ "ldp x24, x25, [x5, " SC_R24_OFF "]\n\t" -+ "ldp x26, x27, [x5, " SC_R26_OFF "]\n\t" -+ "ldp x28, x29, [x5, " SC_R28_OFF "]\n\t" -+ "ldr x30, [x5, " SC_R30_OFF "]\n\t" -+ "ldr d8, [x4, " FP_R08_OFF "]\n\t" -+ "ldr d9, [x4, " FP_R09_OFF "]\n\t" -+ "ldr d10, [x4, " FP_R10_OFF "]\n\t" -+ "ldr d11, [x4, " FP_R11_OFF "]\n\t" -+ "ldr d12, [x4, " FP_R12_OFF "]\n\t" -+ "ldr d13, [x4, " FP_R13_OFF "]\n\t" -+ "ldr d14, [x4, " FP_R14_OFF "]\n\t" -+ "ldr d15, [x4, " FP_R15_OFF "]\n\t" -+ "ldr x5, [x5, " SC_SP_OFF "]\n\t" - "mov sp, x5\n\t" - "ret\n" -- : -- : [x0] "m"(uc->uc_mcontext.regs[0]), -- [x1] "m"(uc->uc_mcontext.regs[1]), -- [x2] "m"(uc->uc_mcontext.regs[2]), -- [x3] "m"(uc->uc_mcontext.regs[3]), -- [x19] "m"(uc->uc_mcontext.regs[19]), -- [x20] "m"(uc->uc_mcontext.regs[20]), -- [x21] "m"(uc->uc_mcontext.regs[21]), -- [x22] "m"(uc->uc_mcontext.regs[22]), -- [x23] "m"(uc->uc_mcontext.regs[23]), -- [x24] "m"(uc->uc_mcontext.regs[24]), -- [x25] "m"(uc->uc_mcontext.regs[25]), -- [x26] "m"(uc->uc_mcontext.regs[26]), -- [x27] "m"(uc->uc_mcontext.regs[27]), -- [x28] "m"(uc->uc_mcontext.regs[28]), -- [x29] "m"(uc->uc_mcontext.regs[29]), /* FP */ -- [x30] "m"(uc->uc_mcontext.regs[30]), /* LR */ -- [d8] "m"(GET_FPCTX(uc)->vregs[8]), -- [d9] "m"(GET_FPCTX(uc)->vregs[9]), -- [d10] "m"(GET_FPCTX(uc)->vregs[10]), -- [d11] "m"(GET_FPCTX(uc)->vregs[11]), -- [d12] "m"(GET_FPCTX(uc)->vregs[12]), -- [d13] "m"(GET_FPCTX(uc)->vregs[13]), -- [d14] "m"(GET_FPCTX(uc)->vregs[14]), -- [d15] "m"(GET_FPCTX(uc)->vregs[15]), -- [sp] "m"(uc->uc_mcontext.sp) -- : "x0", "x1", "x2", "x3", "x19", "x20", "x21", "x22", "x23", "x24", -- "x25", "x26", "x27", "x28", "x29", "x30" -- ); -+ : -+ : [uc_mcontext] "r"(uc_mcontext), -+ [fpctx] "r"(fpctx) -+ ); - } - else - { diff --git a/deps/patches/libunwind-configure-static-lzma.patch b/deps/patches/libunwind-configure-static-lzma.patch index f8b428f60550b..16991d5cd2656 100644 --- a/deps/patches/libunwind-configure-static-lzma.patch +++ b/deps/patches/libunwind-configure-static-lzma.patch @@ -1,20 +1,20 @@ ---- configure.orig 2023-06-04 05:19:04 -+++ configure 2023-06-07 08:35:11 -@@ -18117,7 +18117,7 @@ - $as_echo_n "(cached) " >&6 - else +--- configure.orig 2025-05-26 15:25:01 ++++ configure 2025-05-26 15:25:41 +@@ -20878,7 +20878,7 @@ + printf %s "(cached) " >&6 + else $as_nop ac_check_lib_save_LIBS=$LIBS -LIBS="-llzma $LIBS" -+LIBS="-L${libdir} -l:liblzma.a $LIBS" ++LIBS="-L${libdir} -l:liblzma.a $LIBS" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -@@ -18148,7 +18148,7 @@ - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lzma_lzma_mf_is_supported" >&5 - $as_echo "$ac_cv_lib_lzma_lzma_mf_is_supported" >&6; } - if test "x$ac_cv_lib_lzma_lzma_mf_is_supported" = xyes; then : +@@ -20908,7 +20908,7 @@ + printf "%s\n" "$ac_cv_lib_lzma_lzma_mf_is_supported" >&6; } + if test "x$ac_cv_lib_lzma_lzma_mf_is_supported" = xyes + then : - LIBLZMA=-llzma + LIBLZMA="-L${libdir} -l:liblzma.a" - $as_echo "#define HAVE_LZMA 1" >>confdefs.h + printf "%s\n" "#define HAVE_LZMA 1" >>confdefs.h diff --git a/deps/patches/libunwind-missing-parameter-names.patch b/deps/patches/libunwind-missing-parameter-names.patch new file mode 100644 index 0000000000000..59a33f1a56880 --- /dev/null +++ b/deps/patches/libunwind-missing-parameter-names.patch @@ -0,0 +1,28 @@ +From 9ebe5e12b8a0063953f9ef196b2433eca9933559 Mon Sep 17 00:00:00 2001 +From: Stephen Webb +Date: Tue, 15 Apr 2025 10:48:20 -0400 +Subject: [PATCH] Fix FTBFS in src/ptrace/_UPT_ptrauth_insn_mask.c + +Added missing parameter names to make C code comply to ISO/IEC 9899. +--- + src/ptrace/_UPT_ptrauth_insn_mask.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/src/ptrace/_UPT_ptrauth_insn_mask.c b/src/ptrace/_UPT_ptrauth_insn_mask.c +index dcc512370..e7b3a514b 100644 +--- a/src/ptrace/_UPT_ptrauth_insn_mask.c ++++ b/src/ptrace/_UPT_ptrauth_insn_mask.c +@@ -49,9 +49,10 @@ unw_word_t _UPT_ptrauth_insn_mask (UNUSED unw_addr_space_t as, void *arg) + + #else + +-unw_word_t _UPT_ptrauth_insn_mask (unw_addr_space_t, void *) ++unw_word_t _UPT_ptrauth_insn_mask (UNUSED unw_addr_space_t as, UNUSED void *arg) + { + return 0; + } + +-#endif +\ No newline at end of file ++#endif ++ diff --git a/deps/pcre.mk b/deps/pcre.mk index 3ff85d5569ad9..8fbdb1ba79a35 100644 --- a/deps/pcre.mk +++ b/deps/pcre.mk @@ -21,7 +21,7 @@ $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2: | $(SRCCACHE) $(SRCCACHE)/pcre2-$(PCRE_VER)/source-extracted: $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2 $(JLCHECKSUM) $< - cd $(dir $<) && $(TAR) jxf $(notdir $<) + cd $(dir $<) && $(TAR) -jxf $(notdir $<) echo 1 > $@ checksum-pcre: $(SRCCACHE)/pcre2-$(PCRE_VER).tar.bz2 diff --git a/deps/pcre.version b/deps/pcre.version index 78245a5777a0c..6810ee36fe83f 100644 --- a/deps/pcre.version +++ b/deps/pcre.version @@ -3,4 +3,4 @@ PCRE_JLL_NAME := PCRE2 ## source build -PCRE_VER := 10.44 +PCRE_VER := 10.45 diff --git a/deps/sanitizers.mk b/deps/sanitizers.mk index ace86c16bcefe..2b685b8d80aef 100644 --- a/deps/sanitizers.mk +++ b/deps/sanitizers.mk @@ -17,10 +17,11 @@ install-sanitizers: $$(addprefix $$(build_libdir)/, $$(notdir $$(call pathsearch echo "Sanitizer library $(1) not found in $$(SANITIZER_LIB_PATH)"; \ exit 1; \ fi -$$(addprefix $$(build_shlibdir)/,$(2)): $$(addprefix $$(dir $$(call pathsearch_all,$(1),$$(SANITIZER_LIB_PATH))),$(2)) | $$(build_shlibdir) +$$(addprefix $$(build_shlibdir)/,$(2)): $$(addprefix $$(dir $$(call pathsearch_all,$(1),$$(SANITIZER_LIB_PATH))),$(2)) $$(PATCHELF_MANIFEST) | $$(build_shlibdir) -cp $$< $$@ - $(if $(filter $(OS), Linux), \ - -$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$$$ORIGIN' $$@ , 0) +ifneq (,$(findstring $(OS),Linux)) + -$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$$$ORIGIN' $$@ +endif endef ifeq ($(USECLANG),1) diff --git a/deps/tools/bb-install.mk b/deps/tools/bb-install.mk index ee7f833a8ac2b..66e3d716c90ff 100644 --- a/deps/tools/bb-install.mk +++ b/deps/tools/bb-install.mk @@ -55,9 +55,9 @@ ifneq (bsdtar,$(findstring bsdtar,$(TAR_TEST))) @# work-around a gtar bug: they do some complicated work to avoid the mkdir @# syscall, which is buggy when working with Tar.jl files so we manually do @# the mkdir calls first in a pre-pass - $(TAR) -tzf $$< | xargs -n 1 dirname | sort -u | (cd $$(build_prefix) && xargs -t mkdir -p) + $$(TAR) -tzf $$< | xargs -n 1 dirname | sort -u | (cd $$(build_prefix) && xargs -t mkdir -p) endif - $(UNTAR) $$< -C $$(build_prefix) + $$(UNTAR) $$< -C $$(build_prefix) echo '$$(UNINSTALL_$(strip $1))' > $$@ # Special "checksum-foo" target to speed up `contrib/refresh_checksums.sh` diff --git a/deps/tools/common.mk b/deps/tools/common.mk index 01b57316f9d1a..b7ebc39169221 100644 --- a/deps/tools/common.mk +++ b/deps/tools/common.mk @@ -5,8 +5,15 @@ # apparently not on FreeBSD). Ref PR #22352 CONFIGURE_COMMON = --prefix=$(abspath $(build_prefix)) --build=$(BUILD_MACHINE) --libdir=$(abspath $(build_libdir)) --bindir=$(abspath $(build_depsbindir)) $(CUSTOM_LD_LIBRARY_PATH) + +CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) +CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir) + ifneq ($(XC_HOST),) CONFIGURE_COMMON += --host=$(XC_HOST) +else +# Defeat bad automatic cross compile detection (e.g. clang on mingw) +# CMAKE_COMMON += -DCMAKE_CROSSCOMPILING=0 endif ifeq ($(OS),WINNT) CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) -Wl,--stack,8388608" @@ -15,8 +22,6 @@ CONFIGURE_COMMON += LDFLAGS="$(LDFLAGS) $(RPATH_ESCAPED_ORIGIN) $(SANITIZE_LDFLA endif CONFIGURE_COMMON += F77="$(FC)" CC="$(CC) $(SANITIZE_OPTS)" CXX="$(CXX) $(SANITIZE_OPTS)" LD="$(LD)" -CMAKE_COMMON := -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) -DCMAKE_PREFIX_PATH=$(build_prefix) -CMAKE_COMMON += -DLIB_INSTALL_DIR=$(build_shlibdir) ifneq ($(OS),WINNT) CMAKE_COMMON += -DCMAKE_INSTALL_LIBDIR=$(build_libdir) endif @@ -38,6 +43,12 @@ CMAKE_CC := "$$(which $(shell echo $(CC_ARG) | cut -d' ' -f1))" CMAKE_CXX := "$$(which $(shell echo $(CXX_ARG) | cut -d' ' -f1))" CMAKE_CC_ARG := $(shell echo $(CC_ARG) | cut -s -d' ' -f2-) CMAKE_CXX_ARG := $(shell echo $(CXX_ARG) | cut -s -d' ' -f2-) +else ifneq (,$(findstring MINGW,$(RAW_BUILD_OS))) +# `cmake` is mingw-native and needs `cygpath -w`, rather than `cygpath -m`, which is the msys2 conversion default +CMAKE_CC := "$(shell echo $(call cygpath_w, $(shell which $(CC_BASE))))" +CMAKE_CXX := "$(shell echo $(call cygpath_w, $(shell which $(CXX_BASE))))" +CMAKE_CC_ARG := $(CC_ARG) +CMAKE_CXX_ARG := $(CXX_ARG) else CMAKE_CC := "$$(which $(CC_BASE))" CMAKE_CXX := "$$(which $(CXX_BASE))" @@ -55,11 +66,15 @@ endif CMAKE_COMMON += -DCMAKE_LINKER="$$(which $(LD))" -DCMAKE_AR="$$(which $(AR))" -DCMAKE_RANLIB="$$(which $(RANLIB))" ifeq ($(OS),WINNT) +ifeq ($(BUILD_OS),WINNT) +# Don't make CMake think we're cross compiling, but do make sure it knows we're Windows +CMAKE_COMMON += -DCMAKE_HOST_SYSTEM_NAME=Windows +else CMAKE_COMMON += -DCMAKE_SYSTEM_NAME=Windows +endif CMAKE_COMMON += -DCMAKE_RC_COMPILER="$$(which $(CROSS_COMPILE)windres)" endif -# For now this is LLVM specific, but I expect it won't be in the future ifeq ($(CMAKE_GENERATOR),Ninja) CMAKE_GENERATOR_COMMAND := -G Ninja else ifeq ($(CMAKE_GENERATOR),make) @@ -68,6 +83,27 @@ else $(error Unknown CMake generator '$(CMAKE_GENERATOR)'. Options are 'Ninja' and 'make') endif +ifneq (,$(findstring MINGW,$(RAW_BUILD_OS))) +ifneq (,$(shell ldd $(shell which cmake) | grep msys-2.0.dll)) +# Detect MSYS2 with cygwin CMake rather than MinGW cmake - the former fails to +# properly drive MinGW tools +override CMAKE := echo "ERROR: CMake is Cygwin CMake, not MinGW CMake. Build will fail. Use 'pacman -S mingw-w64-{i686,x86_64}-cmake'."; exit 1; $(CMAKE) +endif +# In our setup, CMAKE_INSTALL_PREFIX is a relative path inside usr-staging. +# We do not want this converted to a windows path, because our make system +# assumes it to be relative to msys `/`. +override CMAKE := MSYS2_ARG_CONV_EXCL="-DCMAKE_INSTALL_PREFIX" $(CMAKE) +endif + +# Some dependencies' tarballs contains symlinks to non-existent targets. This breaks the +# the default msys strategy `deepcopy` symlink strategy. To workaround this, +# switch to `native` which tries native windows symlinks (possible if the +# machine is in developer mode) - or if not, falls back to cygwin-style +# symlinks. We don't particularly care either way - we just need to symlinks +# to succeed. We could guard this by a uname check, but it's harmless elsewhere, +# so let's not incur the additional overhead. +MSYS_NONEXISTENT_SYMLINK_TARGET_FIX := winsymlinks:native + # If the top-level Makefile is called with environment variables, # they will override the values passed above to ./configure MAKE_COMMON := DESTDIR="" prefix=$(build_prefix) bindir=$(build_depsbindir) libdir=$(build_libdir) shlibdir=$(build_shlibdir) libexecdir=$(build_libexecdir) datarootdir=$(build_datarootdir) includedir=$(build_includedir) sysconfdir=$(build_sysconfdir) O= @@ -144,7 +180,7 @@ upper = $(shell echo $1 | tr a-z A-Z) # this rule ensures that make install is more nearly atomic # so it's harder to get half-installed (or half-reinstalled) dependencies # # and enables sharing deps compiles, uninstall, and fast reinstall -MAKE_INSTALL = +$$(MAKE) -C $1 install $$(MAKE_COMMON) $3 DESTDIR="$2" +MAKE_INSTALL = MSYS2_ARG_CONV_EXCL="prefix=" $$(MAKE) -C $1 install $$(MAKE_COMMON) $3 DESTDIR="$2" define SHLIBFILE_INSTALL mkdir -p $2/$$(build_shlibdir) @@ -187,7 +223,7 @@ UNINSTALL_$(strip $1) := $2 staged-uninstaller $$(build_prefix)/manifest/$(strip $1): $$(build_staging)/$2.tar | $(build_prefix)/manifest -+[ ! -e $$@ ] || $$(MAKE) uninstall-$(strip $1) - $(UNTAR) $$< -C $$(build_prefix) + $$(UNTAR) $$< -C $$(build_prefix) $6 echo '$$(UNINSTALL_$(strip $1))' > $$@ .PHONY: $(addsuffix -$(strip $1),stage install distclean uninstall reinstall) diff --git a/deps/unwind.mk b/deps/unwind.mk index d144f4d0a509b..2b49b3cf0973a 100644 --- a/deps/unwind.mk +++ b/deps/unwind.mk @@ -20,7 +20,7 @@ $(SRCCACHE)/libunwind-$(UNWIND_VER).tar.gz: | $(SRCCACHE) $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted: $(SRCCACHE)/libunwind-$(UNWIND_VER).tar.gz $(JLCHECKSUM) $< - cd $(dir $<) && $(TAR) xfz $< + cd $(dir $<) && $(TAR) -xzf $< touch -c $(SRCCACHE)/libunwind-$(UNWIND_VER)/configure # old target echo 1 > $@ @@ -35,21 +35,21 @@ $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-revert_prelink_unwind.patch-applie cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-revert_prelink_unwind.patch echo 1 > $@ -$(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-aarch64-inline-asm.patch-applied: $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-revert_prelink_unwind.patch-applied - cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-aarch64-inline-asm.patch +$(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied: $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-revert_prelink_unwind.patch-applied + cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-disable-initial-exec-tls.patch echo 1 > $@ -$(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied: $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-aarch64-inline-asm.patch-applied - cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-disable-initial-exec-tls.patch +$(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-missing-parameter-names.patch-applied: $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied + cd $(SRCCACHE)/libunwind-$(UNWIND_VER) && patch -p1 -f -u -l < $(SRCDIR)/patches/libunwind-missing-parameter-names.patch echo 1 > $@ # note minidebuginfo requires liblzma, which we do not have a source build for # (it will be enabled in BinaryBuilder-based downloads however) # since https://github.com/JuliaPackaging/Yggdrasil/commit/0149e021be9badcb331007c62442a4f554f3003c -$(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured: $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-disable-initial-exec-tls.patch-applied +$(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured: $(SRCCACHE)/libunwind-$(UNWIND_VER)/source-extracted $(SRCCACHE)/libunwind-$(UNWIND_VER)/libunwind-missing-parameter-names.patch-applied mkdir -p $(dir $@) cd $(dir $@) && \ - $(dir $<)/configure $(CONFIGURE_COMMON) CPPFLAGS="$(CPPFLAGS) $(LIBUNWIND_CPPFLAGS)" CFLAGS="$(CFLAGS) $(LIBUNWIND_CFLAGS)" LDFLAGS="$(LDFLAGS) $(LIBUNWIND_LDFLAGS)" --enable-shared --disable-minidebuginfo --disable-tests --enable-zlibdebuginfo --disable-conservative-checks --enable-per-thread-cache + $(dir $<)/configure $(CONFIGURE_COMMON) CPPFLAGS="$(CPPFLAGS) $(LIBUNWIND_CPPFLAGS)" CFLAGS="$(CFLAGS) $(LIBUNWIND_CFLAGS)" LDFLAGS="$(LDFLAGS) $(LIBUNWIND_LDFLAGS)" --enable-shared --disable-minidebuginfo --disable-tests --enable-debug_frame --enable-zlibdebuginfo --disable-conservative-checks --enable-per-thread-cache echo 1 > $@ $(BUILDDIR)/libunwind-$(UNWIND_VER)/build-compiled: $(BUILDDIR)/libunwind-$(UNWIND_VER)/build-configured @@ -102,7 +102,7 @@ $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER).tar.xz: | $(SRCCACHE) $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER)/source-extracted: $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER).tar.xz $(JLCHECKSUM) $< - cd $(dir $<) && $(TAR) xf $< + cd $(dir $<) && $(TAR) -xf $< mv $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER).src $(SRCCACHE)/llvm-project-$(LLVMUNWIND_VER) echo 1 > $@ diff --git a/deps/unwind.version b/deps/unwind.version index e3ed63675fd8c..d2709230d18fc 100644 --- a/deps/unwind.version +++ b/deps/unwind.version @@ -1,6 +1,8 @@ +# -*- makefile -*- + ## jll artifact UNWIND_JLL_NAME := LibUnwind ## source build -UNWIND_VER_TAG := 1.8.1 -UNWIND_VER := 1.8.1 +UNWIND_VER_TAG := 1.8.2 +UNWIND_VER := 1.8.2 diff --git a/deps/zlib.mk b/deps/zlib.mk index 5548a0791f4d2..347b8d4cf53b6 100644 --- a/deps/zlib.mk +++ b/deps/zlib.mk @@ -10,7 +10,7 @@ ZLIB_BUILD_OPTS += -DCMAKE_POSITION_INDEPENDENT_CODE=ON $(BUILDDIR)/$(ZLIB_SRC_DIR)/build-configured: $(SRCCACHE)/$(ZLIB_SRC_DIR)/source-extracted mkdir -p $(dir $@) - cd $(dir $@) && $(CMAKE) $(ZLIB_BUILD_OPTS) $(dir $<) + cd $(dir $@) && $(CMAKE) -G"Unix Makefiles" $(ZLIB_BUILD_OPTS) $(dir $<) echo 1 > $@ $(BUILDDIR)/$(ZLIB_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ZLIB_SRC_DIR)/build-configured diff --git a/deps/zstd.mk b/deps/zstd.mk new file mode 100644 index 0000000000000..ecce416ab3f38 --- /dev/null +++ b/deps/zstd.mk @@ -0,0 +1,66 @@ +## Zstd ## +ifneq ($(USE_BINARYBUILDER_ZSTD), 1) +ZSTD_GIT_URL := https://github.com/facebook/zstd.git +ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1 +$(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR))) +$(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted: export MSYS=$(MSYS_NONEXISTENT_SYMLINK_TARGET_FIX) + +ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)" bindir=$(build_private_libexecdir) +ifeq ($(OS), WINNT) +# Zstd detects "Windows" not WINNT, ordinarily from the inherited $(OS), but it expects the +# override to be done using TARGET_SYSTEM. +ZSTD_BUILD_OPTS += TARGET_SYSTEM="Windows" +endif + +$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted + echo 1 > $@ + +$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured + $(MAKE) -C $(dir $<) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) + echo 1 > $@ + +$(eval $(call staged-install, \ + zstd,$(ZSTD_SRC_DIR), \ + MAKE_INSTALL,$(ZSTD_BUILD_OPTS) MT=1,, \ + $(INSTALL_NAME_CMD)libzstd.$(SHLIB_EXT) $(build_private_libexecdir)/libzstd.$(SHLIB_EXT))) + +clean-zstd: + -rm -f $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled + -$(MAKE) -C $(BUILDDIR)/$(ZSTD_SRC_DIR) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) clean + +get-zstd: $(ZSTD_SRC_FILE) +extract-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted +configure-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured +compile-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled +fastcheck-zstd: check-zstd +check-zstd: compile-zstd + +else # USE_BINARYBUILDER_ZSTD + +$(eval $(call bb-install,zstd,ZSTD,false)) +# move from bindir to shlibdir, where we expect to install it +install-zstd: post-install-zstd +uninstall-zstd: pre-uninstall-zstd +post-install-zstd: $(build_prefix)/manifest/zstd $(PATCHELF_MANIFEST) + mkdir -p $(build_private_libexecdir)/ + [ ! -e $(build_bindir)/zstdmt$(EXE) ] || mv $(build_bindir)/zstdmt$(EXE) $(build_private_libexecdir)/zstdmt$(EXE) + [ ! -e $(build_bindir)/zstd$(EXE) ] || mv $(build_bindir)/zstd$(EXE) $(build_private_libexecdir)/zstd$(EXE) + [ -e $(build_private_libexecdir)/zstd$(EXE) ] + [ -e $(build_private_libexecdir)/zstdmt$(EXE) ] +ifeq ($(OS), Darwin) + for j in zstd zstdmt ; do \ + [ -L $(build_private_libexecdir)/$$j ] && continue; \ + install_name_tool -rpath @executable_path/$(reverse_build_private_libexecdir_rel) @loader_path/$(build_libdir_rel) $(build_private_libexecdir)/$$j 2>/dev/null || true; \ + install_name_tool -rpath @loader_path/$(build_libdir_rel) @executable_path/$(reverse_build_private_libexecdir_rel) $(build_private_libexecdir)/$$j || exit 1; \ + done +else ifneq (,$(findstring $(OS),Linux FreeBSD)) + for j in zstd zstdmt ; do \ + [ -L $(build_private_libexecdir)/$$j ] && continue; \ + $(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_build_private_libexecdir_rel)' $(build_private_libexecdir)/$$j || exit 1; \ + done +endif + +pre-uninstall-zstd: + -rm -f $(build_private_libexecdir)/zstd$(EXE) $(build_private_libexecdir)/zstdmt$(EXE) + +endif # USE_BINARYBUILDER_ZSTD diff --git a/deps/zstd.version b/deps/zstd.version new file mode 100644 index 0000000000000..d4d960aa6f04b --- /dev/null +++ b/deps/zstd.version @@ -0,0 +1,8 @@ +# -*- makefile -*- +## jll artifact +ZSTD_JLL_NAME := Zstd + +## source build +ZSTD_VER := 1.5.7 +ZSTD_BRANCH=v1.5.7 +ZSTD_SHA1=f8745da6ff1ad1e7bab384bd1f9d742439278e99 diff --git a/doc/Makefile b/doc/Makefile index 4469a40f74248..177eeac5e6a85 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -21,19 +21,29 @@ help: @echo "To fix outdated doctests, use 'make doctest=fix'" @echo "To run doctests using Revise (to test changes without rebuilding the sysimage), use 'make doctest=true revise=true'" - +VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) DOCUMENTER_OPTIONS := linkcheck=$(linkcheck) doctest=$(doctest) buildroot=$(call cygpath_w,$(BUILDROOT)) \ - texplatform=$(texplatform) revise=$(revise) + texplatform=$(texplatform) revise=$(revise) stdlibdir=$(call cygpath_w,$(build_datarootdir)/julia/stdlib/$(VERSDIR)/) -UNICODE_DATA_VERSION=13.0.0 +UNICODE_DATA_VERSION=16.0.0 $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt: @mkdir -p "$(SRCCACHE)" $(JLDOWNLOAD) "$@" https://www.unicode.org/Public/$(UNICODE_DATA_VERSION)/ucd/UnicodeData.txt +# NEWS.md and stdlib are in-tree build artifacts - don't link them for oot builds. +DOC_FILES=$(filter-out NEWS.md stdlib,$(notdir $(wildcard $(SRCDIR)/src/*))) +src/%: + @mkdir -p src + ln -s $(SRCDIR)/src/$* $@ +src: $(addprefix src/,$(DOC_FILES)) + deps: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt $(JLCHECKSUM) "$<" cp "$<" UnicodeData.txt +alldeps: deps + $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) deps $(DOCUMENTER_OPTIONS) + checksum-unicodedata: $(SRCCACHE)/UnicodeData-$(UNICODE_DATA_VERSION).txt $(JLCHECKSUM) "$<" @@ -57,3 +67,7 @@ deploy: deps @echo "Deploying HTML documentation." $(JULIA_EXECUTABLE) --color=yes $(call cygpath_w,$(SRCDIR)/make.jl) -- deploy $(DOCUMENTER_OPTIONS) @echo "Build & deploy of docs finished." + +update-documenter: + @echo "Updating Documenter." + JULIA_PKG_PRECOMPILE_AUTO=0 $(JULIA_EXECUTABLE) --project=$(call cygpath_w,$(SRCDIR)/../deps/jlutilities/documenter/) --color=yes -e 'using Pkg; Pkg.update("Documenter")' diff --git a/doc/README.md b/doc/README.md index be5426018084d..d282485bc584a 100644 --- a/doc/README.md +++ b/doc/README.md @@ -27,3 +27,11 @@ $ make -C doc doctest=true ``` from the root directory. + +## Customizing Doctest Execution + +By default, doctests are run using the in-tree Julia executable. +This behavior can be changed by setting the `JULIA_EXECUTABLE` Makefile variable. + +> [!WARNING] +> Using a custom `JULIA_EXECUTABLE` will not pick up changes to docstrings for Base or any standard library built into the system image. To see the list of standard libraries that are part of the system image, you can run the `contrib/print_sorted_stdlibs.jl` script (e.g., `julia contrib/print_sorted_stdlibs.jl --only-sysimg`). diff --git a/doc/make.jl b/doc/make.jl index 068531be24a1d..2604c801f19aa 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -1,13 +1,39 @@ +# Get the buildroot and stdlibdir from the make environment to make sure we're +# generating docs for the current julia source tree, regardless of what julia +# executable we're using. If these arguments are not passed, fall back to +# assuming that we're running a just-built version of julia and generating docs +# in tree. +let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS) + if i === nothing + global const buildrootdoc = @__DIR__ + global const buildroot = abspath(joinpath(buildrootdoc, "..")) + else + global const buildroot = first(match(r, ARGS[i]).captures) + global const buildrootdoc = joinpath(buildroot, "doc") + end +end + +let r = r"stdlibdir=(.+)", i = findfirst(x -> occursin(r, x), ARGS) + if i === nothing + global const STDLIB_DIR = Sys.STDLIB + else + global const STDLIB_DIR = first(match(r, ARGS[i]).captures) + end +end + # Install dependencies needed to build the documentation. -Base.ACTIVE_PROJECT[] = nothing -empty!(LOAD_PATH) -push!(LOAD_PATH, @__DIR__, "@stdlib") +documenter_project_dir = joinpath(@__DIR__, "..", "deps", "jlutilities", "documenter") empty!(DEPOT_PATH) -push!(DEPOT_PATH, joinpath(@__DIR__, "deps")) +push!(DEPOT_PATH, joinpath(buildroot, "deps", "jlutilities", "depot")) push!(DEPOT_PATH, abspath(Sys.BINDIR, "..", "share", "julia")) using Pkg +Pkg.activate(documenter_project_dir) Pkg.instantiate() +if "deps" in ARGS + exit() +end + using Documenter import LibGit2 @@ -20,9 +46,8 @@ cp_q(src, dest) = isfile(dest) || cp(src, dest) # make links for stdlib package docs, this is needed until #552 in Documenter.jl is finished const STDLIB_DOCS = [] -const STDLIB_DIR = Sys.STDLIB const EXT_STDLIB_DOCS = ["Pkg"] -cd(joinpath(@__DIR__, "src")) do +cd(joinpath(buildrootdoc, "src")) do Base.rm("stdlib"; recursive=true, force=true) mkdir("stdlib") for dir in readdir(STDLIB_DIR) @@ -66,7 +91,8 @@ function parse_stdlib_version_file(path) end # This generates the value that will be passed to the `remotes` argument of makedocs(), # by looking through all *.version files in stdlib/. -documenter_stdlib_remotes = let stdlib_dir = realpath(joinpath(@__DIR__, "..", "stdlib")) +documenter_stdlib_remotes = let stdlib_dir = realpath(joinpath(@__DIR__, "..", "stdlib")), + stdlib_build_dir = joinpath(buildrootdoc, "..", "stdlib") # Get a list of all *.version files in stdlib/.. version_files = filter(readdir(stdlib_dir)) do fname isfile(joinpath(stdlib_dir, fname)) && endswith(fname, ".version") @@ -93,7 +119,7 @@ documenter_stdlib_remotes = let stdlib_dir = realpath(joinpath(@__DIR__, "..", " versionfile[sha_key] end # Construct the absolute (local) path to the stdlib package's root directory - package_root_dir = joinpath(stdlib_dir, "$(package)-$(package_sha)") + package_root_dir = joinpath(stdlib_build_dir, "$(package)-$(package_sha)") # Documenter needs package_root_dir to exist --- it's just a sanity check it does on the remotes= keyword. # In normal (local) builds, this will be the case, since the Makefiles will have unpacked the standard # libraries. However, on CI we do this thing where we actually build docs in a clean worktree, just @@ -123,7 +149,7 @@ function generate_markdown(basename) @assert length(splitted) == 2 replaced_links = replace(splitted[1], r"\[\#([0-9]*?)\]" => s"[#\g<1>](https://github.com/JuliaLang/julia/issues/\g<1>)") write( - joinpath(@__DIR__, "src", "$basename.md"), + joinpath(buildrootdoc, "src", "$basename.md"), """ ```@meta EditURL = "https://github.com/JuliaLang/julia/blob/master/$basename.md" @@ -166,6 +192,7 @@ Manual = [ "manual/code-loading.md", "manual/profile.md", "manual/stacktraces.md", + "manual/memory-management.md", "manual/performance-tips.md", "manual/workflow-tips.md", "manual/style-guide.md", @@ -173,6 +200,7 @@ Manual = [ "manual/noteworthy-differences.md", "manual/unicode-input.md", "manual/command-line-interface.md", + "manual/worldage.md", ] BaseDocs = [ @@ -251,6 +279,16 @@ DevDocs = [ "devdocs/build/arm.md", "devdocs/build/riscv.md", "devdocs/build/distributing.md", + ], + "Contributor's Guide" => [ + "devdocs/contributing/code-changes.md", + "devdocs/contributing/tests.md", + "devdocs/contributing/documentation.md", + "devdocs/contributing/jldoctests.md", + "devdocs/contributing/patch-releases.md", + "devdocs/contributing/formatting.md", + "devdocs/contributing/git-workflow.md", + "devdocs/contributing/aiagents.md" ] ] @@ -276,16 +314,11 @@ end const use_revise = "revise=true" in ARGS if use_revise - let revise_env = joinpath(@__DIR__, "deps", "revise") - Pkg.activate(revise_env) - Pkg.add("Revise"; preserve=Pkg.PRESERVE_NONE) - Base.ACTIVE_PROJECT[] = nothing - pushfirst!(LOAD_PATH, revise_env) - end + Pkg.activate(joinpath(@__DIR__, "..", "deps", "jlutilities", "revise")) + Pkg.instantiate() end function maybe_revise(ex) use_revise || return ex - STDLIB_DIR = Sys.STDLIB STDLIBS = filter!(x -> isfile(joinpath(STDLIB_DIR, x, "src", "$(x).jl")), readdir(STDLIB_DIR)) return quote $ex @@ -346,10 +379,6 @@ DocMeta.setdocmeta!( recursive=true, warn=false, ) -let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS) - global const buildroot = i === nothing ? (@__DIR__) : first(match(r, ARGS[i]).captures) -end - const format = if render_pdf Documenter.LaTeX( platform = "texplatform=docker" in ARGS ? "docker" : "native" @@ -372,8 +401,9 @@ else ) end -const output_path = joinpath(buildroot, "doc", "_build", (render_pdf ? "pdf" : "html"), "en") +const output_path = joinpath(buildrootdoc, "_build", (render_pdf ? "pdf" : "html"), "en") makedocs( + source = joinpath(buildrootdoc, "src"), build = output_path, modules = [Main, Base, Core, [Base.root_module(Base, stdlib.stdlib) for stdlib in STDLIB_DOCS]...], clean = true, @@ -449,6 +479,7 @@ const devurl = "v$(VERSION.major).$(VERSION.minor)-dev" # Hack to make rc docs visible in the version selector struct Versions versions end +Documenter.determine_deploy_subfolder(deploy_decision, ::Versions) = deploy_decision.subfolder function Documenter.Writers.HTMLWriter.expand_versions(dir::String, v::Versions) # Find all available docs available_folders = readdir(dir) @@ -473,7 +504,7 @@ if "deploy" in ARGS deploydocs( repo = "github.com/JuliaLang/docs.julialang.org.git", deploy_config = BuildBotConfig(), - target = joinpath(buildroot, "doc", "_build", "html", "en"), + target = joinpath(buildrootdoc, "_build", "html", "en"), dirname = "en", devurl = devurl, versions = Versions(["v#.#", devurl => devurl]), diff --git a/doc/man/julia.1 b/doc/man/julia.1 index 2da11ae1b3f18..9646464e1e63d 100644 --- a/doc/man/julia.1 +++ b/doc/man/julia.1 @@ -321,7 +321,7 @@ Website: https://julialang.org/ .br Documentation: https://docs.julialang.org/ .br -Downloads: https://julialang.org/downloads/ +Install: https://julialang.org/install/ .SH LICENSING Julia is an open-source project. It is made available under the MIT license. diff --git a/doc/src/base/arrays.md b/doc/src/base/arrays.md index defe497daf00c..982c4b49cb2c4 100644 --- a/doc/src/base/arrays.md +++ b/doc/src/base/arrays.md @@ -32,6 +32,7 @@ Base.StridedMatrix Base.StridedVecOrMat Base.GenericMemory Base.Memory +Base.Memory(::UndefInitializer, ::Int) Base.memoryref Base.Slices Base.RowSlices diff --git a/doc/src/base/base.md b/doc/src/base/base.md index b1f70c6230a18..ab4bfdb6b105b 100644 --- a/doc/src/base/base.md +++ b/doc/src/base/base.md @@ -3,7 +3,7 @@ ## Introduction Julia Base contains a range of functions and macros appropriate for performing -scientific and numerical computing, but is also as broad as those of many general purpose programming +scientific and numerical computing, but is also as broad as those of many general-purpose programming languages. Additional functionality is available from a growing collection of [available packages](https://julialang.org/packages/). Functions are grouped by topic below. @@ -139,6 +139,8 @@ Core.:(===) Core.isa Base.isequal Base.isless +Base.ispositive +Base.isnegative Base.isunordered Base.ifelse Core.typeassert @@ -191,6 +193,7 @@ Base.typeintersect Base.promote_type Base.promote_rule Base.promote_typejoin +Base.iskindtype Base.isdispatchtuple ``` @@ -207,6 +210,7 @@ Base.isstructtype Base.nameof(::DataType) Base.fieldnames Base.fieldname +Base.fieldindex Core.fieldtype Base.fieldtypes Base.fieldcount @@ -248,6 +252,7 @@ Base.instances Core.Any Core.Union Union{} +Core.TypeofBottom Core.UnionAll Core.Tuple Core.NTuple @@ -392,6 +397,7 @@ Base.Sys.total_memory Base.Sys.free_physical_memory Base.Sys.total_physical_memory Base.Sys.uptime +Base.Sys.sysimage_target Base.Sys.isjsvm Base.Sys.loadavg Base.Sys.isexecutable @@ -475,6 +481,7 @@ Base.moduleroot __module__ __source__ Base.@__MODULE__ +Base.@__FUNCTION__ Base.@__FILE__ Base.@__DIR__ Base.@__LINE__ diff --git a/doc/src/base/collections.md b/doc/src/base/collections.md index e724930222a13..55cf1ba5dd30d 100644 --- a/doc/src/base/collections.md +++ b/doc/src/base/collections.md @@ -275,7 +275,7 @@ Partially implemented by: * [`Array`](@ref) -## Dequeues +## Deques ```@docs Base.push! diff --git a/doc/src/base/multi-threading.md b/doc/src/base/multi-threading.md index 81d1d83d765ac..88dc2b7514a2a 100644 --- a/doc/src/base/multi-threading.md +++ b/doc/src/base/multi-threading.md @@ -63,7 +63,9 @@ Base.@threadcall These building blocks are used to create the regular synchronization objects. ```@docs +Base.Threads.AbstractSpinLock Base.Threads.SpinLock +Base.Threads.PaddedSpinLock ``` ## Task metrics (Experimental) diff --git a/doc/src/base/numbers.md b/doc/src/base/numbers.md index aad4e94901054..0bd9d2d4c57d0 100644 --- a/doc/src/base/numbers.md +++ b/doc/src/base/numbers.md @@ -148,7 +148,7 @@ Base.@uint128_str ## [BigFloats and BigInts](@id BigFloats-and-BigInts) -The [`BigFloat`](@ref) and [`BigInt`](@ref) types implements +The [`BigFloat`](@ref) and [`BigInt`](@ref) types implement arbitrary-precision floating point and integer arithmetic, respectively. For [`BigFloat`](@ref) the [GNU MPFR library](https://www.mpfr.org/) is used, and for [`BigInt`](@ref) the [GNU Multiple Precision Arithmetic Library (GMP)] diff --git a/doc/src/base/parallel.md b/doc/src/base/parallel.md index cd5c95f17994a..e382e8edc56ee 100644 --- a/doc/src/base/parallel.md +++ b/doc/src/base/parallel.md @@ -159,5 +159,5 @@ non-atomic assignment of `ev.task`) In this example, `notify(ev::OneWayEvent)` is allowed to call `schedule(ev.task)` if and only if *it* modifies the state from `OWE_WAITING` to `OWE_NOTIFYING`. This lets us know that the task executing `wait(ev::OneWayEvent)` is now in the `ok` branch and that there cannot be -other tasks that tries to `schedule(ev.task)` since their +other tasks that try to `schedule(ev.task)` since their `@atomicreplace(ev.state, state => OWE_NOTIFYING)` will fail. diff --git a/doc/src/base/reflection.md b/doc/src/base/reflection.md index d88c3c8b0d0cf..16c0d1fadba4e 100644 --- a/doc/src/base/reflection.md +++ b/doc/src/base/reflection.md @@ -87,7 +87,7 @@ be passed instead!). For example: ```jldoctest; setup = :(using InteractiveUtils) julia> InteractiveUtils.macroexpand(@__MODULE__, :(@edit println("")) ) -:(InteractiveUtils.edit(println, (Base.typesof)(""))) +:(InteractiveUtils.edit(println, InteractiveUtils.Tuple{(InteractiveUtils.Core).Typeof("")})) ``` The functions `Base.Meta.show_sexpr` and [`dump`](@ref) are used to display S-expr style views @@ -142,7 +142,7 @@ For more information see [`@code_lowered`](@ref), [`@code_typed`](@ref), [`@code ### Printing of debug information The aforementioned functions and macros take the keyword argument `debuginfo` that controls the level -debug information printed. +of debug information printed. ```jldoctest; setup = :(using InteractiveUtils), filter = r"int.jl:\d+" julia> InteractiveUtils.@code_typed debuginfo=:source +(1,1) diff --git a/doc/src/base/scopedvalues.md b/doc/src/base/scopedvalues.md index 6ad553429bb1f..21d075daf9389 100644 --- a/doc/src/base/scopedvalues.md +++ b/doc/src/base/scopedvalues.md @@ -25,40 +25,59 @@ provided scoped value taking priority over previous definitions. Let's first look at an example of **lexical** scope. A `let` statement begins a new lexical scope within which the outer definition of `x` is shadowed by -it's inner definition. +its inner definition. -```julia +```jldoctest +julia> x = 1 +1 + +julia> let x = 5 + @show x + end; +x = 5 + +julia> @show x; x = 1 -let x = 5 - @show x # 5 -end -@show x # 1 ``` In the following example, since Julia uses lexical scope, the variable `x` in the body of `f` refers to the `x` defined in the global scope, and entering a `let` scope does not change the value `f` observes. -```julia +```jldoctest +julia> x = 1 +1 + +julia> f() = @show x +f (generic function with 1 method) + +julia> let x = 5 + f() + end; +x = 1 + +julia> f(); x = 1 -f() = @show x -let x = 5 - f() # 1 -end -f() # 1 ``` Now using a `ScopedValue` we can use **dynamic** scoping. -```julia -using Base.ScopedValues +```jldoctest +julia> using Base.ScopedValues -x = ScopedValue(1) -f() = @show x[] -with(x=>5) do - f() # 5 -end -f() # 1 +julia> x = ScopedValue(1) +ScopedValue{Int64}(1) + +julia> f() = @show x[] +f (generic function with 1 method) + +julia> with(x=>5) do + f() + end; +x[] = 5 + +julia> f(); +x[] = 1 ``` Note that the observed value of the `ScopedValue` is dependent on the execution diff --git a/doc/src/base/sort.md b/doc/src/base/sort.md index cef080c5f8995..cde0d571424a0 100644 --- a/doc/src/base/sort.md +++ b/doc/src/base/sort.md @@ -39,8 +39,8 @@ julia> a Instead of directly sorting an array, you can compute a permutation of the array's indices that puts the array into sorted order: -```julia-repl -julia> v = randn(5) +```jldoctest sort_example +julia> v = [0.297288, 0.382396, -0.597634, -0.0104452, -0.839027] 5-element Vector{Float64}: 0.297288 0.382396 @@ -67,7 +67,7 @@ julia> v[p] Arrays can be sorted according to an arbitrary transformation of their values: -```julia-repl +```jldoctest sort_example julia> sort(v, by=abs) 5-element Vector{Float64}: -0.0104452 @@ -79,7 +79,7 @@ julia> sort(v, by=abs) Or in reverse order by a transformation: -```julia-repl +```jldoctest sort_example julia> sort(v, by=abs, rev=true) 5-element Vector{Float64}: -0.839027 @@ -91,7 +91,7 @@ julia> sort(v, by=abs, rev=true) If needed, the sorting algorithm can be chosen: -```julia-repl +```jldoctest sort_example julia> sort(v, alg=InsertionSort) 5-element Vector{Float64}: -0.839027 diff --git a/doc/src/base/strings.md b/doc/src/base/strings.md index a9637a1a7be3a..15a7c0531de4a 100644 --- a/doc/src/base/strings.md +++ b/doc/src/base/strings.md @@ -29,6 +29,7 @@ Base.SubstitutionString Base.@s_str Base.@raw_str Base.@b_str +Base.takestring! Base.Docs.@html_str Base.Docs.@text_str Base.isvalid(::Any) diff --git a/doc/src/devdocs/ast.md b/doc/src/devdocs/ast.md index 1330940862b99..db90a6c87e45d 100644 --- a/doc/src/devdocs/ast.md +++ b/doc/src/devdocs/ast.md @@ -119,10 +119,18 @@ parses as `(macrocall (|.| Core '@doc) (line) "some docs" (= (call f x) (block x | `import Base: x` | `(import (: (. Base) (. x)))` | | `import Base: x, y` | `(import (: (. Base) (. x) (. y)))` | | `export a, b` | `(export a b)` | +| `public a, b` | `(public a b)` | `using` has the same representation as `import`, but with expression head `:using` instead of `:import`. +To programmatically create a `public` statement, you can use `Expr(:public, :a, :b)` or, +closer to regular code, `Meta.parse("public a, b")`. This approach is necessary due to +[current limitations on `public`](@ref Export-lists). The `public` keyword is only +recognized at the syntactic top level within a file (`parse_stmts`) or module. This +restriction was implemented to prevent breaking existing code that used `public` as an +identifier when it was introduced in Julia 1.11. + ### Numbers Julia supports more number types than many scheme implementations, so not all numbers are represented @@ -155,7 +163,7 @@ parses as: ``` (if a (block (line 2) b) (elseif (block (line 3) c) (block (line 4) d) - (block (line 6 e)))) + (block (line 6) e))) ``` A `while` loop parses as `(while condition body)`. diff --git a/doc/src/devdocs/backtraces.md b/doc/src/devdocs/backtraces.md index d0533ebe57fcb..7ecfa20f89780 100644 --- a/doc/src/devdocs/backtraces.md +++ b/doc/src/devdocs/backtraces.md @@ -128,4 +128,4 @@ Note that this is only works on Linux. The blog post on [Time Travelling Bug Rep A few terms have been used as shorthand in this guide: * `` refers to the root directory of the Julia source tree; e.g. it should contain folders - such as `base`, `deps`, `src`, `test`, etc..... + such as `base`, `deps`, `src`, `test`, etc. diff --git a/doc/src/devdocs/build/build.md b/doc/src/devdocs/build/build.md index ff7e64926f1f7..b6a949c74e74a 100644 --- a/doc/src/devdocs/build/build.md +++ b/doc/src/devdocs/build/build.md @@ -304,6 +304,20 @@ LLVM_ASSERTIONS=1 Please note that assert builds of Julia will be slower than regular (non-assert) builds. +## Building a debug build of Julia + +A full debug build of Julia can be built with `make debug`. This builds a debug +version of `libjulia` and uses it to bootstrap the compiler, before creating a +system image with debug symbols enabled. This can take more than 15 minutes. + +Although it may result in some differences, a debug build can be built much +quicker by bootstrapping from a release build: + +```sh +$ make julia-src-release julia-sysbase-release +$ make julia-sysimg-debug CROSS_BOOTSTRAP_JULIA=$PWD/usr/bin/julia CROSS_BOOTSTRAP_SYSBASE=$PWD/usr/lib/julia/sysbase.so +``` + ## Building 32-bit Julia on a 64-bit machine Occasionally, bugs specific to 32-bit architectures may arise, and when this happens it is useful to be able to debug the problem on your local machine. Since most modern 64-bit systems support running programs built for 32-bit ones, if you don't have to recompile Julia from source (e.g. you mainly need to inspect the behavior of a 32-bit Julia without having to touch the C code), you can likely use a 32-bit build of Julia for your system that you can obtain from the [official downloads page](https://julialang.org/downloads/). diff --git a/doc/src/devdocs/build/riscv.md b/doc/src/devdocs/build/riscv.md index 7c0e7ab29d9f8..51939c25e41a1 100644 --- a/doc/src/devdocs/build/riscv.md +++ b/doc/src/devdocs/build/riscv.md @@ -11,17 +11,9 @@ including the output from `cat /proc/cpuinfo`. ## Compiling Julia -For now, Julia will need to be compiled entirely from source, i.e., including -all of its dependencies. This can be accomplished with the following -`Make.user`: - -```make -USE_BINARYBUILDER := 0 -``` - -Additionally, it is required to indicate what architecture, and optionally which -CPU to build for. This can be done by setting the `MARCH` and `MCPU` variables -in `Make.user` +To compilie Julia for RISC-V, you need to manually indicate what architecture, and +optionally which CPU to build for. This can be done by setting the `MARCH` and `MCPU` +variables in `Make.user` The `MARCH` variable needs to be set to a RISC-V ISA string, which can be found by looking at the documentation of your device, or by inspecting `/proc/cpuinfo`. Only diff --git a/doc/src/devdocs/build/windows.md b/doc/src/devdocs/build/windows.md index 77042b352047c..ed35ea10d198b 100644 --- a/doc/src/devdocs/build/windows.md +++ b/doc/src/devdocs/build/windows.md @@ -32,7 +32,7 @@ or edit `%USERPROFILE%\.gitconfig` and add/edit the lines: ## Binary distribution For the binary distribution installation notes on Windows please see the instructions at -[https://julialang.org/downloads/platform/#windows](https://julialang.org/downloads/platform/#windows). +[https://julialang.org/downloads/platform/#windows](https://julialang.org/downloads/platform/#windows). Note, however, that on all platforms [using `juliaup`](https://julialang.org/install/) is recommended over manually installing binaries. ## Source distribution @@ -141,19 +141,19 @@ Note: MSYS2 requires **64 bit** Windows 7 or newer. 4. Then install tools required to build julia: ``` - pacman -S cmake diffutils git m4 make patch tar p7zip curl python + pacman -S diffutils git m4 make patch tar p7zip curl python ``` For 64 bit Julia, install the x86_64 version: ``` - pacman -S mingw-w64-x86_64-gcc + pacman -S mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake mingw-w64-x86_64-clang ``` For 32 bit Julia, install the i686 version: ``` - pacman -S mingw-w64-i686-gcc + pacman -S mingw-w64-i686-gcc mingw-w64-i686-cmake mingw-w64-i686-clang ``` 5. Configuration of MSYS2 is complete. Now `exit` the MSYS2 shell. @@ -171,7 +171,15 @@ Note: MSYS2 requires **64 bit** Windows 7 or newer. cd julia ``` - 3. Start the build + 3. If you want to use clang (currently required if building LLVM from source), put the following in your Make.user + ``` + CC=/mingw64/bin/clang + CXX=/mingw64/bin/clang++ + ``` +!!! warning "UCRT Unsupported" + Do not try to use any other clang that MSYS2 may install (which may not have the correct default target) or the "Clang" environment(which defaults to the currently unsupported ucrt). + + 4. Start the build ``` make -j$(nproc) diff --git a/doc/src/devdocs/builtins.md b/doc/src/devdocs/builtins.md index e53321f3e70a0..ce56a7f9a6b91 100644 --- a/doc/src/devdocs/builtins.md +++ b/doc/src/devdocs/builtins.md @@ -1,12 +1,25 @@ # [Core.Builtins](@id lib-builtins) -## Builtin Function APIs +The following builtin functions are considered unstable, but provide the basic +definitions for what defines the abilities and behaviors of a Julia +program. They are typically accessed through a higher level generic API. -The following Builtin function APIs are considered unstable, but provide the basic -definitions for what defines the abilities and behaviors of a Julia program. They are -typically accessed through a higher level generic API. +## Raw access to memory ```@docs +Core.Intrinsics.pointerref +Core.Intrinsics.pointerset +Core.Intrinsics.atomic_pointerref +Core.Intrinsics.atomic_pointerset +Core.Intrinsics.atomic_pointerswap +Core.Intrinsics.atomic_pointermodify +Core.Intrinsics.atomic_pointerreplace +``` + +## Managed memory + +```@docs +Core.memorynew Core.memoryrefnew Core.memoryrefoffset Core.memoryrefget @@ -16,13 +29,20 @@ Core.memoryrefswap! Core.memoryrefmodify! Core.memoryrefreplace! Core.memoryrefsetonce! -Core.Intrinsics.atomic_pointerref -Core.Intrinsics.atomic_pointerset -Core.Intrinsics.atomic_pointerswap -Core.Intrinsics.atomic_pointermodify -Core.Intrinsics.atomic_pointerreplace +``` + +## Module bindings + +```@docs Core.get_binding_type +``` + +## Other + +```@docs Core.IntrinsicFunction Core.Intrinsics Core.IR +Base.quoted +Base.isa_ast_node ``` diff --git a/doc/src/devdocs/callconv.md b/doc/src/devdocs/callconv.md index 88158cb1eee84..2bf89fb92856b 100644 --- a/doc/src/devdocs/callconv.md +++ b/doc/src/devdocs/callconv.md @@ -18,10 +18,10 @@ signature. * LLVM scalars and vectors are passed by value. * LLVM aggregates (arrays and structs) are passed by reference. -A small return values is returned as LLVM return values. A large return values is returned via +A small return value is returned as LLVM return values. A large return value is returned via the "structure return" (`sret`) convention, where the caller provides a pointer to a return slot. -An argument or return values that is a homogeneous tuple is sometimes represented as an LLVM vector +An argument or return value that is a homogeneous tuple is sometimes represented as an LLVM vector instead of an LLVM array. ## JL Call Convention diff --git a/doc/src/devdocs/compiler.md b/doc/src/devdocs/compiler.md index 8f5f2bb1aa17c..c2ee5f9a375e1 100644 --- a/doc/src/devdocs/compiler.md +++ b/doc/src/devdocs/compiler.md @@ -110,7 +110,7 @@ The general principles are that: - Primitive types get passed in int/float registers. - Tuples of VecElement types get passed in vector registers. - Structs get passed on the stack. -- Return values are handle similarly to arguments, +- Return values are handled similarly to arguments, with a size-cutoff at which they will instead be returned via a hidden sret argument. The total logic for this is implemented by `get_specsig_function` and `deserves_sret`. diff --git a/doc/src/devdocs/contributing/aiagents.md b/doc/src/devdocs/contributing/aiagents.md new file mode 100644 index 0000000000000..6fc7b9a662071 --- /dev/null +++ b/doc/src/devdocs/contributing/aiagents.md @@ -0,0 +1,49 @@ +# Using AI agents to work on Julia + +> ![WARNING] +> You are responsible for the code you submit in PRs. Do not submit PRs +> containing AI-generated code that you do not understand or that does not +> meet the ordinary quality bar for PRs to julia. + +This page documents best practices for setting up AI agents to work with Julia. +If you find additional prompt instructions that work well for common tasks, +consider submitting a PR to add these to AGENTS.md. + +## Google Jules + +Use the following for your `Initial Setup` configuration. + +``` +curl -fsSL https://install.julialang.org | sh -s -- -y --default-channel nightly +. /home/swebot/.profile +``` + +Jules has access to the internet, so you can give it links to issues or additional +documentation in your prompting. + +## OpenAI Codex + +Configure the following: + +Setup Script +``` +apt update +apt install less +curl -fsSL https://install.julialang.org | sh -s -- -y --default-channel nightly +source /root/.bashrc +make -C /workspace/julia/doc alldeps JULIA_EXECUTABLE="/root/.juliaup/bin/julia" +make -C /workspace/julia/test install-revise-deps JULIA_EXECUTABLE="/root/.juliaup/bin/julia" +``` + +Environment Variables +``` +JULIA_PKG_OFFLINE=true +``` + +Codex does not have internet access after initial setup, so you cannot give it +additional information as links - you will need to copy any relevant text into +the prompt. + +Note that Codex rebuilds the environment after every invocation. This can +add significant latency. Codex work best for well-defined tasks that can +be solved in a single shot. diff --git a/doc/src/devdocs/contributing/code-changes.md b/doc/src/devdocs/contributing/code-changes.md new file mode 100644 index 0000000000000..41bb03f01409b --- /dev/null +++ b/doc/src/devdocs/contributing/code-changes.md @@ -0,0 +1,114 @@ +# Code changes + +## Contributing to core functionality or base libraries + +*By contributing code to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* + +The Julia community uses [GitHub issues](https://github.com/JuliaLang/julia/issues) to track and discuss problems, feature requests, and pull requests (PR). + +Issues and pull requests should have self explanatory titles such that they can be understood from the list of PRs and Issues. +i.e. `Add {feature}` and `Fix {bug}` are good, `Fix #12345. Corrects the bug.` is bad. + +You can make pull requests for incomplete features to get code review. The convention is to open these as draft PRs and prefix +the pull request title with "WIP:" for Work In Progress, or "RFC:" for Request for Comments when work is completed and ready +for merging. This will prevent accidental merging of work that is in progress. + +Note: These instructions are for adding to or improving functionality in the base library. Before getting started, it can be helpful to discuss the proposed changes or additions on the [Julia Discourse forum](https://discourse.julialang.org) or in a GitHub issue---it's possible your proposed change belongs in a package rather than the core language. Also, keep in mind that changing stuff in the base can potentially break a lot of things. Finally, because of the time required to build Julia, note that it's usually faster to develop your code in stand-alone files, get it working, and then migrate it into the base libraries. + +Add new code to Julia's base libraries as follows (this is the "basic" approach; see a more efficient approach in the next section): + + 1. Edit the appropriate file in the `base/` directory, or add new files if necessary. Create tests for your functionality and add them to files in the `test/` directory. If you're editing C or Scheme code, most likely it lives in `src/` or one of its subdirectories, although some aspects of Julia's REPL initialization live in `cli/`. + + 2. Add any new files to `sysimg.jl` in order to build them into the Julia system image. + + 3. Add any necessary export symbols in `exports.jl`. + + 4. Include your tests in `test/Makefile` and `test/choosetests.jl`. + +Build as usual, and do `make clean testall` to test your contribution. If your contribution includes changes to Makefiles or external dependencies, make sure you can build Julia from a clean tree using `git clean -fdx` or equivalent (be careful – this command will delete any files lying around that aren't checked into git). + +### Running specific tests + +There are `make` targets for running specific tests: + + make test-bitarray + +You can also use the `runtests.jl` script, e.g. to run `test/bitarray.jl` and `test/math.jl`: + + ./usr/bin/julia test/runtests.jl bitarray math + +### Modifying base more efficiently with Revise.jl + +[Revise](https://github.com/timholy/Revise.jl) is a package that +tracks changes in source files and automatically updates function +definitions in your running Julia session. Using it, you can make +extensive changes to Base without needing to rebuild in order to test +your changes. + +Here is the standard procedure: + +1. If you are planning changes to any types or macros, make those + changes and build julia using `make`. (This is + necessary because `Revise` cannot handle changes to type + definitions or macros.) Unless it's + required to get Julia to build, you do not have to add any + functionality based on the new types, just the type definitions + themselves. + +2. Start a Julia REPL session. Then issue the following commands: + +```julia +using Revise # if you aren't launching it in your `.julia/config/startup.jl` +Revise.track(Base) +``` + +3. Edit files in `base/`, save your edits, and test the + functionality. + +If you need to restart your Julia session, just start at step 2 above. +`Revise.track(Base)` will note any changes from when Julia was last +built and incorporate them automatically. You only need to rebuild +Julia if you made code-changes that Revise cannot handle. + +For convenience, there are also `test-revise-*` targets for every [`test-*` +target](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#running-specific-tests) that use Revise to load any modifications to Base into the current +system image before running the corresponding test. This can be useful as a shortcut +on the command line (since tests aren't always designed to be run outside the +runtest harness). + +## Contributing to the standard library + +The standard library (stdlib) packages are baked into the Julia system image. +When running the ordinary test workflow on the stdlib packages, the system image +version overrides the version you are developing. +To test stdlib packages, you can do the following steps: + +1. Edit the UUID field of the `Project.toml` in the stdlib package +2. Change the current directory to the directory of the stdlib you are developing +3. Start julia with `julia --project=.` +4. You can now test the package by running `pkg> test` in Pkg mode. + +Because you changed the UUID, the package manager treats the stdlib package as +different from the one in the system image, and the system image version will +not override the package. + +Be sure to change the UUID value back before making the pull request. + +### News-worthy changes + +For new functionality and other substantial changes, add a brief summary to `NEWS.md`. The news item should cross reference the pull request (PR) parenthetically, in the form `([#pr])`. To add the PR reference number, first create the PR, then push an additional commit updating `NEWS.md` with the PR reference number. We periodically run `./julia doc/NEWS-update.jl` from the julia directory to update the cross-reference links, but this should not be done in a typical PR in order to avoid conflicting commits. + +### Annotations for new features, deprecations and behavior changes + +API additions and deprecations, and minor behavior changes are allowed in minor version releases. +For documented features that are part of the public API, a compatibility note should be added into +the manual or the docstring. It should state the Julia minor version that changed the behavior +and have a brief message describing the change. + +At the moment, this should always be done with the following `compat` admonition +(so that it would be possible to programmatically find the annotations in the future): + + ``` + !!! compat "Julia 1.X" + This method was added in Julia 1.X. + ``` diff --git a/doc/src/devdocs/contributing/documentation.md b/doc/src/devdocs/contributing/documentation.md new file mode 100644 index 0000000000000..e12ac1d482063 --- /dev/null +++ b/doc/src/devdocs/contributing/documentation.md @@ -0,0 +1,87 @@ +# Improving documentation + +*By contributing documentation to Julia, you are agreeing to release it under the [MIT License](https://github.com/JuliaLang/julia/tree/master/LICENSE.md).* + +Julia's documentation source files are stored in the `doc/` directory and all docstrings are found in `base/`. Like everything else these can be modified using `git`. Documentation is built with [Documenter.jl](https://github.com/JuliaDocs/Documenter.jl), which uses Markdown syntax. The HTML documentation can be built locally by running + +``` +make docs +``` + +from Julia's root directory. This will rebuild the Julia system image, then install or update the package dependencies required to build the documentation, and finally build the HTML documentation and place the resulting files in `doc/_build/html/`. + +> **Note** +> +> When making changes to any of Julia's documentation it is recommended that you run `make docs` to check that your changes are valid and do not produce any errors before opening a pull request. + +Below are outlined the three most common types of documentation changes and the steps required to perform them. Please note that the following instructions do not cover the full range of features provided by Documenter.jl. Refer to [Documenter's documentation](https://juliadocs.github.io/Documenter.jl/stable) if you encounter anything that is not covered by the sections below. + +## Modifying files in `doc/src/` + +Most of the source text for the Julia Manual is located in `doc/src/`. To update or add new text to any one of the existing files the following steps should be followed: + +1. update the text in whichever `.md` files are applicable; +2. run `make docs` from the root directory; +3. check the output in `doc/_build/html/` to make sure the changes are correct; +4. commit your changes and open a pull request. + +> **Note** +> +> The contents of `doc/_build/` does **not** need to be committed when you make changes. + +To add a **new file** to `doc/src/` rather than updating a file replace step `1` above with + +1. add the file to the appropriate subdirectory in `doc/src/` and also add the file path to the `PAGES` vector in `doc/make.jl`. + +## Modifying an existing docstring in `base/` + +All docstrings are written inline above the methods or types they are associated with and can be found by clicking on the `source` link that appears below each docstring in the HTML file. The steps needed to make a change to an existing docstring are listed below: + +1. find the docstring in `base/`; +2. update the text in the docstring; +3. run `make docs` from the root directory; +4. check the output in `doc/_build/html/` to make sure the changes are correct; +5. commit your changes and open a pull request. + +## Adding a new docstring to `base/` + +The steps required to add a new docstring are listed below: + +1. find a suitable definition in `base/` that the docstring will be most applicable to; +2. add a docstring above the definition; +3. find a suitable `@docs` code block in one of the `doc/src/stdlib/` files where you would like the docstring to appear; +4. add the name of the definition to the `@docs` code block. For example, with a docstring added to a function `bar` + + ```julia + "..." + function bar(args...) + # ... + end + ``` + + you would add the name `bar` to a `@docs` block in `doc/src/stdlib/` + + ```@docs + foo + bar # <-- Added this one. + baz + ``` + +5. run `make docs` from the root directory; +6. check the output in `doc/_build/html` to make sure the changes are correct; +7. commit your changes and open a pull request. + +## Doctests + +Examples written within docstrings can be used as testcases known as "doctests" by annotating code blocks with `jldoctest`. + + ```jldoctest + julia> uppercase("Docstring test") + "DOCSTRING TEST" + ``` + +A doctest needs to match an interactive REPL including the `julia>` prompt. It is recommended to add the header `# Examples` above the doctests. + +See the documentation of [writing jldoctests](@ref writing-jldoctests) for best +practices on how to write doctests for common scenarios and the `doc/README.md` +file for how to run the doctests. diff --git a/doc/src/devdocs/contributing/formatting.md b/doc/src/devdocs/contributing/formatting.md new file mode 100644 index 0000000000000..e3fc8ec129908 --- /dev/null +++ b/doc/src/devdocs/contributing/formatting.md @@ -0,0 +1,22 @@ +# Code Formatting Guidelines + +## General Formatting Guidelines for Julia code contributions + + - Follow the latest dev version of [Julia Style Guide](https://docs.julialang.org/en/v1/manual/style-guide/). + - Use whitespace to make the code more readable + - No whitespace at the end of a line (trailing whitespace) + - Comments are good, especially when they explain the algorithm + - Try to adhere to a 92 character line length limit + - It is generally preferred to use ASCII operators and identifiers over + Unicode equivalents whenever possible + - In docstrings refer to the language as "Julia" and the executable as "`julia`" + +## General Formatting Guidelines For C code contributions + + - 4 spaces per indentation level, no tabs + - Space between `if` and `(` (`if (x) ...`) + - Newline before opening `{` in function definitions + - `f(void)` for 0-argument function declarations + - Newline between `}` and `else` instead of `} else {` + - If one part of an `if..else` chain uses `{ }` then all should + - No whitespace at the end of a line diff --git a/doc/src/devdocs/contributing/git-workflow.md b/doc/src/devdocs/contributing/git-workflow.md new file mode 100644 index 0000000000000..2142eed69632a --- /dev/null +++ b/doc/src/devdocs/contributing/git-workflow.md @@ -0,0 +1,19 @@ +# Git workflow recommendations + +## Git Recommendations For Pull Requests + + - Avoid working from the `master` branch of your fork. Create a new branch as it will make it easier to update your pull request if Julia's `master` changes. + - Try to [squash](https://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html) together small commits that make repeated changes to the same section of code, so your pull request is easier to review. A reasonable number of separate well-factored commits is fine, especially for larger changes. + - If any conflicts arise due to changes in Julia's `master`, prefer updating your pull request branch with `git rebase` versus `git merge` or `git pull`, since the latter will introduce merge commits that clutter the git history with noise that makes your changes more difficult to review. + - Descriptive commit messages are good. + - Using `git add -p` or `git add -i` can be useful to avoid accidentally committing unrelated changes. + - When linking to specific lines of code in discussion of an issue or pull request, hit the `y` key while viewing code on GitHub to reload the page with a URL that includes the specific version that you're viewing. That way any lines of code that you refer to will still make sense in the future, even if the content of the file changes. + - Whitespace can be automatically removed from existing commits with `git rebase`. + - To remove whitespace for the previous commit, run + `git rebase --whitespace=fix HEAD~1`. + - To remove whitespace relative to the `master` branch, run + `git rebase --whitespace=fix master`. + +### Git Recommendations For Pull Request Reviewers + +- When merging, we generally like `squash+merge`. Unless it is the rare case of a PR with carefully staged individual commits that you want in the history separately, in which case `merge` is acceptable, but usually prefer `squash+merge`. diff --git a/doc/src/devdocs/contributing/jldoctests.md b/doc/src/devdocs/contributing/jldoctests.md new file mode 100644 index 0000000000000..6631390cf9ca3 --- /dev/null +++ b/doc/src/devdocs/contributing/jldoctests.md @@ -0,0 +1,114 @@ +# [Writing jldoctests](@id writing-jldoctests) + +This page describes how to write and maintain `jldoctest` blocks in the documentation. Following these guidelines helps keep doctests reliable and easy to read. + +## Filters + +Use `filter =` whenever output contains text that might vary across runs. +The following are common situations where this may happen: + +- The output contains arrays with undefined memory (e.g. from `undef` or `similar`) +- The output contains random numbers +- The output contains timing information +- The output contains file system paths + + +### Common filter sequences + +The documentation relies on several recurring patterns: +- `r"int.jl:\\d+"` — remove line numbers from introspection macros. +- `r"Stacktrace:(\\n \\[0-9]+\\].*)*"` — hide stack traces when illustrating + errors. +- `r"Closest candidates.*\\n .*"` — skip the method suggestions printed by + `MethodError`. +- `r"@ .*"` — strip file locations from the output of `methods` or + `@which`. +- `r"\\@world\\(MyStruct, \\d+:\\d+\\)"` — filter world age numbers. +- `r"with \\d+ methods"` — ignore method counts when redefining functions. +- `r"[0-9\\.]+ seconds \\(.*?\\)"` — remove timing output with memory + information. +- `r"[0-9\\.]+ seconds"` — remove simple timing results. +- `r"[0-9\\.]+"` — filter digits from names such as anonymous functions. +- `r"([A-B] [0-5])"` and `r"[A-B] [X-Z] [0-5]"` — account for non-deterministic + process output. +- `r"(world\\nhello|hello\\nworld)"` — allow either ordering of interleaved + output. + +If none of these match your situation, craft a regular expression that +removes the varying text. Using filters keeps doctests stable across +platforms and Julia versions. + +!!! note "Double escaping in docstrings" + When writing regex filters inside docstrings, remember to double escape + backslashes. For example, use `r"[\\d\\.]+"` instead of `r"[\d\.]+"`. + This is necessary because the docstring itself processes escape sequences + before the regex is created. + +## Setup code + +Small setup expressions may be placed inline using the `setup =` option: + +```` +```jldoctest; setup = :(using InteractiveUtils) +... +``` +```` + +For longer setup code or if multiple blocks require the same environment, use the +`DocTestSetup` meta block: + +```` +```@meta +DocTestSetup = :(import Random; Random.seed!(1234)) +``` +```` + +and disable it afterwards with + +```` +```@meta +DocTestSetup = nothing +``` +```` + +### Teardown code + +If you need teardown code (e.g. to delete created temporary files or to reset +the current directory), you can use the `teardown =` option: + +```` +```jldoctest; setup = :(oldpath = pwd(); cd(mktempdir())), teardown = :(cd(oldpath)) +... +``` +```` + +## Maintaining state between snippets + +Related doctest blocks can share state by giving them the same label after the +`jldoctest` keyword. The manual uses this pattern to demonstrate mutation: + +```` +```jldoctest mutation_vs_rebind +julia> a = [1,2,3] +... +``` +```` + +and later + +```` +```jldoctest mutation_vs_rebind +julia> a[1] = 42 +... +``` +```` + +Blocks with the same name execute sequentially during doctesting, so variables +created in the first block remain available in the following ones. + +When a snippet needs to preserve its result for later examples, give it a label +and reuse that label. This avoids repeating setup code and mirrors a REPL +session more closely. + +## Further reading +For a complete reference of doctest syntax, see the [corresponding Documenter.jl docs](https://documenter.juliadocs.org/stable/man/doctests/). diff --git a/doc/src/devdocs/contributing/patch-releases.md b/doc/src/devdocs/contributing/patch-releases.md new file mode 100644 index 0000000000000..515c2cfd35225 --- /dev/null +++ b/doc/src/devdocs/contributing/patch-releases.md @@ -0,0 +1,44 @@ +# Contributing to patch releases + +The process of [creating a patch release](https://docs.julialang.org/en/v1/devdocs/build/distributing/#Point-releasing-101) is roughly as follows: + +1. Create a new branch (e.g. `backports-release-1.10`) against the relevant minor release + branch (e.g. `release-1.10`). Usually a corresponding pull request is created as well. + +2. Add commits, nominally from `master` (hence "backports"), to that branch. + See below for more information on this process. + +3. Run the [BaseBenchmarks.jl](https://github.com/JuliaCI/BaseBenchmarks.jl) benchmark + suite and [PkgEval.jl](https://github.com/JuliaCI/PkgEval.jl) package ecosystem + exerciser against that branch. Nominally BaseBenchmarks.jl and PkgEval.jl are + invoked via [Nanosoldier.jl](https://github.com/JuliaCI/Nanosoldier.jl) from + the pull request associated with the backports branch. Fix any issues. + +4. Once all test and benchmark reports look good, merge the backports branch into + the corresponding release branch (e.g. merge `backports-release-1.10` into + `release-1.10`). + +5. Open a pull request that bumps the version of the relevant minor release to the + next patch version, e.g. as in [this pull request](https://github.com/JuliaLang/julia/pull/37718). + +6. Ping `@JuliaLang/releases` to tag the patch release and update the website. + +7. Open a pull request that bumps the version of the relevant minor release to the + next prerelease patch version, e.g. as in [this pull request](https://github.com/JuliaLang/julia/pull/37724). + +Step 2 above, i.e. backporting commits to the `backports-release-X.Y` branch, has largely +been automated via [`Backporter`](https://github.com/KristofferC/Backporter): Backporter +searches for merged pull requests with the relevant `backport-X.Y` tag, and attempts to +cherry-pick the commits from those pull requests onto the `backports-release-X.Y` branch. +Some commits apply successfully without intervention, others not so much. The latter +commits require "manual" backporting, with which help is generally much appreciated. +Backporter generates a report identifying those commits it managed to backport automatically +and those that require manual backporting; this report is usually copied into the first +post of the pull request associated with `backports-release-X.Y` and maintained as +additional commits are automatically and/or manually backported. + +When contributing a manual backport, if you have the necessary permissions, please push the +backport directly to the `backports-release-X.Y` branch. If you lack the relevant +permissions, please open a pull request against the `backports-release-X.Y` branch with the +manual backport. Once the manual backport is live on the `backports-release-X.Y` branch, +please remove the `backport-X.Y` tag from the originating pull request for the commits. diff --git a/doc/src/devdocs/contributing/tests.md b/doc/src/devdocs/contributing/tests.md new file mode 100644 index 0000000000000..c1f25da5f4a7c --- /dev/null +++ b/doc/src/devdocs/contributing/tests.md @@ -0,0 +1,19 @@ +# Writing tests + +There are never enough tests. Track [code coverage at Codecov](https://codecov.io/github/JuliaLang/julia), and help improve it. + +1. Go visit https://codecov.io/github/JuliaLang/julia. + +2. Browse through the source files and find some untested functionality (highlighted in red) that you think you might be able to write a test for. + +3. Write a test that exercises this functionality---you can add your test to one of the existing files, or start a new one, whichever seems most appropriate to you. If you're adding a new test file, make sure you include it in the list of tests in `test/choosetests.jl`. https://docs.julialang.org/en/v1/stdlib/Test/ may be helpful in explaining how the testing infrastructure works. + +4. Run `make test-all` to rebuild Julia and run your new test(s). If you had to fix a bug or add functionality in `base`, this will ensure that your test passes and that you have not introduced extraneous whitespace. + +5. Submit the test as a pull request (PR). + +* Code for the buildbot configuration is maintained at: https://github.com/staticfloat/julia-buildbot +* You can see the current buildbot setup at: https://build.julialang.org/builders +* [Issue 9493](https://github.com/JuliaLang/julia/issues/9493) and [issue 11885](https://github.com/JuliaLang/julia/issues/11885) have more detailed discussion on code coverage. + +Code coverage shows functionality that still needs "proof of concept" tests. These are important, as are tests for tricky edge cases, such as converting between integer types when the number to convert is near the maximum of the range of one of the integer types. Even if a function already has some coverage on Codecov, it may still benefit from tests for edge cases. diff --git a/doc/src/devdocs/debuggingtips.md b/doc/src/devdocs/debuggingtips.md index 0c7ee9d98f046..514045810cd9e 100644 --- a/doc/src/devdocs/debuggingtips.md +++ b/doc/src/devdocs/debuggingtips.md @@ -200,7 +200,7 @@ Expr(:return, Expr(:call, :box, :Float32, Expr(:call, :fptrunc, :Float32, :x)::A ``` Finally, and perhaps most usefully, we can force the function to be recompiled in order to step -through the codegen process. To do this, clear the cached `functionObject` from the `jl_lamdbda_info_t*`: +through the codegen process. To do this, clear the cached `functionObject` from the `jl_lambda_info_t*`: ``` (gdb) p f->linfo->functionObject diff --git a/doc/src/devdocs/eval.md b/doc/src/devdocs/eval.md index 8f2fd68159676..42a5f779a57d6 100644 --- a/doc/src/devdocs/eval.md +++ b/doc/src/devdocs/eval.md @@ -15,7 +15,7 @@ function, and primitive function, before turning into the desired result (hopefu short. * AST - Abstract Syntax Tree The AST is the digital representation of the code structure. In this form + Abstract Syntax Tree. The AST is the digital representation of the code structure. In this form the code has been tokenized for meaning so that it is more suitable for manipulation and execution. @@ -89,7 +89,7 @@ the expression. Macro expansion involves a handoff from [`eval()`](@ref) (in Jul function `jl_macroexpand()` (written in `flisp`) to the Julia macro itself (written in - what else - Julia) via `fl_invoke_julia_macro()`, and back. -Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`jl_expand()`, +Typically, macro expansion is invoked as a first step during a call to [`Meta.lower()`](@ref)/`Core._lower()`, although it can also be invoked directly by a call to [`macroexpand()`](@ref)/`jl_macroexpand()`. ## [Type Inference](@id dev-type-inference) diff --git a/doc/src/devdocs/external_profilers.md b/doc/src/devdocs/external_profilers.md index 836d821b91df9..5f54c2619559b 100644 --- a/doc/src/devdocs/external_profilers.md +++ b/doc/src/devdocs/external_profilers.md @@ -8,9 +8,21 @@ The currently supported profilers are: ### Adding New Zones +#### From C/C++ code + To add new zones, use the `JL_TIMING` macro. You can find numerous examples throughout the codebase by searching for `JL_TIMING`. To add a new type of zone you add it to `JL_TIMING_OWNERS` (and possibly `JL_TIMING_EVENTS`). +#### From Julia code + +The `Compiler.@zone` macro can be used to add a zone from Julia code, it is used as: + +```julia +Compiler.@zone "ZONE NAME" begin + ... +end +``` + ### Dynamically Enabling and Disabling Zones The [`JULIA_TIMING_SUBSYSTEMS`](@ref JULIA_TIMING_SUBSYSTEMS) environment variable allows you to enable or disable zones for a specific Julia run. For instance, setting the variable to `+GC,-INFERENCE` will enable the `GC` zones and disable the `INFERENCE` diff --git a/doc/src/devdocs/functions.md b/doc/src/devdocs/functions.md index 777afaa56348d..0f414d5df2058 100644 --- a/doc/src/devdocs/functions.md +++ b/doc/src/devdocs/functions.md @@ -1,25 +1,22 @@ # Julia Functions + This document will explain how functions, method definitions, and method tables work. ## Method Tables Every function in Julia is a generic function. A generic function is conceptually a single function, -but consists of many definitions, or methods. The methods of a generic function are stored in -a method table. Method tables (type `MethodTable`) are associated with `TypeName`s. A `TypeName` -describes a family of parameterized types. For example `Complex{Float32}` and `Complex{Float64}` -share the same `Complex` type name object. - -All objects in Julia are potentially callable, because every object has a type, which in turn -has a `TypeName`. +but consists of many definitions, or methods. The methods of a generic function are stored in a +method table. There is one global method table (type `MethodTable`) named `Core.GlobalMethods`. Any +default operation on methods (such as calls) uses that table. ## [Function calls](@id Function-calls) -Given the call `f(x, y)`, the following steps are performed: first, the method table to use is -accessed as `typeof(f).name.mt`. Second, an argument tuple type is formed, `Tuple{typeof(f), typeof(x), typeof(y)}`. -Note that the type of the function itself is the first element. This is because the type might -have parameters, and so needs to take part in dispatch. This tuple type is looked up in the method -table. +Given the call `f(x, y)`, the following steps are performed: First, a tuple type is formed, +`Tuple{typeof(f), typeof(x), typeof(y)}`. Note that the type of the function itself is the first +element. This is because the function itself participates symmetrically in method lookup with the +other arguments. This tuple type is looked up in the global method table. However, the system can +then cache the results, so these steps can be skipped later for similar lookups. This dispatch process is performed by `jl_apply_generic`, which takes two arguments: a pointer to an array of the values `f`, `x`, and `y`, and the number of values (in this case 3). @@ -48,15 +45,6 @@ jl_value_t *jl_call(jl_function_t *f, jl_value_t **args, int32_t nargs); Given the above dispatch process, conceptually all that is needed to add a new method is (1) a tuple type, and (2) code for the body of the method. `jl_method_def` implements this operation. -`jl_method_table_for` is called to extract the relevant method table from what would be -the type of the first argument. This is much more complicated than the corresponding procedure -during dispatch, since the argument tuple type might be abstract. For example, we can define: - -```julia -(::Union{Foo{Int},Foo{Int8}})(x) = 0 -``` - -which works since all possible matching methods would belong to the same method table. ## Creating generic functions @@ -93,9 +81,7 @@ end ## Constructors -A constructor call is just a call to a type. The method table for `Type` contains all -constructor definitions. All subtypes of `Type` (`Type`, `UnionAll`, `Union`, and `DataType`) -currently share a method table via special arrangement. +A constructor call is just a call to a type, to a method defined on `Type{T}`. ## Builtins @@ -117,7 +103,7 @@ function lines(words) n += length(w)+1 end end - String(take!(io)) + takestring!(io) end import Markdown [string(n) for n in names(Core;all=true) @@ -127,18 +113,14 @@ import Markdown Markdown.parse ``` -These are all singleton objects whose types are subtypes of `Builtin`, which is a subtype of -`Function`. Their purpose is to expose entry points in the run time that use the "jlcall" calling -convention: +These are mostly singleton objects all of whose types are subtypes of `Builtin`, which is a +subtype of `Function`. Their purpose is to expose entry points in the run time that use the +"jlcall" calling convention: ```c jl_value_t *(jl_value_t*, jl_value_t**, uint32_t) ``` -The method tables of builtins are empty. Instead, they have a single catch-all method cache entry -(`Tuple{Vararg{Any}}`) whose jlcall fptr points to the correct function. This is kind of a hack -but works reasonably well. - ## Keyword arguments Keyword arguments work by adding methods to the kwcall function. This function @@ -187,7 +169,7 @@ is absent. Finally there is the kwsorter definition: ``` -function (::Core.kwftype(typeof(circle)))(kws, circle, center, radius) +function (::Core.kwcall)(kws, circle, center, radius) if haskey(kws, :color) color = kws.color else @@ -205,30 +187,6 @@ function (::Core.kwftype(typeof(circle)))(kws, circle, center, radius) end ``` -The function `Core.kwftype(t)` creates the field `t.name.mt.kwsorter` (if it hasn't been created -yet), and returns the type of that function. - -This design has the feature that call sites that don't use keyword arguments require no special -handling; everything works as if they were not part of the language at all. Call sites that do -use keyword arguments are dispatched directly to the called function's kwsorter. For example the -call: - -```julia -circle((0, 0), 1.0, color = red; other...) -``` - -is lowered to: - -```julia -kwcall(merge((color = red,), other), circle, (0, 0), 1.0) -``` - -`kwcall` (also in`Core`) denotes a kwcall signature and dispatch. -The keyword splatting operation (written as `other...`) calls the named tuple `merge` function. -This function further unpacks each *element* of `other`, expecting each one to contain two values -(a symbol and a value). -Naturally, a more efficient implementation is available if all splatted arguments are named tuples. -Notice that the original `circle` function is passed through, to handle closures. ## [Compiler efficiency issues](@id compiler-efficiency-issues) @@ -251,18 +209,13 @@ sees an argument in the `Function` type hierarchy passed to a slot declared as ` it behaves as if the `@nospecialize` annotation were applied. This heuristic seems to be extremely effective in practice. -The next issue concerns the structure of method cache hash tables. Empirical studies show that -the vast majority of dynamically-dispatched calls involve one or two arguments. In turn, many -of these cases can be resolved by considering only the first argument. (Aside: proponents of single -dispatch would not be surprised by this at all. However, this argument means "multiple dispatch -is easy to optimize in practice", and that we should therefore use it, *not* "we should use single -dispatch"!) So the method cache uses the type of the first argument as its primary key. Note, -however, that this corresponds to the *second* element of the tuple type for a function call (the -first element being the type of the function itself). Typically, type variation in head position -is extremely low -- indeed, the majority of functions belong to singleton types with no parameters. -However, this is not the case for constructors, where a single method table holds constructors -for every type. Therefore the `Type` method table is special-cased to use the *first* tuple type -element instead of the second. +The next issue concerns the structure of method tables. Empirical studies show that the vast +majority of dynamically-dispatched calls involve one or two arguments. In turn, many of these cases +can be resolved by considering only the first argument. (Aside: proponents of single dispatch would +not be surprised by this at all. However, this argument means "multiple dispatch is easy to optimize +in practice", and that we should therefore use it, *not* "we should use single dispatch"!). So the +method table and cache splits up on the structure based on a left-to-right decision tree so allow +efficient nearest-neighbor searches. The front end generates type declarations for all closures. Initially, this was implemented by generating normal type declarations. However, this produced an extremely large number of constructors, diff --git a/doc/src/devdocs/gc-sa.md b/doc/src/devdocs/gc-sa.md index ffbb7451fce5f..89a285fe4ad3f 100644 --- a/doc/src/devdocs/gc-sa.md +++ b/doc/src/devdocs/gc-sa.md @@ -57,7 +57,7 @@ code base to make things work. ## GC Invariants -There is two simple invariants correctness: +There are two simple invariants for correctness: - All `GC_PUSH` calls need to be followed by an appropriate `GC_POP` (in practice we enforce this at the function level) - If a value was previously not rooted at any safepoint, it may no longer be referenced @@ -101,7 +101,7 @@ we place on a given function are indeed correct given the implementation of said ## The analyzer annotations These annotations are found in src/support/analyzer_annotations.h. -The are only active when the analyzer is being used and expand either +They are only active when the analyzer is being used and expand either to nothing (for prototype annotations) or to no-ops (for function like annotations). ### `JL_NOTSAFEPOINT` diff --git a/doc/src/devdocs/gc.md b/doc/src/devdocs/gc.md index a45e8afb271ce..b307a16735e69 100644 --- a/doc/src/devdocs/gc.md +++ b/doc/src/devdocs/gc.md @@ -23,7 +23,7 @@ Julia's pool allocator follows a "tiered" allocation discipline. When requesting - Try to claim a page from `page_pool_lazily_freed`, which contains pages which were empty on the last stop-the-world phase, but not yet madvised by a concurrent sweeper GC thread. -- If it failed claiming a page from `page_pool_lazily_freed`, it will try to claim a page from `the page_pool_clean`, which contains pages which were mmaped on a previous page allocation request but never accessed. +- If it failed claiming a page from `page_pool_lazily_freed`, it will try to claim a page from `page_pool_clean`, which contains pages which were mmaped on a previous page allocation request but never accessed. - If it failed claiming a page from `pool_page_clean` and from `page_pool_lazily_freed`, it will try to claim a page from `page_pool_freed`, which contains pages which have already been madvised by a concurrent sweeper GC thread and whose underlying virtual address can be recycled. diff --git a/doc/src/devdocs/img/invalidation-example.png b/doc/src/devdocs/img/invalidation-example.png new file mode 100644 index 0000000000000..4a80869063761 Binary files /dev/null and b/doc/src/devdocs/img/invalidation-example.png differ diff --git a/doc/src/devdocs/img/invalidation-example.svg b/doc/src/devdocs/img/invalidation-example.svg new file mode 100644 index 0000000000000..678f5100ba5df --- /dev/null +++ b/doc/src/devdocs/img/invalidation-example.svg @@ -0,0 +1,234 @@ + + + + + + +G + + + +mt + +MethodTable + + + +f_m + +Method +f() + + + +mt->f_m + + + + + +g_m + +Method +g() + + + +mt->g_m + + + + + +m_mod + +Module +M + + + +c2_bind + +Binding +M.c2 + + + +m_mod->c2_bind + + +bindings + + + +f_mi + +MethodInstance +f() + + + +f_m->f_mi + + +specializations + + + +g_mi + +MethodInstance +g() + + + +g_m->g_mi + + +specializations + + + +f_ci2 + +CodeInstance +f() for [2, ∞) + + + +f_mi->f_ci2 + + +cache + + + +g_ci1 + +CodeInstance +g() for [1, 1] + + + +f_mi->g_ci1 + + +backedges[1] + + + +g_ci2 + +CodeInstance +g() for [2, ∞) + + + +f_mi->g_ci2 + + +backedges[2] + + + +g_mi->g_ci2 + + +cache + + + +f_ci1 + +CodeInstance +f() for [1, 1] + + + +f_ci1->c2_bind + + +edges[1] + + + +f_ci2->f_ci1 + + +next + + + +f_ci2->c2_bind + + +edges[1] + + + +g_ci1->f_ci1 + + +edges[1] + + + +g_ci2->f_ci2 + + +edges[1] + + + +g_ci2->g_ci1 + + +next + + + +c2_bind->f_ci1 + + +backedges[1] + + + +c2_bind->f_ci2 + + +backedges[2] + + + +c2_bpart2 + +BindingPartition +constant 3 for [2, ∞) + + + +c2_bind->c2_bpart2 + + +partitions + + + +c2_bpart1 + +BindingPartition +constant 2 for [1, 1] + + + +c2_bpart2->c2_bpart1 + + +next + + + diff --git a/doc/src/devdocs/img/typeinf-promotion.png b/doc/src/devdocs/img/typeinf-promotion.png new file mode 100644 index 0000000000000..044575745f56e Binary files /dev/null and b/doc/src/devdocs/img/typeinf-promotion.png differ diff --git a/doc/src/devdocs/img/typeinf-promotion.svg b/doc/src/devdocs/img/typeinf-promotion.svg new file mode 100644 index 0000000000000..2bf859a70d06c --- /dev/null +++ b/doc/src/devdocs/img/typeinf-promotion.svg @@ -0,0 +1,55 @@ + + + + + + + + + T1 + + + + + acq W + promote + new method + read W + + + + + + + + + + promotefail + + + + + + + + + + + + T2 + + T3 + + + + + acq W + read W + read W+1 + rel W+1 + + + + + + diff --git a/doc/src/devdocs/init.md b/doc/src/devdocs/init.md index 1e0e1173f8695..23012d6ba1eb7 100644 --- a/doc/src/devdocs/init.md +++ b/doc/src/devdocs/init.md @@ -63,7 +63,7 @@ the [LLVM library](https://llvm.org). If there is no sysimg file (`!jl_options.image_file`) then the `Core` and `Main` modules are created and `boot.jl` is evaluated: -`jl_core_module = jl_new_module(jl_symbol("Core"))` creates the Julia `Core` module. +`jl_core_module = jl_new_module(jl_symbol("Core"), NULL)` creates the Julia `Core` module. [`jl_init_intrinsic_functions()`](https://github.com/JuliaLang/julia/blob/master/src/intrinsics.cpp) creates a new Julia module `Intrinsics` containing constant `jl_intrinsic_type` symbols. These define diff --git a/doc/src/devdocs/locks.md b/doc/src/devdocs/locks.md index 8d6672842c3c8..e05c2a07821b0 100644 --- a/doc/src/devdocs/locks.md +++ b/doc/src/devdocs/locks.md @@ -3,154 +3,233 @@ The following strategies are used to ensure that the code is dead-lock free (generally by addressing the 4th Coffman condition: circular wait). -> 1. structure code such that only one lock will need to be acquired at a time -> 2. always acquire shared locks in the same order, as given by the table below -> 3. avoid constructs that expect to need unrestricted recursion - -## Locks - -Below are all of the locks that exist in the system and the mechanisms for using them that avoid -the potential for deadlocks (no Ostrich algorithm allowed here): - -The following are definitely leaf locks (level 1), and must not try to acquire any other lock: - -> * safepoint -> -> > Note that this lock is acquired implicitly by `JL_LOCK` and `JL_UNLOCK`. use the `_NOGC` variants -> > to avoid that for level 1 locks. -> > -> > While holding this lock, the code must not do any allocation or hit any safepoints. Note that -> > there are safepoints when doing allocation, enabling / disabling GC, entering / restoring exception -> > frames, and taking / releasing locks. -> * shared_map -> * finalizers -> * pagealloc -> * gc_perm_lock -> * flisp -> * jl_in_stackwalk (Win32) -> * ResourcePool::mutex -> * RLST_mutex -> * llvm_printing_mutex -> * jl_locked_stream::mutex -> * debuginfo_asyncsafe -> * inference_timing_mutex -> * ExecutionEngine::SessionLock -> -> > flisp itself is already threadsafe, this lock only protects the `jl_ast_context_list_t` pool -> > likewise, the ResourcePool::mutexes just protect the associated resource pool - -The following is a leaf lock (level 2), and only acquires level 1 locks (safepoint) internally: - -> * global_roots_lock -> * Module->lock -> * JLDebuginfoPlugin::PluginMutex -> * newly_inferred_mutex - -The following is a level 3 lock, which can only acquire level 1 or level 2 locks internally: - -> * Method->writelock -> * typecache - -The following is a level 4 lock, which can only recurse to acquire level 1, 2, or 3 locks: - -> * MethodTable->writelock +1. structure code such that only one lock will need to be acquired at a time +2. always acquire shared locks in the same order, as given by the table below +3. avoid constructs that expect to need unrestricted recursion + +## Types of locks + +`uv_mutex_t` (or `std::mutex`) is a wrapper around platform-specific locks +(`pthread_mutex_t` on Unix, `CRITICAL_SECTION` on Windows). It may cause the +current OS thread to block, is not reentrant, and is not a safepoint. + +`jl_mutex_t` is a reentrant spinlock. `jl_mutex_t`s acquired in a `try` block +will be unlocked when we leave the block, either by reaching the end or catching +an exception. `JL_LOCK`/`JL_UNLOCK` are safepoints, while +`JL_LOCK_NOGC`/`JL_UNLOCK_NOGC` are not. `jl_mutex_t` must not be held across +task switches. + +## Lock hierarchy + +Below are all of the locks that exist in the system and the mechanisms for using +them that avoid the potential for deadlocks (no Ostrich algorithm allowed here). +Except in the special cases where a rule for avoiding deadlock is given, no lock +of a lower level may acquire a lock at a higher level. + +### Level 1 + +No other lock may be acquired when one of these locks is held. As a result, the +code must not do any allocation or hit any safepoints. Note that there are +safepoints when doing allocation, enabling/disabling GC, entering/restoring +exception frames, and taking/releasing locks. + +* `safepoint_lock` (`uv_mutex_t`) + !!! danger + + This lock is acquired implicitly by `JL_LOCK` and `JL_UNLOCK`. Use the + `_NOGC` variants to avoid that for level 1 locks. + +* `shared_map_lock.mtx` (`uv_mutex_t`) +* `finalizers_lock` (`jl_mutex_t`) +* `gc_pages_lock` (`uv_mutex_t`) +* `gc_perm_lock` (`uv_mutex_t`) +* `gc_queue_observer_lock` (`uv_mutex_t`) +* `gc_threads_lock` (`uv_mutex_t`) +* `flisp_lock` (`uv_mutex_t`) + !!! note + flisp itself is already threadsafe; this lock only protects the + `jl_ast_context_list_t` pool. Likewise, the `ResourcePool::mutexes` + just protect the associated resource pool. + +* `jl_in_stackwalk` (`uv_mutex_t`, Win32 only) +* `ResourcePool.mutex` (`std::mutex`) +* `RLST_mutex` (`std::mutex`) +* `llvm_printing_mutex` (`std::mutex`) +* `jl_locked_stream.mutex` (`std::mutex`) +* `debuginfo_asyncsafe` (`uv_rwlock_t`) +* `profile_show_peek_cond_lock` (`jl_mutex_t`) +* `trampoline_lock` (`uv_mutex_t`) +* `bt_data_prof_lock` (`uv_mutex_t`) +* `jl_ptls_t.sleep_lock` (`uv_mutex_t`) +* `tls_lock` (`uv_mutex_t`) +* `page_profile_lock` (`uv_mutex_t`) +* `symtab_lock` (`uv_mutex_t`) +* `engine_lock` (`std::mutex`) + +### Level 2 + +* `global_roots_lock` +* `jl_module_t.lock` +* `newly_inferred_mutex` +* `JLDebuginfoPlugin.PluginMutex` (`std::mutex`) +* `precompile_field_replace_lock` +* `live_tasks_lock` (`uv_mutex_t`) +* `heapsnapshot_lock` +* `jitlock` +* `jl_safepoint_suspend_all_threads` and `jl_safepoint_resume_all_threads` + !!! note + Inside a region protected by these functions, all other threads are + blocked inside a safepoint. It is unsafe to take locks that may safepoint + in this region. + +### Level 3 + +* `jl_method_t.writelock` +* `typecache_lock` +* `libmap_lock` + +### Level 4 + +* `jl_methcache_t.writelock` + +### Level 5 + +* `jl_methtable_t.writelock` + +### Level 6 No Julia code may be called while holding a lock above this point. -orc::ThreadSafeContext (TSCtx) locks occupy a special spot in the locking hierarchy. They are used to -protect LLVM's global non-threadsafe state, but there may be an arbitrary number of them. By default, -all of these locks may be treated as level 5 locks for the purposes of comparing with the rest of the -hierarchy. Acquiring a TSCtx should only be done from the JIT's pool of TSCtx's, and all locks on -that TSCtx should be released prior to returning it to the pool. If multiple TSCtx locks must be -acquired at the same time (due to recursive compilation), then locks should be acquired in the order -that the TSCtxs were borrowed from the pool. +* `world_counter_lock` -The following is a level 5 lock +### Level 7 -> * JuliaOJIT::EmissionMutex +* `JuliaOJIT::EmissionMutex` (`std::recursive_mutex`) -The following are a level 6 lock, which can only recurse to acquire locks at lower levels: +* `jl_modules_mutex` -> * jl_modules_mutex +* `jl_uv_mutex` (known as `iolock` from Julia) + !!! danger + Doing any I/O (for example, printing warning messages or debug information) + while holding any other lock listed above may result in pernicious and + hard-to-find deadlocks. -The following lock synchronizes IO operation. Be aware that doing any I/O (for example, -printing warning messages or debug information) while holding any other lock listed above -may result in pernicious and hard-to-find deadlocks. BE VERY CAREFUL! +* Individual `ThreadSynchronizer` locks + !!! danger + This may continue to be held after releasing the iolock, or acquired + without it, but be very careful to never attempt to acquire the iolock + while holding it. -> * iolock -> * Individual ThreadSynchronizers locks -> -> > this may continue to be held after releasing the iolock, or acquired without it, -> > but be very careful to never attempt to acquire the iolock while holding it -> -> * Libdl.LazyLibrary lock +* `Libdl.LazyLibrary.lock` (`ReentrantLock`) +* `orc::ThreadSafeContext` -The following is the root lock, meaning no other lock shall be held when trying to acquire it: +* `cfun_lock` -> * toplevel -> -> > this should be held while attempting a top-level action (such as making a new type or defining -> > a new method): trying to obtain this lock inside a staged function will cause a deadlock condition! -> > -> > -> > additionally, it's unclear if *any* code can safely run in parallel with an arbitrary toplevel -> > expression, so it may require all threads to get to a safepoint first +### Level 8 -## Broken Locks +* `precomp_statement_out_lock` +* `dispatch_statement_out_lock` -The following locks are broken: - - * toplevel - - > doesn't exist right now - > - > fix: create it - - * Module->lock - - > This is vulnerable to deadlocks since it can't be certain it is acquired in sequence. - > Some operations (such as `import_module`) are missing a lock. - > - > fix: replace with `jl_modules_mutex`? - - * loading.jl: `require` and `register_root_module` - - > This file potentially has numerous problems. - > - > fix: needs locks - -## Shared Global Data Structures +## Exceptions to the lock hierarchy -These data structures each need locks due to being shared mutable global state. It is the inverse -list for the above lock priority list. This list does not include level 1 leaf resources due to -their simplicity. +Ordinarily, it is forbidden to acquire locks of equal level to a lock already +held. In these specific cases we use a special protocol for acquiring locks at +the same level: -MethodTable modifications (def, cache) : MethodTable->writelock +- `jl_method_t.writelock` -Type declarations : toplevel lock + Invalidation acquires the lock for every method during its depth-first search + for backedges. To avoid deadlocks, we must already hold `world_counter_lock` + before acquiring multiple `jl_method_t.writelock`s. -Type application : typecache lock +### Broken locks -Global variable tables : Module->lock - -Module serializer : toplevel lock - -JIT & type-inference : codegen lock - -MethodInstance/CodeInstance updates : Method->writelock - -> * These are set at construction and immutable: -> * specTypes -> * sparam_vals -> * def -> * owner - -> * Function pointers: -> * these transition once, from `NULL` to a value, which is coordinated internal to the JIT -> - -Method : Method->writelock +The following locks are broken: - * roots array (serializer and codegen) - * invoke / specializations / tfunc modifications +* `loading.jl`: `require` and `register_root_module` + + This file potentially has numerous problems. (fix: needs locks) + +## Updates to the world counter + +Thanks to the [world age](@ref man-world-age) mechanism, Julia can allow the +replacement of both methods and bindings, yet remain amenable to optimization. +Every compiled `CodeInstance` has a range of valid world ages; we could +conservatively assume all CIs are stale after a world age increment. However, +to avoid spurious recompilation, we track dependencies, called "edges", while +maintaining the following invariant: + +For every published `CodeInstance`, either: +- `min_world` and `max_world` are finite, and the CI is valid for every world + in that range. +- `max_world` is ∞ (`-1`), and this CI is ready for invalidation, meaning + for every forward edge: + - If the edge is a `CodeInstance` that is invoked or inlined into this CI, + the edge's `MethodInstance` `backedge` array has an entry pointing back. + - If the edge is a `Binding`: + - If the binding is in another module, it has an entry for this CI in its + `backedges` array. + - If the binding is in the same module, the `Method` for this CI is in the + module's `scanned_methods` array. + +For example, the following code replaces a constant in another module, causing a +chain of invalidations: +```julia +const c1 = 1 +module M const c2 = 2 end +f() = getfield(M, :c2) +g() = f() + c1 + +g() # compile g + +@eval M const c2 = 3 # invalidate f, g +g() # recompile g +``` + +After compiling the two versions of `g()`, the global cache looks like this: +![Global cache state after invalidation](./img/invalidation-example.png) + +The maximum world age, `jl_world_counter`, is protected by the +`world_counter_lock`. Julia uses a form of optimistic concurrency control to +allow type inference without holding `world_counter_lock`. + +Publishing a new method or binding follows these steps: +- Acquire `world_counter_lock`. +- Relaxed-load `jl_world_counter` and let `new_world = jl_world_counter + 1`. +- Publish the new binding partitions or method table entries with world range + `[new_world, ∞)`. This step is described in the section on the [lock free + data structures](@ref man-lock-free-data). +- Release-store `new_world` to `jl_world_counter`. +- Release `world_counter_lock`. + +Type inference proceeds like so: +- Acquire-load `jl_world_counter` (call this `validation_world`). +- Perform type inference in that world, reading the bindings and method table in + that world using the lock-free data structures. +- Store back edges for every inferred `CodeInstance`: + - For non-local bindings, this acquires the binding's module's lock. + - For CIs, this acquires the method's lock. +- Acquire `world_counter_lock`. +- Relaxed-load `jl_world_counter` and compare it to `validation_world`: + - If it is different, leave the valid world ranges for the inferred CIs + unchanged. + - If it is unchanged, our optimism was rewarded. We can promote all the + inferred CIs valid in `validation_world` to `[validation_world, ∞)` and rely + on the backedges for invalidation. +- Release `world_counter_lock`. + +![Two threads doing type inference while another adds a method](./img/typeinf-promotion.png) + +In the above diagram, threads 1 and 2 are doing type inference (the dotted +line), while thread 3 is activating a new method. The solid boxes represent +critical sections where the `world_counter_lock` is held. `acq`, `rel`, and +`read`, are acquire loads, release stores, and relaxed loads respectively. + +T1 promotes its CI in time, but T2 takes too long, blocking on +`world_counter_lock` until T3 has finished publishing the new method and +incrementing the world counter. It reads `W+1` and fails to promote its CI, +leaving it with a maximum world of `W`. + +## [Lock free data structures](@id man-lock-free-data) +TODO diff --git a/doc/src/devdocs/pkgimg.md b/doc/src/devdocs/pkgimg.md index 64f4e640b7c19..0bc28b07b0c29 100644 --- a/doc/src/devdocs/pkgimg.md +++ b/doc/src/devdocs/pkgimg.md @@ -33,8 +33,10 @@ Dynamic libraries on macOS need to link against `-lSystem`. On recent macOS vers To that effect we link with `-undefined dynamic_lookup`. ## [Package images optimized for multiple microarchitectures](@id pkgimgs-multi-versioning) -Similar to [multi-versioning](@ref sysimg-multi-versioning) for system images, package images support multi-versioning. If you are in a heterogeneous environment, with a unified cache, -you can set the environment variable `JULIA_CPU_TARGET=generic` to multi-version the object caches. + +Similar to [multi-versioning](@ref sysimg-multi-versioning) for system images, package images support multi-versioning. This allows creating package caches that can run efficiently on different CPU architectures within the same environment. + +See the [`JULIA_CPU_TARGET`](@ref JULIA_CPU_TARGET) environment variable for more information on how to set the CPU target for package images. ## Flags that impact package image creation and selection diff --git a/doc/src/devdocs/sanitizers.md b/doc/src/devdocs/sanitizers.md index 5eaf4b45d9f57..dd5b518f54840 100644 --- a/doc/src/devdocs/sanitizers.md +++ b/doc/src/devdocs/sanitizers.md @@ -21,14 +21,14 @@ If you require customization or further detail, see the documentation below. ## General considerations -Using Clang's sanitizers obviously requires you to use Clang (`USECLANG=1`), but there's another +Using Clang's sanitizers obviously requires you to use Clang, but there's another catch: most sanitizers require a run-time library, provided by the host compiler, while the instrumented code generated by Julia's JIT relies on functionality from that library. This implies that the LLVM version of your host compiler must match that of the LLVM library used within Julia. An easy solution is to have a dedicated build folder for providing a matching toolchain, by building with `BUILD_LLVM_CLANG=1`. You can then refer to this toolchain from another build -folder by specifying `USECLANG=1` while overriding the `CC` and `CXX` variables. +folder by overriding the `CC` and `CXX` variables. The sanitizers error out when they detect a shared library being opened using `RTLD_DEEPBIND` (ref: [google/sanitizers#611](https://github.com/google/sanitizers/issues/611)). @@ -44,7 +44,7 @@ look like this, plus one or more of the `SANITIZE_*` flags listed below: make -C deps USE_BINARYBUILDER_LLVM=0 LLVM_VER=svn stage-llvm - make -C src SANITIZE=1 USECLANG=1 \ + make -C src SANITIZE=1 \ CC=~+/deps/scratch/llvm-svn/build_Release/bin/clang \ CXX=~+/deps/scratch/llvm-svn/build_Release/bin/clang++ \ CPPFLAGS="-isysroot $(xcode-select -p)/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" \ @@ -99,7 +99,6 @@ Checkout a Git worktree (or create out-of-tree build directory) at TOOLCHAIN=$(TOOLCHAIN_WORKTREE)/usr/tools # use our new toolchain -USECLANG=1 override CC=$(TOOLCHAIN)/clang override CXX=$(TOOLCHAIN)/clang++ export ASAN_SYMBOLIZER_PATH=$(TOOLCHAIN)/llvm-symbolizer diff --git a/doc/src/devdocs/sysimg.md b/doc/src/devdocs/sysimg.md index 2cbba2744d4a1..e8202736e57e1 100644 --- a/doc/src/devdocs/sysimg.md +++ b/doc/src/devdocs/sysimg.md @@ -176,7 +176,7 @@ debug info, respectively, and so will make debugging more difficult. We have identified many small changes to Base that significantly increase the set of programs that can be reliably trimmed. Unfortunately some of those changes would be considered breaking, and so are only applied when trimming is requested (this is done by an external build script, -currently maintained inside the test suite as `contrib/juliac-buildscript.jl`). +currently maintained inside the test suite as `contrib/juliac/juliac-buildscript.jl`). Therefore in many cases trimming will require you to opt in to new variants of Base and some standard libraries. diff --git a/doc/src/devdocs/types.md b/doc/src/devdocs/types.md index a09df61e4881d..fc4a93b94ca3c 100644 --- a/doc/src/devdocs/types.md +++ b/doc/src/devdocs/types.md @@ -176,7 +176,12 @@ julia> dump(Array{Int,1}.name) TypeName name: Symbol Array module: Module Core - names: empty SimpleVector + singletonname: Symbol Array + names: SimpleVector + 1: Symbol ref + 2: Symbol size + atomicfields: Ptr{Nothing}(0x0000000000000000) + constfields: Ptr{Nothing}(0x0000000000000000) wrapper: UnionAll var: TypeVar name: Symbol T @@ -188,21 +193,20 @@ TypeName lb: Union{} ub: abstract type Any body: mutable struct Array{T, N} <: DenseArray{T, N} + Typeofwrapper: abstract type Type{Array} <: Any cache: SimpleVector ... - linearcache: SimpleVector ... - - hash: Int64 -7900426068641098781 - mt: MethodTable - name: Symbol Array - defs: Nothing nothing - cache: Nothing nothing - max_args: Int64 0 - module: Module Core - : Int64 0 - : Int64 0 + hash: Int64 2594190783455944385 + backedges: #undef + partial: #undef + max_args: Int32 0 + n_uninitialized: Int32 0 + flags: UInt8 0x02 + cache_entry_count: UInt8 0x00 + max_methods: UInt8 0x00 + constprop_heuristic: UInt8 0x00 ``` In this case, the relevant field is `wrapper`, which holds a reference to the top-level type used diff --git a/doc/src/index.md b/doc/src/index.md index 8c88af424e8e3..8342ff448625d 100644 --- a/doc/src/index.md +++ b/doc/src/index.md @@ -37,7 +37,7 @@ Markdown.parse(""" Below is a non-exhaustive list of links that will be useful as you learn and use the Julia programming language. - [Julia Homepage](https://julialang.org) -- [Download Julia](https://julialang.org/downloads/) +- [Install Julia](https://julialang.org/install/) - [Discussion forum](https://discourse.julialang.org) - [Julia YouTube](https://www.youtube.com/user/JuliaLanguage) - [Find Julia Packages](https://julialang.org/packages/) diff --git a/doc/src/manual/arrays.md b/doc/src/manual/arrays.md index ee9421ba1730b..ba2d261301b40 100644 --- a/doc/src/manual/arrays.md +++ b/doc/src/manual/arrays.md @@ -103,7 +103,7 @@ same type, then that is its `eltype`. If they all have a common [promotion type](@ref conversion-and-promotion) then they get converted to that type using [`convert`](@ref) and that type is the array's `eltype`. Otherwise, a heterogeneous array that can hold anything — a `Vector{Any}` — is constructed; this includes the literal `[]` -where no arguments are given. [Array literal can be typed](@ref man-array-typed-literal) with +where no arguments are given. [Array literals can be typed](@ref man-array-typed-literal) with the syntax `T[A, B, C, ...]` where `T` is a type. ```jldoctest @@ -368,26 +368,26 @@ of the variable ranges `rx`, `ry`, etc. and each `F(x,y,...)` evaluation returns The following example computes a weighted average of the current element and its left and right neighbor along a 1-d grid: -```julia-repl -julia> x = rand(8) -8-element Vector{Float64}: - 0.843025 - 0.869052 - 0.365105 - 0.699456 - 0.977653 - 0.994953 - 0.41084 - 0.809411 +```jldoctest +julia> x = [4, 8, 2, 6, 10, 10, 2, 8] +8-element Vector{Int64}: + 4 + 8 + 2 + 6 + 10 + 10 + 2 + 8 julia> [ 0.25*x[i-1] + 0.5*x[i] + 0.25*x[i+1] for i=2:length(x)-1 ] 6-element Vector{Float64}: - 0.736559 - 0.57468 - 0.685417 - 0.912429 - 0.8446 - 0.656511 + 5.5 + 4.5 + 6.0 + 9.0 + 8.0 + 5.5 ``` The resulting array type depends on the types of the computed elements just like [array literals](@ref man-array-literals) do. In order to control the @@ -413,9 +413,12 @@ julia> sum(1/n^2 for n=1:1000) When writing a generator expression with multiple dimensions inside an argument list, parentheses are needed to separate the generator from subsequent arguments: -```julia-repl +```jldoctest julia> map(tuple, 1/(i+j) for i=1:2, j=1:2, [1:4;]) -ERROR: syntax: invalid iteration specification +ERROR: ParseError: +# Error @ none:1:44 +map(tuple, 1/(i+j) for i=1:2, j=1:2, [1:4;]) +# └ ── invalid iteration spec: expected one of `=` `in` or `∈` ``` All comma-separated expressions after `for` are interpreted as ranges. Adding parentheses lets @@ -1036,33 +1039,33 @@ It is sometimes useful to perform element-by-element binary operations on arrays sizes, such as adding a vector to each column of a matrix. An inefficient way to do this would be to replicate the vector to the size of the matrix: -```julia-repl -julia> a = rand(2, 1); A = rand(2, 3); +```jldoctest broadcast_example +julia> a = [0.2, 0.5]; A = [1.0 1.6 1.05; 1.07 1.36 1.18]; julia> repeat(a, 1, 3) + A 2×3 Matrix{Float64}: - 1.20813 1.82068 1.25387 - 1.56851 1.86401 1.67846 + 1.2 1.8 1.25 + 1.57 1.86 1.68 ``` This is wasteful when dimensions get large, so Julia provides [`broadcast`](@ref), which expands singleton dimensions in array arguments to match the corresponding dimension in the other array without using extra memory, and applies the given function elementwise: -```julia-repl +```jldoctest broadcast_example julia> broadcast(+, a, A) 2×3 Matrix{Float64}: - 1.20813 1.82068 1.25387 - 1.56851 1.86401 1.67846 + 1.2 1.8 1.25 + 1.57 1.86 1.68 -julia> b = rand(1,2) +julia> b = [0.9 0.1] 1×2 Matrix{Float64}: - 0.867535 0.00457906 + 0.9 0.1 julia> broadcast(+, a, b) 2×2 Matrix{Float64}: - 1.71056 0.847604 - 1.73659 0.873631 + 1.1 0.3 + 1.4 0.6 ``` [Dotted operators](@ref man-dot-operators) such as `.+` and `.*` are equivalent diff --git a/doc/src/manual/asynchronous-programming.md b/doc/src/manual/asynchronous-programming.md index d1d095c48b2ff..eccd924aec6b3 100644 --- a/doc/src/manual/asynchronous-programming.md +++ b/doc/src/manual/asynchronous-programming.md @@ -162,7 +162,7 @@ constructors to explicitly link a set of channels with a set of producer/consume ### More on Channels -A channel can be visualized as a pipe, i.e., it has a write end and a read end : +A channel can be visualized as a pipe, i.e., it has a write end and a read end: * Multiple writers in different tasks can write to the same channel concurrently via [`put!`](@ref) calls. @@ -203,7 +203,7 @@ A channel can be visualized as a pipe, i.e., it has a write end and a read end : freely via [`take!`](@ref) and [`put!`](@ref) calls. [`close`](@ref) closes a [`Channel`](@ref). On a closed [`Channel`](@ref), [`put!`](@ref) will fail. For example: - ```julia-repl + ```jldoctest channel_example julia> c = Channel(2); julia> put!(c, 1) # `put!` on an open channel succeeds @@ -220,7 +220,7 @@ A channel can be visualized as a pipe, i.e., it has a write end and a read end : * [`take!`](@ref) and [`fetch`](@ref) (which retrieves but does not remove the value) on a closed channel successfully return any existing values until it is emptied. Continuing the above example: - ```julia-repl + ```jldoctest channel_example julia> fetch(c) # Any number of `fetch` calls succeed. 1 diff --git a/doc/src/manual/calling-c-and-fortran-code.md b/doc/src/manual/calling-c-and-fortran-code.md index aa317468b0f75..bbc3ccdeb6fc0 100644 --- a/doc/src/manual/calling-c-and-fortran-code.md +++ b/doc/src/manual/calling-c-and-fortran-code.md @@ -69,7 +69,7 @@ julia> unsafe_string(path) In practice, especially when providing reusable functionality, one generally wraps `@ccall` uses in Julia functions that set up arguments and then check for errors in whatever manner the -C or Fortran function specifies. And if an error occurs it is thrown as a normal Julia exception. This is especially +C or Fortran function specifies. If an error occurs it is thrown as a normal Julia exception. This is especially important since C and Fortran APIs are notoriously inconsistent about how they indicate error conditions. For example, the `getenv` C library function is wrapped in the following Julia function, which is a simplified version of the actual definition from [`env.jl`](https://github.com/JuliaLang/julia/blob/master/base/env.jl): @@ -224,7 +224,7 @@ julia> A ``` As the example shows, the original Julia array `A` has now been sorted: `[-2.7, 1.3, 3.1, 4.4]`. Note that Julia -[takes care of converting the array to a `Ptr{Cdouble}`](@ref automatic-type-conversion)), computing +[takes care of converting the array to a `Ptr{Cdouble}`](@ref automatic-type-conversion), computing the size of the element type in bytes, and so on. For fun, try inserting a `println("mycompare($a, $b)")` line into `mycompare`, which will allow @@ -357,7 +357,7 @@ an `Int` in Julia). | `unsigned long long` | | `Culonglong` | `UInt64` | | `intmax_t` | | `Cintmax_t` | `Int64` | | `uintmax_t` | | `Cuintmax_t` | `UInt64` | -| `float` | `REAL*4i` | `Cfloat` | `Float32` | +| `float` | `REAL*4` | `Cfloat` | `Float32` | | `double` | `REAL*8` | `Cdouble` | `Float64` | | `complex float` | `COMPLEX*8` | `ComplexF32` | `Complex{Float32}` | | `complex double` | `COMPLEX*16` | `ComplexF64` | `Complex{Float64}` | @@ -1015,7 +1015,7 @@ be a calling convention specifier (the `@ccall` macro currently does not support giving a calling convention). Without any specifier, the platform-default C calling convention is used. Other supported conventions are: `stdcall`, `cdecl`, `fastcall`, and `thiscall` (no-op on 64-bit Windows). For example (from -`base/libc.jl`) we see the same `gethostname``ccall` as above, but with the +`base/libc.jl`) we see the same `gethostname` `ccall` as above, but with the correct signature for Windows: ```julia diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 24e64b0ca068e..301f5e5def618 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -36,12 +36,15 @@ An *environment* determines what `import X` and `using X` mean in various code c These can be intermixed to create **a stacked environment**: an ordered set of project environments and package directories, overlaid to make a single composite environment. The precedence and visibility rules then combine to determine which packages are available and where they get loaded from. Julia's load path forms a stacked environment, for example. -These environment each serve a different purpose: +These environments each serve a different purpose: * Project environments provide **reproducibility**. By checking a project environment into version control—e.g. a git repository—along with the rest of the project's source code, you can reproduce the exact state of the project and all of its dependencies. The manifest file, in particular, captures the exact version of every dependency, identified by a cryptographic hash of its source tree, which makes it possible for `Pkg` to retrieve the correct versions and be sure that you are running the exact code that was recorded for all dependencies. * Package directories provide **convenience** when a full carefully-tracked project environment is unnecessary. They are useful when you want to put a set of packages somewhere and be able to directly use them, without needing to create a project environment for them. * Stacked environments allow for **adding** tools to the primary environment. You can push an environment of development tools onto the end of the stack to make them available from the REPL and scripts, but not from inside packages. +!!! note + When loading a package from another environment in the stack other than the active environment the package is loaded in the context of the active environment. This means that the package will be loaded as if it were imported in the active environment, which may affect how its dependencies versions are resolved. When such a package is precompiling it will be marked as a `(serial)` precompile job, which means that its dependencies will be precompiled in series within the same job, which will likely be slower. + At a high-level, each environment conceptually defines three maps: roots, graph and paths. When resolving the meaning of `import X`, the roots and graph maps are used to determine the identity of `X`, while the paths map is used to locate the source code of `X`. The specific roles of the three maps are: - **roots:** `name::Symbol` ⟶ `uuid::UUID` @@ -123,7 +126,7 @@ This manifest file describes a possible complete dependency graph for the `App` - There are two different packages named `Priv` that the application uses. It uses a private package, which is a root dependency, and a public one, which is an indirect dependency through `Pub`. These are differentiated by their distinct UUIDs, and they have different deps: * The private `Priv` depends on the `Pub` and `Zebra` packages. * The public `Priv` has no dependencies. -- The application also depends on the `Pub` package, which in turn depends on the public `Priv ` and the same `Zebra` package that the private `Priv` package depends on. +- The application also depends on the `Pub` package, which in turn depends on the public `Priv` and the same `Zebra` package that the private `Priv` package depends on. This dependency graph represented as a dictionary, looks like this: @@ -155,7 +158,7 @@ graph[UUID("c07ecb7d-0dc9-4db7-8803-fadaaeaf08e1")][:Priv] and gets `2d15fe94-a1f7-436c-a4d8-07a9a496e01c`, which indicates that in the context of the `Pub` package, `import Priv` refers to the public `Priv` package, rather than the private one which the app depends on directly. This is how the name `Priv` can refer to different packages in the main project than it does in one of its package's dependencies, which allows for duplicate names in the package ecosystem. -What happens if `import Zebra` is evaluated in the main `App` code base? Since `Zebra` does not appear in the project file, the import will fail even though `Zebra` *does* appear in the manifest file. Moreover, if `import Zebra` occurs in the public `Priv` package—the one with UUID `2d15fe94-a1f7-436c-a4d8-07a9a496e01c`—then that would also fail since that `Priv` package has no declared dependencies in the manifest file and therefore cannot load any packages. The `Zebra` package can only be loaded by packages for which it appear as an explicit dependency in the manifest file: the `Pub` package and one of the `Priv` packages. +What happens if `import Zebra` is evaluated in the main `App` code base? Since `Zebra` does not appear in the project file, the import will fail even though `Zebra` *does* appear in the manifest file. Moreover, if `import Zebra` occurs in the public `Priv` package—the one with UUID `2d15fe94-a1f7-436c-a4d8-07a9a496e01c`—then that would also fail since that `Priv` package has no declared dependencies in the manifest file and therefore cannot load any packages. The `Zebra` package can only be loaded by packages for which it appears as an explicit dependency in the manifest file: the `Pub` package and one of the `Priv` packages. **The paths map** of a project environment is extracted from the manifest file. The path of a package `uuid` named `X` is determined by these rules (in order): @@ -191,7 +194,7 @@ paths = Dict( # Priv – the public one: (UUID("2d15fe94-a1f7-436c-a4d8-07a9a496e01c"), :Priv) => # package installed in the system depot: - "/usr/local/julia/packages/Priv/HDkr/src/Priv.jl", + "/usr/local/julia/packages/Priv/HDkrT/src/Priv.jl", # Pub: (UUID("c07ecb7d-0dc9-4db7-8803-fadaaeaf08e1"), :Pub) => # package installed in the user depot: diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 14dd60d89b384..b8ae83620da50 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -41,7 +41,7 @@ See also [Scripting](@ref man-scripting) for more information on writing Julia s ## The `Main.main` entry point -As of Julia, 1.11, `Base` exports the macro `@main`. This macro expands to the symbol `main`, +As of Julia 1.11, `Base` exports the macro `@main`. This macro expands to the symbol `main`, but at the conclusion of executing a script or expression, `julia` will attempt to execute `Main.main(Base.ARGS)` if such a function `Main.main` has been defined and this behavior was opted into by using the `@main` macro. @@ -148,7 +148,8 @@ atreplinit() do repl # ... end ``` - +If [`JULIA_DEPOT_PATH`](@ref JULIA_DEPOT_PATH) is set, the startup file should be located there: +`$JULIA_DEPOT_PATH/config/startup.jl`. ## [Command-line switches for Julia](@id command-line-interface) @@ -176,10 +177,10 @@ The following is a complete list of command-line switches available when launchi |`--pkgimages={yes*\|no\|existing}` |Enable or disable usage of native code caching in the form of pkgimages. The `existing` option allows use of existing pkgimages but disallows creation of new ones| |`-e`, `--eval ` |Evaluate ``| |`-E`, `--print ` |Evaluate `` and display the result| -|`-m`, `--module [args]` |Run entry point of `Package` (`@main` function) with `args'| +|`-m`, `--module [args]` |Run entry point of `Package` (`@main` function) with `args`| |`-L`, `--load ` |Load `` immediately on all processors| |`-t`, `--threads {auto\|N[,auto\|M]}` |Enable N[+M] threads; N threads are assigned to the `default` threadpool, and if M is specified, M threads are assigned to the `interactive` threadpool; `auto` tries to infer a useful default number of threads to use but the exact behavior might change in the future. Currently sets N to the number of CPUs assigned to this Julia process based on the OS-specific affinity assignment interface if supported (Linux and Windows) or to the number of CPU threads if not supported (MacOS) or if process affinity is not configured, and sets M to 1.| -| `--gcthreads=N[,M]` |Use N threads for the mark phase of GC and M (0 or 1) threads for the concurrent sweeping phase of GC. N is set to the number of compute threads and M is set to 0 if unspecified.| +| `--gcthreads=N[,M]` |Use N threads for the mark phase of GC and M (0 or 1) threads for the concurrent sweeping phase of GC. N is set to the number of compute threads and M is set to 0 if unspecified. See [Memory Management and Garbage Collection](@ref man-memory-management) for more details.| |`-p`, `--procs {N\|auto}` |Integer value N launches N additional local worker processes; `auto` launches as many workers as the number of local CPU threads (logical cores)| |`--machine-file ` |Run processes on hosts listed in ``| |`-i`, `--interactive` |Interactive mode; REPL runs and `isinteractive()` is true| @@ -194,7 +195,7 @@ The following is a complete list of command-line switches available when launchi |`-O`, `--optimize={0\|1\|2*\|3}` |Set the optimization level (level is 3 if `-O` is used without a level) ($)| |`--min-optlevel={0*\|1\|2\|3}` |Set the lower bound on per-module optimization| |`-g`, `--debug-info={0\|1*\|2}` |Set the level of debug info generation (level is 2 if `-g` is used without a level) ($)| -|`--inline={yes\|no}` |Control whether inlining is permitted, including overriding `@inline` declarations| +|`--inline={yes*\|no}` |Control whether inlining is permitted, including overriding `@inline` declarations| |`--check-bounds={yes\|no\|auto*}` |Emit bounds checks always, never, or respect `@inbounds` declarations ($)| |`--math-mode={ieee\|user*}` |Always follow `ieee` floating point semantics or respect `@fastmath` declarations| |`--polly={yes*\|no}` |Enable or disable the polyhedral optimizer Polly (overrides @polly declaration)| @@ -205,7 +206,7 @@ The following is a complete list of command-line switches available when launchi |`--track-allocation=@` |Count bytes but only in files that fall under the given file path/directory. The `@` prefix is required to select this option. A `@` with no path will track the current directory.| |`--task-metrics={yes\|no*}` |Enable the collection of per-task metrics| |`--bug-report=KIND` |Launch a bug report session. It can be used to start a REPL, run a script, or evaluate expressions. It first tries to use BugReporting.jl installed in current environment and falls back to the latest compatible BugReporting.jl if not. For more information, see `--bug-report=help`.| -|`--heap-size-hint=` |Forces garbage collection if memory usage is higher than the given value. The value may be specified as a number of bytes, optionally in units of KB, MB, GB, or TB, or as a percentage of physical memory with %.| +|`--heap-size-hint=` |Forces garbage collection if memory usage is higher than the given value. The value may be specified as a number of bytes, optionally in units of KB, MB, GB, or TB, or as a percentage of physical memory with %. See [Memory Management and Garbage Collection](@ref man-memory-management) for more details.| |`--compile={yes*\|no\|all\|min}` |Enable or disable JIT compiler, or request exhaustive or minimal compilation| |`--output-o ` |Generate an object file (including system image data)| |`--output-ji ` |Generate a system image data file (.ji)| diff --git a/doc/src/manual/distributed-computing.md b/doc/src/manual/distributed-computing.md index 873a94ffb2181..9addd491fcf79 100644 --- a/doc/src/manual/distributed-computing.md +++ b/doc/src/manual/distributed-computing.md @@ -197,7 +197,7 @@ loaded From worker 2: loaded ``` -As usual, this does not bring `DummyModule` into scope on any of the process, which requires +As usual, this does not bring `DummyModule` into scope on any of the processes, which requires [`using`](@ref) or [`import`](@ref). Moreover, when `DummyModule` is brought into scope on one process, it is not on any other: diff --git a/doc/src/manual/documentation.md b/doc/src/manual/documentation.md index aa58e5b600f49..dff410c74aa2e 100644 --- a/doc/src/manual/documentation.md +++ b/doc/src/manual/documentation.md @@ -34,9 +34,11 @@ The basic syntax is simple: any string appearing just before an object the documented object. Here is a basic example: ```julia -"Tell whether there are too foo items in the array." +"Tell whether there are too many foo items in the array." foo(xs::Array) = ... ``` +!!! note "Reminder" + Any empty lines between the docstring and the object being documented detach the former from the latter, making the docstring ineffective. Documentation is interpreted as [Markdown](https://en.wikipedia.org/wiki/Markdown), so you can use indentation and code fences to delimit code examples from text. Technically, any object can @@ -331,6 +333,10 @@ documentation between different versions of a function: @doc (@doc foo!) foo ``` +!!! compat "Julia 1.11" + In Julia 1.11 and newer, retrieving documentation with the `@doc` macro requires that + the `REPL` stdlib is loaded. + Or for use with Julia's metaprogramming functionality: ```julia diff --git a/doc/src/manual/embedding.md b/doc/src/manual/embedding.md index 64104771cd9cd..7fb40ba5bd649 100644 --- a/doc/src/manual/embedding.md +++ b/doc/src/manual/embedding.md @@ -54,7 +54,7 @@ linking against `libjulia`. The first thing that must be done before calling any other Julia C function is to initialize Julia. This is done by calling `jl_init`, which tries to automatically determine Julia's install location. If you need to specify a custom location, or specify which system -image to load, use `jl_init_with_image` instead. +image to load, use `jl_init_with_image_file` or `jl_init_with_image_handle` instead. The second statement in the test program evaluates a Julia statement using a call to `jl_eval_string`. diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index ff505db1f11f2..4a3018c481276 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -479,9 +479,15 @@ stored in memory. Valid values for [`JULIA_CPU_TARGET`](@ref JULIA_CPU_TARGET) can be obtained by executing `julia -C help`. +To get the CPU target string that was used to build the current system image, +use [`Sys.sysimage_target()`](@ref). This can be useful for reproducing +the same system image or understanding what CPU features were enabled during compilation. + Setting [`JULIA_CPU_TARGET`](@ref JULIA_CPU_TARGET) is important for heterogeneous compute systems where processors of distinct types or features may be present. This is commonly encountered in high performance -computing (HPC) clusters since the component nodes may be using distinct processors. +computing (HPC) clusters since the component nodes may be using distinct processors. In this case, +you may want to use the `sysimage` CPU target to maintain the same configuration as the sysimage. +See below for more details. The CPU target string is a list of strings separated by `;` each string starts with a CPU or architecture name and followed by an optional list of features separated by `,`. @@ -489,14 +495,26 @@ A `generic` or empty CPU name means the basic required feature set of the target which is at least the architecture the C/C++ runtime is compiled with. Each string is interpreted by LLVM. +!!! note + Package images can only target the same or more specific CPU features than + their base system image. + A few special features are supported: -1. `clone_all` + +1. `sysimage` + + A special keyword that can be used as a CPU target name, which will be replaced + with the CPU target string that was used to build the current system image. This allows + you to specify CPU targets that build upon or extend the current sysimage's target, which + is particularly helpful for creating package images that are as flexible as the sysimage. + +2. `clone_all` This forces the target to have all functions in sysimg cloned. When used in negative form (i.e. `-clone_all`), this disables full clone that's enabled by default for certain targets. -2. `base([0-9]*)` +3. `base([0-9]*)` This specifies the (0-based) base target index. The base target is the target that the current target is based on, i.e. the functions that are not being cloned @@ -504,11 +522,11 @@ A few special features are supported: fully cloned (as if `clone_all` is specified for it) if it is not the default target (0). The index can only be smaller than the current index. -3. `opt_size` +4. `opt_size` Optimize for size with minimum performance impact. Clang/GCC's `-Os`. -4. `min_size` +5. `min_size` Optimize only for size. Clang's `-Oz`. @@ -530,17 +548,6 @@ Allows you to enable or disable zones for a specific Julia run. For instance, setting the variable to `+GC,-INFERENCE` will enable the `GC` zones and disable the `INFERENCE` zones. See [Dynamically Enabling and Disabling Zones](@ref). -### [`JULIA_GC_NO_GENERATIONAL`](@id JULIA_GC_NO_GENERATIONAL) - -If set to anything besides `0`, then the Julia garbage collector never performs -"quick sweeps" of memory. - -!!! note - - This environment variable only has an effect if Julia was compiled with - garbage-collection debugging (that is, if `WITH_GC_DEBUG_ENV` is set to `1` - in the build configuration). - ### [`JULIA_GC_WAIT_FOR_DEBUGGER`](@id JULIA_GC_WAIT_FOR_DEBUGGER) If set to anything besides `0`, then the Julia garbage collector will wait for diff --git a/doc/src/manual/faq.md b/doc/src/manual/faq.md index be4f331cd6233..16c17a92e6b3f 100644 --- a/doc/src/manual/faq.md +++ b/doc/src/manual/faq.md @@ -391,6 +391,12 @@ julia> twothreearr() 3 ``` +### Is a function that ends with `!` allowed to allocate? + +Yes! A function name ending with `!` indicates that the function mutates at +least one of its arguments (typically the first argument). However, it may +still allocate a scratch space to expedite computation or produce that result. + ## Types, type declarations, and constructors ### [What does "type-stable" mean?](@id man-type-stability) @@ -1090,8 +1096,7 @@ You may wish to test against the nightly version to ensure that such regressions Finally, you may also consider building Julia from source for yourself. This option is mainly for those individuals who are comfortable at the command line, or interested in learning. If this describes you, you may also be interested in reading our [guidelines for contributing](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md). -Links to each of these download types can be found on the download page at [https://julialang.org/downloads/](https://julialang.org/downloads/). -Note that not all versions of Julia are available for all platforms. +The [`juliaup` install manager](https://julialang.org/install/) has pre-defined channels named `release` and `lts` for the latest stable release and the current LTS release, as well as version-specific channels. ### How can I transfer the list of installed packages after updating my version of Julia? diff --git a/doc/src/manual/functions.md b/doc/src/manual/functions.md index fd97b44cdc345..b2c72cb648d6b 100644 --- a/doc/src/manual/functions.md +++ b/doc/src/manual/functions.md @@ -65,18 +65,22 @@ a function will be visible to the caller. (This is the same behavior found in Sc Python, Ruby and Perl, among other dynamic languages.) For example, in the function -```julia +```jldoctest argpassing; output = false function f(x, y) x[1] = 42 # mutates x y = 7 + y # new binding for y, no mutation return y end + +# output + +f (generic function with 1 method) ``` The statement `x[1] = 42` *mutates* the object `x`, and hence this change *will* be visible in the array passed by the caller for this argument. On the other hand, the assignment `y = 7 + y` changes the *binding* ("name") `y` to refer to a new value `7 + y`, rather than mutating the *original* object referred to by `y`, and hence does *not* change the corresponding argument passed by the caller. This can be seen if we call `f(x, y)`: -```julia-repl +```jldoctest argpassing julia> a = [4, 5, 6] 3-element Vector{Int64}: 4 @@ -174,7 +178,7 @@ julia> function hypot(x, y) return x*sqrt(1 + r*r) end if y == 0 - return x + return float(x) end r = x/y return y*sqrt(1 + r*r) diff --git a/doc/src/manual/getting-started.md b/doc/src/manual/getting-started.md index 2c69aabbda192..502fbd59166f7 100644 --- a/doc/src/manual/getting-started.md +++ b/doc/src/manual/getting-started.md @@ -1,7 +1,7 @@ # [Getting Started](@id man-getting-started) Julia installation is straightforward, whether using precompiled binaries or compiling from source. -Download and install Julia by following the instructions at [https://julialang.org/downloads/](https://julialang.org/downloads/). +Download and install Julia by following the instructions at [https://julialang.org/install/](https://julialang.org/install/). If you are coming to Julia from one of the following languages, then you should start by reading the section on noteworthy differences from [MATLAB](@ref Noteworthy-differences-from-MATLAB), [R](@ref Noteworthy-differences-from-R), [Python](@ref Noteworthy-differences-from-Python), [C/C++](@ref Noteworthy-differences-from-C/C) or [Common Lisp](@ref Noteworthy-differences-from-Common-Lisp). This will help you avoid some common pitfalls since Julia differs from those languages in many subtle ways. @@ -13,7 +13,7 @@ known as a read-eval-print loop or "REPL") by double-clicking the Julia executab using REPL io = IOBuffer() REPL.banner(io) -banner = String(take!(io)) +banner = takestring!(io) import Markdown Markdown.parse("```\n\$ julia\n\n$(banner)\njulia> 1 + 2\n3\n\njulia> ans\n3\n```") ``` diff --git a/doc/src/manual/installation.md b/doc/src/manual/installation.md index f45aba2c37a28..60a52e8cb6e19 100644 --- a/doc/src/manual/installation.md +++ b/doc/src/manual/installation.md @@ -18,7 +18,7 @@ On Windows Julia can be installed directly from the Windows store exactly the same version by executing ``` -winget install julia -s msstore +winget install --name Julia --id 9NJNWW8PVKMN -e -s msstore ``` in any shell. diff --git a/doc/src/manual/integers-and-floating-point-numbers.md b/doc/src/manual/integers-and-floating-point-numbers.md index fa0ee228e873b..845d42e33abfb 100644 --- a/doc/src/manual/integers-and-floating-point-numbers.md +++ b/doc/src/manual/integers-and-floating-point-numbers.md @@ -59,7 +59,7 @@ julia> 1234 The default type for an integer literal depends on whether the target system has a 32-bit architecture or a 64-bit architecture: -```julia-repl +```julia-repl ; nodoctest = "Results depend on system word size" # 32-bit system: julia> typeof(1) Int32 @@ -72,7 +72,7 @@ Int64 The Julia internal variable [`Sys.WORD_SIZE`](@ref) indicates whether the target system is 32-bit or 64-bit: -```julia-repl +```julia-repl ; nodoctest = "Results depend on system word size" # 32-bit system: julia> Sys.WORD_SIZE 32 @@ -85,7 +85,7 @@ julia> Sys.WORD_SIZE Julia also defines the types `Int` and `UInt`, which are aliases for the system's signed and unsigned native integer types respectively: -```julia-repl +```julia-repl ; nodoctest = "Results depend on system word size" # 32-bit system: julia> Int Int32 @@ -600,7 +600,7 @@ julia> parse(BigFloat, "1.23456789012345678901") 1.234567890123456789010000000000000000000000000000000000000000000000000000000004 julia> BigFloat(2.0^66) / 3 -2.459565876494606882133333333333333333333333333333333333333333333333333333333344e+19 +2.459565876494606882133333333333333333333333333333333333333333333333333333333344e19 julia> factorial(BigInt(40)) 815915283247897734345611269596115894272000000000 diff --git a/doc/src/manual/mathematical-operations.md b/doc/src/manual/mathematical-operations.md index d2cef68bd6fff..20abf40917a18 100644 --- a/doc/src/manual/mathematical-operations.md +++ b/doc/src/manual/mathematical-operations.md @@ -47,18 +47,6 @@ julia> 3*2/12 operators. For instance, we would generally write `-x + 2` to reflect that first `x` gets negated, and then `2` is added to that result.) -When used in multiplication, `false` acts as a *strong zero*: - -```jldoctest -julia> NaN * false -0.0 - -julia> false * Inf -0.0 -``` - -This is useful for preventing the propagation of `NaN` values in quantities that are known to be zero. See [Knuth (1992)](https://arxiv.org/abs/math/9205211) for motivation. - ## Boolean Operators The following [Boolean operators](https://en.wikipedia.org/wiki/Boolean_algebra#Operations) are supported on [`Bool`](@ref) types: @@ -71,7 +59,29 @@ The following [Boolean operators](https://en.wikipedia.org/wiki/Boolean_algebra# Negation changes `true` to `false` and vice versa. The short-circuiting operations are explained on the linked page. -Note that `Bool` is an integer type and all the usual promotion rules and numeric operators are also defined on it. +## Arithmetic operations with `Bool` values + +Note that `Bool` is an integer type, such that `false` is numerically equal to `0` and `true` is numerically equal to `1`. All the usual promotion rules and numeric operators are also defined on it, with a special behavior of arithmetic (non-Boolean) operations when all the arguments are `Bool`: in those cases, the arguments are promoted to `Int` instead of keeping their type. Compare e.g. the following equivalent operations with `Bool` and with a different numeric type (`UInt8`): + +```jldoctest +julia> true - true +0 + +julia> 0x01 - 0x01 +0x00 +``` + +Also, when used in multiplication, `false` acts as a *strong zero*: + +```jldoctest +julia> NaN * false +0.0 + +julia> false * Inf +0.0 +``` + +This is useful for preventing the propagation of `NaN` values in quantities that are known to be zero. See [Knuth (1992)](https://arxiv.org/abs/math/9205211) for motivation. ## Bitwise Operators diff --git a/doc/src/manual/memory-management.md b/doc/src/manual/memory-management.md new file mode 100644 index 0000000000000..4efa683e3f249 --- /dev/null +++ b/doc/src/manual/memory-management.md @@ -0,0 +1,177 @@ +# [Memory Management and Garbage Collection](@id man-memory-management) + +Julia uses automatic memory management through its built-in garbage collector (GC). This section provides an overview of how Julia manages memory and how you can configure and optimize memory usage for your applications. + +## [Garbage Collection Overview](@id man-gc-overview) + +Julia features a garbage collector with the following characteristics: + +* **Non-moving**: Objects are not relocated in memory during garbage collection +* **Generational**: Younger objects are collected more frequently than older ones +* **Parallel and partially concurrent**: The GC can use multiple threads and run concurrently with your program +* **Mostly precise**: The GC accurately identifies object references for pure Julia code, and it provides conservative scanning APIs for users calling Julia from C + +The garbage collector automatically reclaims memory used by objects that are no longer reachable from your program, freeing you from manual memory management in most cases. + +## [Memory Architecture](@id man-memory-architecture) + +Julia uses a two-tier allocation strategy: + +* **Small objects** (currently ≤ 2032 bytes but may change): Allocated using a fast per-thread pool allocator +* **Large objects** : Allocated directly through the system's `malloc` + +This hybrid approach optimizes for both allocation speed and memory efficiency, with the pool allocator providing fast allocation for the many small objects typical in Julia programs. + +## [System Memory Requirements](@id man-system-memory) + +### Swap Space + +Julia's garbage collector is designed with the expectation that your system has adequate swap space configured. The GC uses heuristics that assume it can allocate memory beyond physical RAM when needed, relying on the operating system's virtual memory management. + +If your system has limited or no swap space, you may experience out-of-memory errors during garbage collection. In such cases, you can use the `--heap-size-hint` option to limit Julia's memory usage. + +### Memory Hints + +You can provide a hint to Julia about the maximum amount of memory to use: + +```bash +julia --heap-size-hint=4G # To set the hint to ~4GB +julia --heap-size-hint=50% # or to 50% of physical memory +``` + +The `--heap-size-hint` option tells the garbage collector to trigger collection more aggressively when approaching the specified limit. This is particularly useful in: + +* Containers with memory limits +* Systems without swap space +* Shared systems where you want to limit Julia's memory footprint + +You can also set this via the `JULIA_HEAP_SIZE_HINT` environment variable: + +```bash +export JULIA_HEAP_SIZE_HINT=2G +julia +``` + +## [Multithreaded Garbage Collection](@id man-gc-multithreading) + +Julia's garbage collector can leverage multiple threads to improve performance on multi-core systems. + +### GC Thread Configuration + +By default, Julia uses multiple threads for garbage collection: + +* **Mark threads**: Used during the mark phase to trace object references (default: 1, which is shared with the compute thread if there is only one, otherwise half the number of compute threads) +* **Sweep threads**: Used for concurrent sweeping of freed memory (default: 0, disabled) + +You can configure GC threading using: + +```bash +julia --gcthreads=4,1 # 4 mark threads, 1 sweep thread +julia --gcthreads=8 # 8 mark threads, 0 sweep threads +``` + +Or via environment variable: + +```bash +export JULIA_NUM_GC_THREADS=4,1 +julia +``` + +### Recommendations + +For compute-intensive workloads: + +* Use multiple mark threads (the default configuration is usually appropriate) +* Consider enabling concurrent sweeping with 1 sweep thread for allocation-heavy workloads + +For memory-intensive workloads: + +* Enable concurrent sweeping to reduce GC pauses +* Monitor GC time using `@time` and adjust thread counts accordingly + +## [Monitoring and Debugging](@id man-gc-monitoring) + +### Basic Memory Monitoring + +Use the `@time` macro to see memory allocation and GC overhead: + +```julia +julia> @time some_computation() + 2.123456 seconds (1.50 M allocations: 58.725 MiB, 17.17% gc time) +``` + +### GC Logging + +Enable detailed GC logging to understand collection patterns: + +```julia +julia> GC.enable_logging(true) +julia> # Run your code +julia> GC.enable_logging(false) +``` + +This logs each garbage collection event with timing and memory statistics. + +### Manual GC Control + +While generally not recommended, you can manually trigger garbage collection: + +```julia +GC.gc() # Force a garbage collection +GC.enable(false) # Disable automatic GC (use with caution!) +GC.enable(true) # Re-enable automatic GC +``` + +**Warning**: Disabling GC can lead to memory exhaustion. Only use this for specific performance measurements or debugging. + +## [Performance Considerations](@id man-gc-performance) + +### Reducing Allocations + +The best way to minimize GC impact is to reduce unnecessary allocations: + +* Use in-place operations when possible (e.g., `x .+= y` instead of `x = x + y`) +* Pre-allocate arrays and reuse them +* Avoid creating temporary objects in tight loops +* Consider using `StaticArrays.jl` for small, fixed-size arrays + +### Memory-Efficient Patterns + +* Avoid global variables that change type +* Use `const` for global constants + +### Profiling Memory Usage + +For detailed guidance on profiling memory allocations and identifying performance bottlenecks, see the [Profiling](@ref man-profiling) section. + +## [Advanced Configuration](@id man-gc-advanced) + +### Integration with System Memory Management + +Julia works best when: + +* The system has adequate swap space (recommended: 2x physical RAM) +* Virtual memory is properly configured +* Other processes leave sufficient memory available +* Container memory limits are set appropriately with `--heap-size-hint` + +## [Troubleshooting Memory Issues](@id man-gc-troubleshooting) + +### High GC Overhead + +If garbage collection is taking too much time: + +1. **Reduce allocation rate**: Focus on algorithmic improvements +2. **Adjust GC threads**: Experiment with different `--gcthreads` settings +3. **Use concurrent sweeping**: Enable background sweeping with `--gcthreads=N,1` +4. **Profile memory patterns**: Identify allocation hotspots and optimize them + +### Memory Leaks + +While Julia's GC prevents most memory leaks, issues can still occur: + +* **Global references**: Avoid holding references to large objects in global variables +* **Closures**: Be careful with closures that capture large amounts of data +* **C interop**: Ensure proper cleanup when interfacing with C libraries + +For more detailed information about Julia's garbage collector internals, see the Garbage Collection section in the Developer Documentation. diff --git a/doc/src/manual/metaprogramming.md b/doc/src/manual/metaprogramming.md index 9ed579594571a..38d688f427078 100644 --- a/doc/src/manual/metaprogramming.md +++ b/doc/src/manual/metaprogramming.md @@ -363,6 +363,14 @@ QuoteNode `QuoteNode` can also be used for certain advanced metaprogramming tasks. +Note that while it does not support `$`, it also does not prevent it, nor does +it preserve the identity of the wrapped object: + +```jldoctest +julia> b = 2; eval(Expr(:quote, QuoteNode(Expr(:$, :b)))) +:($(QuoteNode(2))) +``` + ### Evaluating expressions Given an expression object, one can cause Julia to evaluate (execute) it at global scope using @@ -711,7 +719,7 @@ user to optionally specify their own error message, instead of just printing the Just like in functions with a variable number of arguments ([Varargs Functions](@ref)), this is specified with an ellipses following the last argument: -```jldoctest assert2 +```julia-repl assert2 julia> macro assert(ex, msgs...) msg_body = isempty(msgs) ? ex : msgs[1] msg = string(msg_body) diff --git a/doc/src/manual/methods.md b/doc/src/manual/methods.md index e7f6514a64506..cebd9dbb6061b 100644 --- a/doc/src/manual/methods.md +++ b/doc/src/manual/methods.md @@ -288,13 +288,17 @@ Such specializations are *not* listed by `methods`, as this doesn't create new ` For example, if you create a method -``` +```jldoctest mysum_example; output = false mysum(x::Real, y::Real) = x + y + +# output + +mysum (generic function with 1 method) ``` you've given the function `mysum` one new method (possibly its only method), and that method takes any pair of `Real` number inputs. But if you then execute -```julia-repl +```jldoctest mysum_example julia> mysum(1, 2) 3 @@ -578,73 +582,8 @@ However, future calls to `tryeval` will continue to see the definition of `newfu You may want to try this for yourself to see how it works. -The implementation of this behavior is a "world age counter". -This monotonically increasing value tracks each method definition operation. -This allows describing "the set of method definitions visible to a given runtime environment" -as a single number, or "world age". -It also allows comparing the methods available in two worlds just by comparing their ordinal value. -In the example above, we see that the "current world" (in which the method `newfun` exists), -is one greater than the task-local "runtime world" that was fixed when the execution of `tryeval` started. - -Sometimes it is necessary to get around this (for example, if you are implementing the above REPL). -Fortunately, there is an easy solution: call the function using [`Base.invokelatest`](@ref) or -the macro version [`Base.@invokelatest`](@ref): - -```jldoctest -julia> function tryeval2() - @eval newfun2() = 2 - @invokelatest newfun2() - end -tryeval2 (generic function with 1 method) - -julia> tryeval2() -2 -``` - -Finally, let's take a look at some more complex examples where this rule comes into play. -Define a function `f(x)`, which initially has one method: - -```jldoctest redefinemethod -julia> f(x) = "original definition" -f (generic function with 1 method) -``` - -Start some other operations that use `f(x)`: - -```jldoctest redefinemethod -julia> g(x) = f(x) -g (generic function with 1 method) - -julia> t = @async f(wait()); yield(); -``` - -Now we add some new methods to `f(x)`: - -```jldoctest redefinemethod -julia> f(x::Int) = "definition for Int" -f (generic function with 2 methods) - -julia> f(x::Type{Int}) = "definition for Type{Int}" -f (generic function with 3 methods) -``` - -Compare how these results differ: - -```jldoctest redefinemethod -julia> f(1) -"definition for Int" - -julia> g(1) -"definition for Int" - -julia> fetch(schedule(t, 1)) -"original definition" - -julia> t = @async f(wait()); yield(); - -julia> fetch(schedule(t, 1)) -"definition for Int" -``` +The implementation of this behavior is a "world age counter", which is further described in the [Worldage](@ref man-worldage) +manual chapter. ## Design Patterns with Parametric Methods diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index a6c66bbd14f77..72b9d586945f9 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -114,6 +114,12 @@ and above. To maintain compatibility with Julia 1.10 and below, use the `@compat VERSION >= v"1.11.0-DEV.469" && eval(Meta.parse("public a, b, c")) ``` +`export` is a keyword wherever it occurs whereas the `public` keyword is currently limited to the +syntactic top level within a file or module. This limitation exists for compatibility reasons, +as `public` was introduced as a new keyword in Julia 1.11 while `export` has existed since Julia +1.0. However, this restriction on `public` may be lifted in future releases, so do not use `public` +as an identifier. + ### Standalone `using` and `import` For interactive use, the most common way of loading a module is `using ModuleName`. This [loads](@ref @@ -316,6 +322,68 @@ Here, Julia cannot decide which `f` you are referring to, so you have to make a 3. When the names in question *do* share a meaning, it is common for one module to import it from another, or have a lightweight “base” package with the sole function of defining an interface like this, which can be used by other packages. It is conventional to have such package names end in `...Base` (which has nothing to do with Julia's `Base` module). +### Precedence order of definitions + +There are in general four kinds of binding definitions: + 1. Those provided via implicit import through `using M` + 2. Those provided via explicit import (e.g. `using M: x`, `import M: x`) + 3. Those declared implicitly as global (via `global x` without type specification) + 4. Those declared explicitly using definition syntax (`const`, `global x::T`, `struct`, etc.) + +Syntactically, we divide these into three precedence levels (from weakest to strongest) + 1. Implicit imports + 2. Implicit declarations + 3. Explicit declarations and imports + +In general, we permit replacement of weaker bindings by stronger ones: + +```julia-repl +julia> module M1; const x = 1; export x; end +Main.M1 + +julia> using .M1 + +julia> x # Implicit import from M1 +1 + +julia> begin; f() = (global x; x = 1) end + +julia> x # Implicit declaration +ERROR: UndefVarError: `x` not defined in `Main` +Suggestion: add an appropriate import or assignment. This global was declared but not assigned. + +julia> const x = 2 # Explicit declaration +2 +``` + +However, within the explicit precedence level, replacement is syntactically disallowed: +```julia-repl +julia> module M1; const x = 1; export x; end +Main.M1 + +julia> import .M1: x + +julia> const x = 2 +ERROR: cannot declare Main.x constant; it was already declared as an import +Stacktrace: + [1] top-level scope + @ REPL[3]:1 +``` + +or ignored: + +```julia-repl +julia> const y = 2 +2 + +julia> import .M1: x as y +WARNING: import of M1.x into Main conflicts with an existing identifier; ignored. +``` + +The resolution of an implicit binding depends on the set of all `using`'d modules visible +in the current world age. See [the manual chapter on world age](@ref man-worldage) for more +details. + ### Default top-level definitions and bare modules Modules automatically contain `using Core`, `using Base`, and definitions of the [`eval`](@ref) diff --git a/doc/src/manual/multi-threading.md b/doc/src/manual/multi-threading.md index ec470f867cc47..ad4b3e6ef1312 100644 --- a/doc/src/manual/multi-threading.md +++ b/doc/src/manual/multi-threading.md @@ -8,7 +8,7 @@ of Julia multi-threading features. By default, Julia starts up with 2 threads of execution; 1 worker thread and 1 interactive thread. This can be verified by using the command [`Threads.nthreads()`](@ref): -```jldoctest +```julia julia> Threads.nthreads(:default) 1 julia> Threads.nthreads(:interactive) @@ -37,6 +37,7 @@ each threadpool. !!! compat "Julia 1.12" Starting by default with 1 interactive thread, as well as the 1 worker thread, was made as such in Julia 1.12 + If the number of threads is set to 1 by either doing `-t1` or `JULIA_NUM_THREADS=1` an interactive thread will not be spawned. Lets start Julia with 4 threads: @@ -46,7 +47,7 @@ $ julia --threads 4 Let's verify there are 4 threads at our disposal. -```julia-repl +```jldoctest; filter = r"[0-9]+" julia> Threads.nthreads() 4 ``` @@ -83,13 +84,15 @@ julia> Threads.threadid() ### Multiple GC Threads -The Garbage Collector (GC) can use multiple threads. The amount used is either half the number -of compute worker threads or configured by either the `--gcthreads` command line argument or by using the +The Garbage Collector (GC) can use multiple threads. The amount used by default matches the compute +worker threads or can configured by either the `--gcthreads` command line argument or by using the [`JULIA_NUM_GC_THREADS`](@ref JULIA_NUM_GC_THREADS) environment variable. !!! compat "Julia 1.10" The `--gcthreads` command line argument requires at least Julia 1.10. +For more details about garbage collection configuration and performance tuning, see [Memory Management and Garbage Collection](@ref man-memory-management). + ## [Threadpools](@id man-threadpools) When a program's threads are busy with many tasks to run, tasks may experience @@ -144,6 +147,8 @@ julia> nthreads(:interactive) julia> nthreads() 3 ``` +!!! note + Explicitly asking for 1 thread by doing `-t1` or `JULIA_NUM_THREADS=1` does not add an interactive thread. !!! note The zero-argument version of `nthreads` returns the number of threads @@ -209,7 +214,7 @@ Note that [`Threads.@threads`](@ref) does not have an optional reduction paramet ### Using `@threads` without data-races -The concept of a data-race is elaborated on in ["Communication and data races between threads"](@ref man-communication-and-data-races). For now, just known that a data race can result in incorrect results and dangerous errors. +The concept of a data-race is elaborated on in ["Communication and data races between threads"](@ref man-communication-and-data-races). For now, just know that a data race can result in incorrect results and dangerous errors. Lets say we want to make the function `sum_single` below multithreaded. ```julia-repl diff --git a/doc/src/manual/performance-tips.md b/doc/src/manual/performance-tips.md index d506ac9946ba6..90a40337e081a 100644 --- a/doc/src/manual/performance-tips.md +++ b/doc/src/manual/performance-tips.md @@ -58,14 +58,16 @@ Passing arguments to functions is better style. It leads to more reusable code a In the following REPL session: -```julia-repl +```jldoctest julia> x = 1.0 +1.0 ``` is equivalent to: -```julia-repl +```jldoctest julia> global x = 1.0 +1.0 ``` so all the performance issues discussed previously apply. @@ -114,6 +116,8 @@ Consequently, in addition to the allocation itself, it's very likely that the code generated for your function is far from optimal. Take such indications seriously and follow the advice below. +For more information about memory management and garbage collection in Julia, see [Memory Management and Garbage Collection](@ref man-memory-management). + In this particular case, the memory allocation is due to the usage of a type-unstable global variable `x`, so if we instead pass `x` as an argument to the function it no longer allocates memory (the remaining allocation reported below is due to running the `@time` macro in global scope) and is significantly faster after the first call: @@ -915,6 +919,40 @@ In the mean time, some user-contributed packages like [FastClosures](https://github.com/c42f/FastClosures.jl) automate the insertion of `let` statements as in `abmult3`. +#### Use `@__FUNCTION__` for recursive closures + +For recursive closures specifically, the [`@__FUNCTION__`](@ref) macro can avoid both type instability and boxing. + +First, let's see the unoptimized version: + +```julia +function make_fib_unoptimized() + fib(n) = n <= 1 ? 1 : fib(n - 1) + fib(n - 2) # fib is boxed + return fib +end +``` + +The `fib` function is boxed, meaning the return type is inferred as `Any`: + +```julia +@code_warntype make_fib_unoptimized() +``` + +Now, to eliminate this type instability, we can instead use `@__FUNCTION__` to refer to the concrete function object: + +```julia +function make_fib_optimized() + fib(n) = n <= 1 ? 1 : (@__FUNCTION__)(n - 1) + (@__FUNCTION__)(n - 2) + return fib +end +``` + +This gives us a concrete return type: + +```julia +@code_warntype make_fib_optimized() +``` + ### [Types with values-as-parameters](@id man-performance-value-type) @@ -1697,7 +1735,7 @@ in generated code by using Julia's [`code_native`](@ref) function. Note that `@fastmath` also assumes that `NaN`s will not occur during the computation, which can lead to surprising behavior: -```julia-repl +```jldoctest julia> f(x) = isnan(x); julia> f(NaN) diff --git a/doc/src/manual/profile.md b/doc/src/manual/profile.md index 49b58ba9671c2..77f9fb44e2a29 100644 --- a/doc/src/manual/profile.md +++ b/doc/src/manual/profile.md @@ -562,8 +562,7 @@ Passing `sample_rate=1.0` will make it record everything (which is slow); Since Julia 1.11, all allocations should have a type reported. -For more details on how to use this tool, please see the following talk from JuliaCon 2022: -https://www.youtube.com/watch?v=BFvpwC8hEWQ +For more details on how to use this tool, please see [the talk from JuliaCon 2022](https://www.youtube.com/watch?v=BFvpwC8hEWQ). ##### Allocation Profiler Example diff --git a/doc/src/manual/strings.md b/doc/src/manual/strings.md index 57431d07c0aa5..2948c8aad0335 100644 --- a/doc/src/manual/strings.md +++ b/doc/src/manual/strings.md @@ -482,7 +482,7 @@ The resulting string may contain different characters than the input strings, and its number of characters may be lower than sum of numbers of characters of the concatenated strings, e.g.: -```julia-repl +```jldoctest julia> a, b = "\xe2\x88", "\x80" ("\xe2\x88", "\x80") @@ -992,7 +992,7 @@ The `r"..."` literal is constructed without interpolation and unescaping (except quotation mark `"` which still has to be escaped). Here is an example showing the difference from standard string literals: -```julia-repl +```jldoctest julia> x = 10 10 @@ -1006,7 +1006,10 @@ julia> r"\x" r"\x" julia> "\x" -ERROR: syntax: invalid escape sequence +ERROR: ParseError: +# Error @ none:1:2 +"\x" +#└┘ ── invalid hex escape sequence ``` Triple-quoted regex strings, of the form `r"""..."""`, are also supported (and may be convenient diff --git a/doc/src/manual/types.md b/doc/src/manual/types.md index f13e2e6865d0f..17314f8d1d288 100644 --- a/doc/src/manual/types.md +++ b/doc/src/manual/types.md @@ -31,8 +31,11 @@ each other: all concrete types are final and may only have abstract types as the While this might at first seem unduly restrictive, it has many beneficial consequences with surprisingly few drawbacks. It turns out that being able to inherit behavior is much more important than being able to inherit structure, and inheriting both causes significant difficulties in traditional -object-oriented languages. Other high-level aspects of Julia's type system that should be mentioned -up front are: +object-oriented languages. While concrete types do have abstract subtypes, there are only two examples of this +([`Union{}`](@ref man-abstract-types) and [`Type{T}`](@ref man-typet-type))) and additional subtypes +of concrete types cannot be declared. + +Other high-level aspects of Julia's type system that should be mentioned up front are: * There is no division between object and non-object values: all values in Julia are true objects having a type that belongs to a single, fully connected type graph, all nodes of which are equally @@ -182,7 +185,7 @@ When no supertype is given, the default supertype is `Any` -- a predefined abstr all objects are instances of and all types are subtypes of. In type theory, `Any` is commonly called "top" because it is at the apex of the type graph. Julia also has a predefined abstract "bottom" type, at the nadir of the type graph, which is written as `Union{}`. It is the exact -opposite of `Any`: no object is an instance of `Union{}` and all types are supertypes of `Union{}`. +opposite of `Any`: no object is an instance of `Union{}` and all types (including concrete types) are supertypes of `Union{}`. Let's consider some of the abstract types that make up Julia's numerical hierarchy: @@ -298,7 +301,7 @@ a name. A primitive type can optionally be declared to be a subtype of some supe is omitted, then the type defaults to having `Any` as its immediate supertype. The declaration of [`Bool`](@ref) above therefore means that a boolean value takes eight bits to store, and has [`Integer`](@ref) as its immediate supertype. Currently, only sizes that are multiples of -8 bits are supported and you are likely to experience LLVM bugs with sizes other than those used above. +8 bits are supported. Therefore, boolean values, although they really need just a single bit, cannot be declared to be any smaller than eight bits. @@ -1306,6 +1309,9 @@ julia> WrapType(Float64) # sharpened constructor, note more precise Type{Float64 WrapType{Type{Float64}}(Float64) ``` +This behavior of `Type{Float64}` is an example of an abstract type subtyping a +concrete type (here `DataType`). + ## Type Aliases Sometimes it is convenient to introduce a new name for an already expressible type. diff --git a/doc/src/manual/variables-and-scoping.md b/doc/src/manual/variables-and-scoping.md index ab2d969dd9b0e..9756b4ec5bd87 100644 --- a/doc/src/manual/variables-and-scoping.md +++ b/doc/src/manual/variables-and-scoping.md @@ -28,7 +28,7 @@ a global variable by the same name is allowed or not. !!! tip "A Common Confusion" If you run into an unexpectedly undefined variable, - ```julia + ```julia ; nodoctest = "Pseudocode" # Print the numbers 1 through 5 i = 0 while i < 5 @@ -40,7 +40,7 @@ a global variable by the same name is allowed or not. a simple fix is to change all global variable definitions into local definitions by wrapping the code in a `let` block or `function`. - ```julia + ```julia ; nodoctest = "Pseudocode" # Print the numbers 1 through 5 let i = 0 while i < 5 @@ -136,12 +136,14 @@ inside of another local scope, the scope it creates is nested inside of all the local scopes that it appears within, which are all ultimately nested inside of the global scope of the module in which the code is evaluated. Variables in outer scopes are visible from any scope they contain — meaning that they can be -read and written in inner scopes — unless there is a local variable with the -same name that "shadows" the outer variable of the same name. This is true even -if the outer local is declared after (in the sense of textually below) an inner +read and written in inner scopes — unless there is a variable with the same name +that "shadows" the outer variable of the same name. This is true even if the +outer local is declared after (in the sense of textually below) an inner block. When we say that a variable "exists" in a given scope, this means that a variable by that name exists in any of the scopes that the current scope is -nested inside of, including the current one. +nested inside of, including the current one. If a variable's value is used in a +local scope, but nothing with its name exists in this scope, it is assumed to be +a global. Some programming languages require explicitly declaring new variables before using them. Explicit declaration works in Julia too: in any local scope, writing @@ -368,7 +370,7 @@ scope rule applies and `x` is created as local to the `for` loop and therefore g undefined after the loop executes. Next, let's consider the body of `sum_to_def` extracted into global scope, fixing its argument to `n = 10` -```julia +```julia ; nodoctest = "Specifically shows scope differences" s = 0 for i = 1:10 t = s + i @@ -470,7 +472,7 @@ years were confused about this behavior and complained that it was complicated a explain and understand. Fair point. Second, and arguably worse, is that it's bad for programming "at scale." When you see a small piece of code in one place like this, it's quite clear what's going on: -```julia +```julia ; nodoctest="Expects global file scope" s = 0 for i = 1:10 s += i @@ -481,7 +483,7 @@ Obviously the intention is to modify the existing global variable `s`. What else However, not all real world code is so short or so clear. We found that code like the following often occurs in the wild: -```julia +```julia ; nodoctest="Expects global file scope" x = 123 # much later @@ -731,21 +733,24 @@ object (such as an array), and that object may still be modified. Additionally w to assign a value to a variable that is declared constant the following scenarios are possible: * Attempting to replace a constant without the const `keyword` is disallowed: -```jldoctest -julia> const x = 1.0 -1.0 -julia> x = 1 -ERROR: invalid assignment to constant x. This redefinition may be permitted using the `const` keyword. -``` -* All other defefinitions of constants are permitted, but may cause significant re-compilation: -```jldoctest -julia> const y = 1.0 -1.0 + ```jldoctest + julia> const x = 1.0 + 1.0 -julia> const y = 2.0 -2.0 -``` + julia> x = 1 + ERROR: invalid assignment to constant x. This redefinition may be permitted using the `const` keyword. + ``` + +* All other definitions of constants are permitted, but may cause significant re-compilation: + + ```jldoctest + julia> const y = 1.0 + 1.0 + + julia> const y = 2.0 + 2.0 + ``` !!! compat "Julia 1.12" Prior to julia 1.12, redefinition of constants was poorly supported. It was restricted to diff --git a/doc/src/manual/variables.md b/doc/src/manual/variables.md index 4c3e98ca57281..074a7207698d1 100644 --- a/doc/src/manual/variables.md +++ b/doc/src/manual/variables.md @@ -3,21 +3,17 @@ A variable, in Julia, is a name associated (or bound) to a value. It's useful when you want to store a value (that you obtained after some math, for example) for later use. For example: -```julia-repl -# Assign the value 10 to the variable x -julia> x = 10 +```jldoctest +julia> x = 10 # Assign the value 10 to the variable x 10 -# Doing math with x's value -julia> x + 1 +julia> x + 1 # Doing math with x's value 11 -# Reassign x's value -julia> x = 1 + 1 +julia> x = 1 + 1 # Reassign x's value 2 -# You can assign values of other types, like strings of text -julia> x = "Hello World!" +julia> x = "Hello World!" # You can assign values of other types, like strings of text "Hello World!" ``` @@ -125,7 +121,7 @@ it from `+ ᵃx` where `ᵃx` is the variable name. A particular class of variable names is one that contains only underscores. These identifiers are write-only. I.e. they can only be assigned values, which are immediately discarded, and their values cannot be used in any way. -```julia-repl +```jldoctest julia> x, ___ = size([2 2; 1 1]) (2, 2) @@ -138,12 +134,18 @@ ERROR: syntax: all-underscore identifiers are write-only and their values cannot The only explicitly disallowed names for variables are the names of the built-in [Keywords](@ref Keywords): -```julia-repl +```jldoctest julia> else = false -ERROR: syntax: unexpected "else" +ERROR: ParseError: +# Error @ none:1:1 +else = false +└──┘ ── invalid identifier julia> try = "No" -ERROR: syntax: unexpected "=" +ERROR: ParseError: +# Error @ none:1:1 +try = "No" +└────────┘ ── try without catch or finally ``` Some Unicode characters are considered to be equivalent in identifiers. diff --git a/doc/src/manual/worldage.md b/doc/src/manual/worldage.md new file mode 100644 index 0000000000000..41cceec3b87c6 --- /dev/null +++ b/doc/src/manual/worldage.md @@ -0,0 +1,295 @@ +# [The World Age mechanism](@id man-world-age) + +!!! note + World age is an advanced concept. For the vast majority of Julia users, the world age + mechanism operates invisibly in the background. This documentation is intended for the + few users who may encounter world-age related issues or error messages. + +!!! compat "Julia 1.12" + Prior to Julia 1.12, the world age mechanism did not apply to changes to the global binding table. + The documentation in this chapter is specific to Julia 1.12+. + +!!! warning + This manual chapter uses internal functions to introspect world age and runtime data structures + as an explanatory aid. In general, unless otherwise noted the world age mechanism is not a stable + interface and should be interacted with in packages through stable APIs (e.g. `invokelatest`) only. + In particular, do not assume that world ages are always integers or that they have a linear order. + +## World age in general + +The "world age counter" is a monotonically increasing counter that is incremented for every +change to the global method table or the global binding table (e.g. through method definition, +type definition, `import`/`using` declaration, creation of (typed) globals or definition of constants). + +The current value of the global world age counter can be retrieved using the (internal) function [`Base.get_world_counter`](@ref). + +```julia-repl +julia> Base.get_world_counter() +0x0000000000009632 + +julia> const x = 1 + +julia> Base.get_world_counter() +0x0000000000009633 +``` + +In addition, each [`Task`](@ref) stores a local world age that determines which modifications to +the global binding and method tables are currently visible to the running task. The world age of +the running task will never exceed the global world age counter, but may run arbitrarily behind it. +In general the term "current world age" refers to the local world age of the currently running task. +The current world age may be retrieved using the (internal) function [`Base.tls_world_age`](@ref) + +```julia-repl +julia> function f end +f (generic function with 0 methods) + +julia> begin + @show (Int(Base.get_world_counter()), Int(Base.tls_world_age())) + Core.eval(@__MODULE__, :(f() = 1)) + @show (Int(Base.get_world_counter()), Int(Base.tls_world_age())) + f() + end +(Int(Base.get_world_counter()), Int(Base.tls_world_age())) = (38452, 38452) +(Int(Base.get_world_counter()), Int(Base.tls_world_age())) = (38453, 38452) +ERROR: MethodError: no method matching f() +The applicable method may be too new: running in current world age 38452, while global world is 38453. + +Closest candidates are: + f() (method too new to be called from this world context.) + @ Main REPL[2]:3 + +Stacktrace: + [1] top-level scope + @ REPL[2]:5 + +julia> (f(), Int(Base.tls_world_age())) +(1, 38453) +``` + +Here the definition of the method `f` raised the global world counter, but the current world +age did not change. As a result, the definition of `f` was not visible in the currently +executing task and a [`MethodError`](@ref) resulted. + +!!! note + The method error printing provided additional information that `f()` is available in a newer world age. + This information is added by the error display, not the task that threw the `MethodError`. + The thrown `MethodError` is identical whether or not a matching definition of `f()` exists + in a newer world age. + +However, note that the definition of `f()` was subsequently available at the next REPL prompt, because +the current task's world age had been raised. In general, certain syntactic constructs (in particular most definitions) +will raise the current task's world age to the latest global world age, thus making all changes +(both from the current task and any concurrently executing other tasks) visible. The following statements +raise the current world age: + +1. An explicit invocation of `Core.@latestworld` +2. The start of every top-level statement +3. The start of every REPL prompt +4. Any type or struct definition +5. Any method definition +6. Any constant declaration +7. Any global variable declaration (but not a global variable assignment) +8. Any `using`, `import`, `export` or `public` statement +9. Certain other macros like [`@eval`](@ref) (depends on the macro implementation) + +Note, however, that the current task's world age may only ever be permanently incremented at +top level. As a general rule, using any of the above statements in non-top-level scope is a syntax error: + +```julia-repl +julia> f() = Core.@latestworld +ERROR: syntax: World age increment not at top level +Stacktrace: + [1] top-level scope + @ REPL[5]:1 +``` + +When it isn't (for example for `@eval`), the world age side effect is ignored. + +As a result of these rules, Julia may assume that the world age does not change +within the execution of an ordinary function. + +```julia +function my_function() + before = Base.tls_world_age() + # Any arbitrary code + after = Base.tls_world_age() + @assert before === after # always true +end +``` + +This is the key invariant that allows Julia to optimize based on the current state +of its global data structures, while still having the well-defined ability to change +these data structures. + +## Temporarily raising the world age using `invokelatest` + +As described above, it is not possible to permanently raise the world age for the remainder of +a `Task`'s execution unless the task is executing top-level statements. However, it is possible to +temporarily change the world age in a scoped manner using `invokelatest`: + +```jldoctest +julia> function f end +f (generic function with 0 methods) + +julia> begin + Core.eval(@__MODULE__, :(f() = 1)) + invokelatest(f) + end +1 +``` + +`invokelatest` will temporarily raise the current task's world age to the latest global world age (at +entry to `invokelatest`) and execute the provided function. Note that the world age will return +to its prior value upon exit from `invokelatest`. + +## World age and const struct redefinitions + +The semantics described above for method redefinition also apply to redefinition of constants: + +```jldoctest +julia> const x = 1 +1 + +julia> get_const() = x +get_const (generic function with 1 method) + +julia> begin + @show get_const() + Core.eval(@__MODULE__, :(const x = 2)) + @show get_const() + Core.@latestworld + @show get_const() + end +get_const() = 1 +get_const() = 1 +get_const() = 2 +2 +``` + +However, for the avoidance of doubt, they do not apply to ordinary assignment to global variables, which becomes visible immediately: +```jldoctest +julia> global y = 1 +1 + +julia> get_global() = y +get_global (generic function with 1 method) + +julia> begin + @show get_global() + Core.eval(@__MODULE__, :(y = 2)) + @show get_global() + end +get_global() = 1 +get_global() = 2 +2 +``` + +One particular special case of constant reassignment is the redefinition of struct types: + +```jldoctest; filter = r"\@world\(MyStruct, \d+\:\d+\)" +julia> struct MyStruct + x::Int + end + +julia> const one_field = MyStruct(1) +MyStruct(1) + +julia> struct MyStruct + x::Int + y::Float64 + end + +julia> const two_field = MyStruct(1, 2.0) +MyStruct(1, 2.0) + +julia> one_field +@world(MyStruct, 38452:38455)(1) + +julia> two_field +MyStruct(1, 2.0) +``` + +Internally the two definitions of `MyStruct` are entirely separate types. However, +after the new `MyStruct` type is defined, there is no longer any default binding +for the original definition of `MyStruct`. To nevertheless facilitate access to +these types, the special [`@world`](@ref) macro may be used to access the meaning +of a name in a previous world. However, this facility is intended for introspection +only and in particular note that world age numbers are not stable across precompilation +and should in general be treated opaquely. + +### Binding partition introspection + +In certain cases, it can be helpful to introspect the system's understanding of what +a binding means in any particular world age. The default display printing of `Core.Binding` +provides a helpful summary (e.g. on the `MyStruct` example from above): + +```julia-repl +julia> convert(Core.Binding, GlobalRef(@__MODULE__, :MyStruct)) +Binding Main.MyStruct + 38456:∞ - constant binding to MyStruct + 38452:38455 - constant binding to @world(MyStruct, 38452:38455) + 38451:38451 - backdated constant binding to @world(MyStruct, 38452:38455) + 0:38450 - backdated constant binding to @world(MyStruct, 38452:38455) +``` + +## World age and `using`/`import` + +Bindings provided via `using` and `import` also operate via the world age mechanism. +Binding resolution is a stateless function of the `import` and `using` definitions +visible in the current world age. For example: + +```julia-repl +julia> module M1; const x = 1; export x; end + +julia> module M2; const x = 2; export x; end + +julia> using .M1 + +julia> x +1 + +julia> using .M2 + +julia> x +ERROR: UndefVarError: `x` not defined in `Main` +Hint: It looks like two or more modules export different bindings with this name, resulting in ambiguity. Try explicitly importing it from a particular module, or qualifying the name with the module it should come from. + +julia> convert(Core.Binding, GlobalRef(@__MODULE__, :x)) +Binding Main.x + 38458:∞ - ambiguous binding - guard entry + 38457:38457 - implicit `using` resolved to constant 1 +``` + +## World age capture + +Certain language features capture the current task's world age. Perhaps the most common of +these is creation of new tasks. Newly created tasks will inherit the creating task's local +world age at creation time and will retain said world age (unless explicitly raised) even +if the originating tasks raises its world age: + +```julia-repl +julia> const x = 1 + +julia> t = @task (wait(); println("Running now"); x); + +julia> const x = 2 + +julia> schedule(t); +Running now + +julia> x +2 + +julia> fetch(t) +1 +``` + +In addition to tasks, opaque closures also capture their world age at creation. See [`Base.Experimental.@opaque`](@ref). + +```@docs +Base.@world +Base.get_world_counter +Base.tls_world_age +Base.invoke_in_world +Base.Experimental.@opaque +``` diff --git a/julia.spdx.json b/julia.spdx.json index 0d7ab1df94688..8664b2c653386 100644 --- a/julia.spdx.json +++ b/julia.spdx.json @@ -240,7 +240,7 @@ "licenseConcluded": "Apache-2.0", "licenseDeclared": "Apache-2.0", "copyrightText": "Copyright (c) 1998-2024 The OpenSSL Project Authors. Copyright (c) 1995-1998 Eric A. Young, Tim J. Hudson.", - "summary": "OpenSSL is a robust, commercial-grade, full-featured Open Source Toolkit for the TLS (formerly SSL), DTLS and QUIC (currently client side only) protocols.", + "summary": "OpenSSL is a robust, commercial-grade, full-featured Open Source Toolkit for the TLS (formerly SSL), DTLS and QUIC (currently client side only) protocols." }, { "name": "mpfr", @@ -432,6 +432,18 @@ "copyrightText": "Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler", "summary": "A massively spiffy yet delicately unobtrusive compression library." }, + { + "name": "zstd", + "SPDXID": "SPDXRef-zstd", + "downloadLocation": "git+https://github.com/facebook/zstd.git", + "filesAnalyzed": false, + "homepage": "https://www.zstd.net", + "sourceInfo": "The git hash of the version in use can be found in the file deps/zstd.version", + "licenseConcluded": "BSD-3-Clause", + "licenseDeclared": "GPL-2.0+ OR BSD-3-Clause", + "copyrightText": "Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.", + "summary": "Zstandard, or zstd as short version, is a fast lossless compression algorithm." + }, { "name": "patchelf", "SPDXID": "SPDXRef-patchelf", @@ -639,6 +651,11 @@ "relationshipType": "BUILD_DEPENDENCY_OF", "relatedSpdxElement": "SPDXRef-JuliaMain" }, + { + "spdxElementId": "SPDXRef-zstd", + "relationshipType": "BUILD_DEPENDENCY_OF", + "relatedSpdxElement": "SPDXRef-JuliaMain" + }, { "spdxElementId": "SPDXRef-patchelf", "relationshipType": "BUILD_TOOL_OF", diff --git a/pkgimage.mk b/pkgimage.mk index 78b2618be549f..ed5e1095c0229 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -4,9 +4,10 @@ JULIAHOME := $(SRCDIR) include $(JULIAHOME)/Make.inc include $(JULIAHOME)/stdlib/stdlib.mk +DEPOTDIR := $(build_prefix)/share/julia # set some influential environment variables -export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(build_prefix)/share/julia)) +export JULIA_DEPOT_PATH := $(shell echo $(call cygpath_w,$(DEPOTDIR))) export JULIA_LOAD_PATH := @stdlib$(PATHSEP)$(shell echo $(call cygpath_w,$(JULIAHOME)/stdlib)) unexport JULIA_PROJECT := unexport JULIA_BINDIR := @@ -18,14 +19,14 @@ release: $(BUILDDIR)/stdlib/release.image debug: $(BUILDDIR)/stdlib/debug.image all: release debug -$(JULIA_DEPOT_PATH)/compiled: +$(DEPOTDIR)/compiled: mkdir -p $@ print-depot-path: @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH') -$(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(JULIA_DEPOT_PATH)/compiled - @$(call PRINT_JULIA, JULIA_CPU_TARGET="$(JULIA_CPU_TARGET)" $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e \ +$(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(DEPOTDIR)/compiled + @$(call PRINT_JULIA, JULIA_CPU_TARGET="sysimage" $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e \ 'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])') touch $@ @@ -33,5 +34,5 @@ $(BUILDDIR)/stdlib/release.image: $(build_private_libdir)/sys.$(SHLIB_EXT) $(BUILDDIR)/stdlib/debug.image: $(build_private_libdir)/sys-debug.$(SHLIB_EXT) clean: - rm -rf $(JULIA_DEPOT_PATH)/compiled + rm -rf $(DEPOTDIR)/compiled rm -f $(BUILDDIR)/stdlib/*.image diff --git a/src/Makefile b/src/Makefile index d6f206e03cadd..f31802d1b78d9 100644 --- a/src/Makefile +++ b/src/Makefile @@ -29,7 +29,7 @@ endif JCFLAGS += -Wold-style-definition -Wstrict-prototypes -Wc++-compat ifeq ($(USECLANG),1) -FLAGS += -Wno-return-type-c-linkage -Wno-atomic-alignment +FLAGS += -Wno-return-type-c-linkage -Wno-atomic-alignment -Wno-nullability-extension -Wno-nullability-completeness # required to be allowed to use nullability extension and not be required to annotate all of everything endif ifeq (${USE_THIRD_PARTY_GC},mmtk) @@ -69,7 +69,7 @@ CG_LLVMLINK := ifeq ($(JULIACODEGEN),LLVM) # Currently these files are used by both GCs. But we should make the list specific to stock, and MMTk should have its own implementation. -GC_CODEGEN_SRCS := llvm-final-gc-lowering llvm-late-gc-lowering llvm-gc-invariant-verifier gc-pinning-log +GC_CODEGEN_SRCS := llvm-final-gc-lowering llvm-late-gc-lowering llvm-gc-invariant-verifier ifeq (${USE_THIRD_PARTY_GC},mmtk) FLAGS += -I$(MMTK_API_INC) GC_CODEGEN_SRCS += llvm-late-gc-lowering-mmtk @@ -77,9 +77,10 @@ else GC_CODEGEN_SRCS += llvm-late-gc-lowering-stock endif CODEGEN_SRCS := codegen jitlayers aotcompile debuginfo disasm llvm-simdloop \ - llvm-pass-helpers llvm-ptls llvm-propagate-addrspaces null_sysimage \ + llvm-pass-helpers llvm-ptls llvm-propagate-addrspaces \ llvm-multiversioning llvm-alloc-opt llvm-alloc-helpers cgmemmgr llvm-remove-addrspaces \ - llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures pipeline llvm_api \ + llvm-remove-ni llvm-julia-licm llvm-demote-float16 llvm-cpufeatures llvm-expand-atomic-modify \ + pipeline llvm_api \ $(GC_CODEGEN_SRCS) FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --includedir) CG_LLVM_LIBS := all @@ -169,6 +170,7 @@ CG_LLVMLINK += $(LLVM_LDFLAGS) -lLLVM else CG_LLVMLINK += $(LLVM_LDFLAGS) $(LLVM_SHARED_LINK_FLAG) endif # OS +CG_LLVMLINK += -lz -lzstd endif # USE_LLVM_SHLIB endif # USE_SYSTEM_LLVM @@ -177,7 +179,9 @@ FLAGS += -DLLVM_SHLIB endif # USE_LLVM_SHLIB == 1 endif # JULIACODEGEN == LLVM -RT_LLVM_LINK_ARGS := $(shell $(LLVM_CONFIG_HOST) --libs $(RT_LLVM_LIBS) --system-libs --link-static) +# Use subst to work around llvm-configure bug in Yggdrasil build: https://github.com/llvm/llvm-project/pull/139945 +CG_LLVMLINK := $(subst /workspace/destdir/lib/libzstd.dll.a,-lzstd,$(CG_LLVMLINK)) +RT_LLVM_LINK_ARGS := $(subst /workspace/destdir/lib/libzstd.dll.a,-lzstd,$(shell $(LLVM_CONFIG_HOST) --libs $(RT_LLVM_LIBS) --system-libs --link-static)) RT_LLVMLINK += $(LLVM_LDFLAGS) $(RT_LLVM_LINK_ARGS) ifeq ($(OS), WINNT) RT_LLVMLINK += -luuid -lole32 @@ -194,6 +198,7 @@ endif COMMON_LIBPATHS := -L$(build_libdir) -L$(build_shlibdir) RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) +# NB: CG needs uv_mutex_* symbols, but we expect to export them from libjulia-internal CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) ifeq (${USE_THIRD_PARTY_GC},mmtk) @@ -212,6 +217,13 @@ DOBJS := $(SRCS:%=$(BUILDDIR)/%.dbg.obj) CODEGEN_OBJS := $(CODEGEN_SRCS:%=$(BUILDDIR)/%.o) CODEGEN_DOBJS := $(CODEGEN_SRCS:%=$(BUILDDIR)/%.dbg.obj) +ifeq ($(OS)_$(BINARY),WINNT_32) +OBJS += $(BUILDDIR)/llvm-Compression.o +DOBJS += $(BUILDDIR)/llvm-Compression.dbg.obj +CODEGEN_OBJS += $(BUILDDIR)/llvm-Compression.o +CODEGEN_DOBJS += $(BUILDDIR)/llvm-Compression.dbg.obj +endif + # Add SONAME defines so we can embed proper `dlopen()` calls. ADDL_SHIPFLAGS := -DJL_SYSTEM_IMAGE_PATH=$(call shell_escape,$(call c_escape,$(call normalize_path,$(build_private_libdir_rel)/sys.$(SHLIB_EXT)))) \ -DJL_LIBJULIA_SONAME=$(call shell_escape,$(call c_escape,$(LIBJULIA_PATH_REL).$(JL_MAJOR_SHLIB_EXT))) @@ -225,6 +237,12 @@ DEBUGFLAGS_GCC += $(FLAGS) $(ADDL_DEBUGFLAGS) SHIPFLAGS_CLANG += $(FLAGS) $(ADDL_SHIPFLAGS) DEBUGFLAGS_CLANG += $(FLAGS) $(ADDL_DEBUGFLAGS) +DEBUGFLAGS_CLANG += -Wno-nullability-extension -Wno-nullability-completeness # required to be allowed to use nullability extension and not be required to annotate all of everything +ifeq ($(USEGCC),1) # TODO: we currently set flags incorrectly for clang analyze, but mostly it works out okay +DEBUGFLAGS_CLANG += -Wno-unknown-warning-option +endif +DEBUGFLAGS_CLANG += -Wno-return-type-c-linkage # TODO: do we care about fixing this instead? (it is not a bug, just a nuisance) + ifeq ($(USE_CROSS_FLISP), 1) FLISPDIR := $(BUILDDIR)/flisp/host FLISP_EXECUTABLE_debug := $(FLISPDIR)/flisp-debug$(BUILD_EXE) @@ -338,7 +356,7 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm flisp/aliase $(call cygpath_w,$(SRCDIR)/mk_julia_flisp_boot.scm) $(call cygpath_w,$(dir $<)) $(notdir $<) $(call cygpath_w,$@)) # additional dependency links -$(BUILDDIR)/codegen-stubs.o $(BUILDDIR)/codegen-stubs.dbg.obj: $(SRCDIR)/intrinsics.h +$(BUILDDIR)/codegen-stubs.o $(BUILDDIR)/codegen-stubs.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.h llvm-julia-passes.inc) $(BUILDDIR)/aotcompile.o $(BUILDDIR)/aotcompile.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/processor.h $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/iddict.c $(SRCDIR)/idset.c $(SRCDIR)/builtin_proto.h @@ -357,7 +375,7 @@ $(BUILDDIR)/gc-alloc-profiler.o $(BUILDDIR)/gc-alloc-profiler.dbg.obj: $(SRCDIR) $(BUILDDIR)/gc-page-profiler.o $(BUILDDIR)/gc-page-profiler.dbg.obj: $(SRCDIR)/gc-page-profiler.h $(BUILDDIR)/init.o $(BUILDDIR)/init.dbg.obj: $(SRCDIR)/builtin_proto.h $(BUILDDIR)/interpreter.o $(BUILDDIR)/interpreter.dbg.obj: $(SRCDIR)/builtin_proto.h -$(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/llvm-codegen-shared.h +$(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/llvm-julia-task-dispatcher.h $(BUILDDIR)/jltypes.o $(BUILDDIR)/jltypes.dbg.obj: $(SRCDIR)/builtin_proto.h $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/llvm-codegen-shared.h $(BUILDDIR)/julia_version.h $(BUILDDIR)/llvm-alloc-helpers.o $(BUILDDIR)/llvm-alloc-helpers.dbg.obj: $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/llvm-pass-helpers.h $(SRCDIR)/llvm-alloc-helpers.h @@ -378,7 +396,8 @@ $(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: $(addprefix $ $(BUILDDIR)/staticdata.o $(BUILDDIR)/staticdata.dbg.obj: $(SRCDIR)/staticdata_utils.c $(SRCDIR)/precompile_utils.c $(SRCDIR)/processor.h $(SRCDIR)/builtin_proto.h $(BUILDDIR)/toplevel.o $(BUILDDIR)/toplevel.dbg.obj: $(SRCDIR)/builtin_proto.h $(BUILDDIR)/ircode.o $(BUILDDIR)/ircode.dbg.obj: $(SRCDIR)/serialize.h $(SRCDIR)/common_symbols1.inc $(SRCDIR)/common_symbols2.inc -$(BUILDDIR)/pipeline.o $(BUILDDIR)/pipeline.dbg.obj: $(SRCDIR)/passes.h $(SRCDIR)/jitlayers.h +$(BUILDDIR)/pipeline.o $(BUILDDIR)/pipeline.dbg.obj: $(addprefix $(SRCDIR)/,passes.h jitlayers.h llvm-julia-passes.inc) +$(BUILDDIR)/llvm_api.o $(BUILDDIR)/llvm_api.dbg.obj: $(SRCDIR)/llvm-julia-passes.inc $(addprefix $(BUILDDIR)/,threading.o threading.dbg.obj gc-common.o gc-stock.o gc.dbg.obj init.c init.dbg.obj task.o task.dbg.obj): $(addprefix $(SRCDIR)/,threading.h) $(addprefix $(BUILDDIR)/,APInt-C.o APInt-C.dbg.obj runtime_intrinsics.o runtime_intrinsics.dbg.obj): $(SRCDIR)/APInt-C.h @@ -476,10 +495,10 @@ libjulia-codegen-debug: $(build_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_MINO libjulia-codegen-debug libjulia-codegen-release: $(PUBLIC_HEADER_TARGETS) # set the exports for the source files based on where they are getting linked -$(OBJS): SHIPFLAGS += -DJL_LIBRARY_EXPORTS_INTERNAL -$(DOBJS): DEBUGFLAGS += -DJL_LIBRARY_EXPORTS_INTERNAL -$(CODEGEN_OBJS): SHIPFLAGS += -DJL_LIBRARY_EXPORTS_CODEGEN -$(CODEGEN_DOBJS): DEBUGFLAGS += -DJL_LIBRARY_EXPORTS_CODEGEN +$(OBJS): SHIPFLAGS += -DJL_LIBRARY_EXPORTS_INTERNAL -DBUILDING_UV_SHARED +$(DOBJS): DEBUGFLAGS += -DJL_LIBRARY_EXPORTS_INTERNAL -DBUILDING_UV_SHARED +$(CODEGEN_OBJS): SHIPFLAGS += -DJL_LIBRARY_EXPORTS_CODEGEN -DUSING_UV_SHARED +$(CODEGEN_DOBJS): DEBUGFLAGS += -DJL_LIBRARY_EXPORTS_CODEGEN -DUSING_UV_SHARED clean: -rm -fr $(build_shlibdir)/libjulia-internal* $(build_shlibdir)/libjulia-codegen* $(build_shlibdir)/libccalltest* $(build_shlibdir)/libllvmcalltest* diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index a4d911d935b1f..b6c0c604031ef 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -109,20 +109,37 @@ jl_get_llvm_mis_impl(void *native_code, size_t *num_elements, jl_method_instance } } +// get the list of global variables managed by the compiler extern "C" JL_DLLEXPORT_CODEGEN void jl_get_llvm_gvs_impl(void *native_code, size_t *num_elements, void **data) { - // map a memory location (jl_value_t or jl_binding_t) to a GlobalVariable jl_native_code_desc_t *desc = (jl_native_code_desc_t *)native_code; - auto &value_map = desc->jl_value_to_llvm; + auto &gvars = desc->jl_sysimg_gvars; if (data == NULL) { - *num_elements = value_map.size(); + *num_elements = gvars.size(); return; } - assert(*num_elements == value_map.size()); - memcpy(data, value_map.data(), *num_elements * sizeof(void *)); + assert(*num_elements == gvars.size()); + memcpy(data, gvars.data(), *num_elements * sizeof(void *)); +} + +// get the initializer values (jl_value_t or jl_binding_t ptr) of managed global variables +extern "C" JL_DLLEXPORT_CODEGEN void jl_get_llvm_gv_inits_impl(void *native_code, + size_t *num_elements, + void **data) +{ + jl_native_code_desc_t *desc = (jl_native_code_desc_t *)native_code; + auto &inits = desc->jl_value_to_llvm; + + if (data == NULL) { + *num_elements = inits.size(); + return; + } + + assert(*num_elements == inits.size()); + memcpy(data, inits.data(), *num_elements * sizeof(void *)); } extern "C" JL_DLLEXPORT_CODEGEN void jl_get_llvm_external_fns_impl(void *native_code, @@ -331,7 +348,11 @@ class egal_set { }; } using ::egal_set; -typedef DenseMap> jl_compiled_functions_t; +struct jl_compiled_function_t { + orc::ThreadSafeModule TSM; + jl_llvm_functions_t decls; +}; +typedef DenseMap jl_compiled_functions_t; static void record_method_roots(egal_set &method_roots, jl_method_instance_t *mi) { @@ -377,7 +398,7 @@ static void aot_optimize_roots(jl_codegen_params_t ¶ms, egal_set &method_roo std::string OldName(GV->getName()); StringRef NewName(mref->second->getName()); for (auto &def : compiled_functions) { - orc::ThreadSafeModule &TSM = std::get<0>(def.second); + orc::ThreadSafeModule &TSM = def.second.TSM; Module &M = *TSM.getModuleUnlocked(); if (GlobalValue *GV2 = M.getNamedValue(OldName)) { if (GV2 == GV) @@ -403,7 +424,7 @@ static void aot_optimize_roots(jl_codegen_params_t ¶ms, egal_set &method_roo static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_roots, jl_compiled_functions_t &compiled_functions) { - decltype(params.workqueue) workqueue; + jl_workqueue_t workqueue; std::swap(params.workqueue, workqueue); jl_code_instance_t *codeinst = NULL; JL_GC_PUSH1(&codeinst); @@ -419,7 +440,7 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root { auto it = compiled_functions.find(codeinst); if (it != compiled_functions.end()) { - auto &decls = it->second.second; + auto &decls = it->second.decls; invokeName = decls.functionObject; if (decls.functionObject == "jl_fptr_args") { preal_decl = decls.specFunctionObject; @@ -443,8 +464,11 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root } if (preal_decl.empty()) { pinvoke = emit_tojlinvoke(codeinst, invokeName, mod, params); - if (!proto.specsig) + if (!proto.specsig) { proto.decl->replaceAllUsesWith(pinvoke); + proto.decl->eraseFromParent(); + proto.decl = pinvoke; + } } if (proto.specsig && !preal_specsig) { // get or build an fptr1 that can invoke codeinst @@ -463,9 +487,12 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root } if (!preal_decl.empty()) { // merge and/or rename this prototype to the real function - if (Value *specfun = mod->getNamedValue(preal_decl)) { - if (proto.decl != specfun) + if (Function *specfun = cast_or_null(mod->getNamedValue(preal_decl))) { + if (proto.decl != specfun) { proto.decl->replaceAllUsesWith(specfun); + proto.decl->eraseFromParent(); + proto.decl = specfun; + } } else { proto.decl->setName(preal_decl); @@ -483,9 +510,12 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root assert(ocinvokeDecl != "jl_fptr_const_return"); assert(ocinvokeDecl != "jl_fptr_sparam"); // merge and/or rename this prototype to the real function - if (Value *specfun = mod->getNamedValue(ocinvokeDecl)) { - if (proto.oc != specfun) + if (Function *specfun = cast_or_null(mod->getNamedValue(ocinvokeDecl))) { + if (proto.oc != specfun) { proto.oc->replaceAllUsesWith(specfun); + proto.oc->eraseFromParent(); + proto.oc = specfun; + } } else { proto.oc->setName(ocinvokeDecl); @@ -497,6 +527,7 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root JL_GC_POP(); } + /// Link the function in the source module into the destination module if /// needed, setting up mapping information. /// Similar to orc::cloneFunctionDecl, but more complete for greater correctness @@ -519,16 +550,16 @@ Function *IRLinker_copyFunctionProto(Module *DstM, Function *SF) { return F; } -static Function *aot_abi_converter(jl_codegen_params_t ¶ms, Module *M, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Module *defM, StringRef func, StringRef specfunc, bool target_specsig) +static Function *aot_abi_converter(jl_codegen_params_t ¶ms, Module *M, jl_abi_t from_abi, jl_code_instance_t *codeinst, Module *defM, StringRef func, StringRef specfunc, bool target_specsig) { std::string gf_thunk_name; if (!specfunc.empty()) { Value *llvmtarget = IRLinker_copyFunctionProto(M, defM->getFunction(specfunc)); - gf_thunk_name = emit_abi_converter(M, params, declrt, sigt, nargs, specsig, codeinst, llvmtarget, target_specsig); + gf_thunk_name = emit_abi_converter(M, params, from_abi, codeinst, llvmtarget, target_specsig); } else { Value *llvmtarget = func.empty() ? nullptr : IRLinker_copyFunctionProto(M, defM->getFunction(func)); - gf_thunk_name = emit_abi_dispatcher(M, params, declrt, sigt, nargs, specsig, codeinst, llvmtarget); + gf_thunk_name = emit_abi_dispatcher(M, params, from_abi, codeinst, llvmtarget); } auto F = M->getFunction(gf_thunk_name); assert(F); @@ -546,28 +577,29 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct } size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); for (cfunc_decl_t &cfunc : params.cfuncs) { - Module *M = cfunc.theFptr->getParent(); - jl_value_t *sigt = jl_pinned_ref_get(cfunc.sigt); + Module *M = cfunc.cfuncdata->getParent(); + jl_value_t *sigt = jl_pinned_ref_get(cfunc.abi.sigt); JL_GC_PROMISE_ROOTED(sigt); - jl_value_t *declrt = jl_pinned_ref_get(cfunc.declrt); + jl_value_t *declrt = jl_pinned_ref_get(cfunc.abi.rt); JL_GC_PROMISE_ROOTED(declrt); - Function *unspec = aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, nullptr, nullptr, "", "", false); + Function *unspec = aot_abi_converter(params, M, cfunc.abi, nullptr, nullptr, "", "", false); jl_code_instance_t *codeinst = nullptr; auto assign_fptr = [¶ms, &cfunc, &codeinst, &unspec](Function *f) { ConstantArray *init = cast(cfunc.cfuncdata->getInitializer()); - SmallVector initvals; + SmallVector initvals; for (unsigned i = 0; i < init->getNumOperands(); ++i) initvals.push_back(init->getOperand(i)); - assert(initvals.size() == 6); + assert(initvals.size() == 8); assert(initvals[0]->isNullValue()); + assert(initvals[2]->isNullValue()); if (codeinst) { Constant *llvmcodeinst = literal_pointer_val_slot(params, f->getParent(), (jl_value_t*)codeinst); - initvals[0] = llvmcodeinst; // plast_codeinst + initvals[2] = llvmcodeinst; // plast_codeinst } - assert(initvals[2]->isNullValue()); - initvals[2] = unspec; + assert(initvals[4]->isNullValue()); + initvals[4] = unspec; + initvals[0] = f; cfunc.cfuncdata->setInitializer(ConstantArray::get(init->getType(), initvals)); - cfunc.theFptr->setInitializer(f); }; Module *defM = nullptr; StringRef func; @@ -578,8 +610,8 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct codeinst = it->second; JL_GC_PROMISE_ROOTED(codeinst); auto defs = compiled_functions.find(codeinst); - defM = std::get<0>(defs->second).getModuleUnlocked(); - const jl_llvm_functions_t &decls = std::get<1>(defs->second); + defM = defs->second.TSM.getModuleUnlocked(); + const jl_llvm_functions_t &decls = defs->second.decls; func = decls.functionObject; StringRef specfunc = decls.specFunctionObject; jl_value_t *astrt = codeinst->rettype; @@ -591,7 +623,7 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); } if (func == "jl_fptr_const_return") { - std::string gf_thunk_name = emit_abi_constreturn(M, params, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst->rettype_const); + std::string gf_thunk_name = emit_abi_constreturn(M, params, cfunc.abi, codeinst->rettype_const); auto F = M->getFunction(gf_thunk_name); assert(F); assign_fptr(F); @@ -599,11 +631,11 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct } else if (func == "jl_fptr_args") { assert(!specfunc.empty()); - if (!cfunc.specsig && jl_subtype(astrt, declrt)) { + if (!cfunc.abi.specsig && jl_subtype(astrt, declrt)) { assign_fptr(IRLinker_copyFunctionProto(M, defM->getFunction(specfunc))); continue; } - assign_fptr(aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst, defM, func, specfunc, false)); + assign_fptr(aot_abi_converter(params, M, cfunc.abi, codeinst, defM, func, specfunc, false)); continue; } else if (func == "jl_fptr_sparam" || func == "jl_f_opaque_closure_call") { @@ -615,16 +647,35 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct assign_fptr(IRLinker_copyFunctionProto(M, defM->getFunction(specfunc))); continue; } - assign_fptr(aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst, defM, func, specfunc, true)); + assign_fptr(aot_abi_converter(params, M, cfunc.abi, codeinst, defM, func, specfunc, true)); continue; } } } - Function *f = codeinst ? aot_abi_converter(params, M, declrt, sigt, cfunc.nargs, cfunc.specsig, codeinst, defM, func, "", false) : unspec; - return assign_fptr(f); + Function *f = codeinst ? aot_abi_converter(params, M, cfunc.abi, codeinst, defM, func, "", false) : unspec; + assign_fptr(f); } } +// destructively move the contents of src into dest +// this assumes that the targets of the two modules are the same +// including the DataLayout and ModuleFlags (for example) +// and that there is no module-level assembly +// Comdat is also removed, since this needs to be re-added later +static void jl_merge_module(Linker &L, orc::ThreadSafeModule srcTSM) JL_NOTSAFEPOINT +{ + srcTSM.consumingModuleDo([&L](std::unique_ptr src) JL_NOTSAFEPOINT { + bool error = L.linkInModule(std::move(src)); + assert(!error && "linking llvmcall modules failed"); + (void)error; + }); +} + +static bool canPartition(const Function &F) +{ + return !F.hasFnAttribute(Attribute::AlwaysInline) && + !F.hasFnAttribute(Attribute::InlineHint); +} // takes the running content that has collected in the shadow module and dump it to disk // this builds the object file portion of the sysimage files for fast startup @@ -662,7 +713,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm fargs[2] = (jl_value_t*)worlds; jl_array_data(worlds, size_t)[0] = jl_typeinf_world; jl_array_data(worlds, size_t)[compiler_world] = world; // might overwrite previous - fargs[3] = jl_box_long(trim); + fargs[3] = jl_box_uint8(trim); size_t last_age = ct->world_age; ct->world_age = jl_typeinf_world; codeinfos = (jl_array_t*)jl_apply(fargs, 4); @@ -730,7 +781,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm orc::ThreadSafeModule backing; if (!llvmmod) { ctx = jl_ExecutionEngine->makeContext(); - backing = jl_create_ts_module("text", ctx); + backing = jl_create_ts_module("text", ctx, jl_ExecutionEngine->getDataLayout(), jl_ExecutionEngine->getTargetTriple()); } orc::ThreadSafeModule &clone = llvmmod ? *unwrap(llvmmod) : backing; auto ctxt = clone.getContext(); @@ -747,6 +798,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm assert(params.imaging_mode); // `_imaging_mode` controls if broken features like code-coverage are disabled params.external_linkage = external_linkage; params.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); + bool safepoint_on_entry = params.safepoint_on_entry; JL_GC_PUSH3(¶ms.temporary_roots, &method_roots.list, &method_roots.keyset); jl_compiled_functions_t compiled_functions; size_t i, l; @@ -761,17 +813,8 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm assert(jl_is_code_info(src)); if (compiled_functions.count(codeinst)) continue; // skip any duplicates that accidentally made there way in here (or make this an error?) - if (external_linkage) { - uint8_t specsigflags; - jl_callptr_t invoke; - void *fptr; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); - if (invoke != NULL && (specsigflags & 0b100)) { - // this codeinst is already available externally - // TODO: for performance, avoid generating the src code when we know it would reach here anyways - continue; - } - } + if (jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) + params.safepoint_on_entry = false; // ensure we don't block ExpandAtomicModifyPass from inlining this code if applicable orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, clone.getModuleUnlocked()->getDataLayout(), Triple(clone.getModuleUnlocked()->getTargetTriple())); @@ -780,6 +823,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm decls.functionObject = "jl_fptr_const_return"; else decls = jl_emit_codeinst(result_m, codeinst, src, params); + params.safepoint_on_entry = safepoint_on_entry; record_method_roots(method_roots, jl_get_ci_mi(codeinst)); if (result_m) compiled_functions[codeinst] = {std::move(result_m), std::move(decls)}; @@ -799,6 +843,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm generate_cfunc_thunks(params, compiled_functions); aot_optimize_roots(params, method_roots, compiled_functions); params.temporary_roots = nullptr; + params.temporary_roots_set.clear(); JL_GC_POP(); // process the globals array, before jl_merge_module destroys them @@ -810,7 +855,6 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm size_t idx = 0; for (auto &global : params.global_targets) { gvars[idx] = global.second->getName().str(); - global.second->setInitializer(literal_static_pointer_val(global.first, global.second->getValueType())); assert(gvars_set.insert(global.second).second && "Duplicate gvar in params!"); assert(gvars_names.insert(gvars[idx]).second && "Duplicate gvar name in params!"); data->jl_value_to_llvm[idx] = jl_pinned_ref_assume(void, global.first); @@ -841,11 +885,27 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm { Linker L(*clone.getModuleUnlocked()); for (auto &def : compiled_functions) { - jl_merge_module(clone, std::move(std::get<0>(def.second))); jl_code_instance_t *this_code = def.first; - jl_llvm_functions_t decls = std::get<1>(def.second); + JL_GC_PROMISE_ROOTED(this_code); + jl_llvm_functions_t &decls = def.second.decls; StringRef func = decls.functionObject; StringRef cfunc = decls.specFunctionObject; + orc::ThreadSafeModule &M = def.second.TSM; + if (external_linkage) { + uint8_t specsigflags; + jl_callptr_t invoke; + void *fptr; + jl_read_codeinst_invoke(this_code, &specsigflags, &invoke, &fptr, 0); + if (invoke != NULL && (specsigflags & 0b100)) { + // this codeinst is already available externally: keep it only if canPartition demands it for local use + // TODO: for performance, avoid generating the src code when we know it would reach here anyways? + if (M.withModuleDo([&](Module &M) { return !canPartition(*cast(M.getNamedValue(cfunc))); })) { + jl_merge_module(L, std::move(M)); + } + continue; + } + } + jl_merge_module(L, std::move(M)); uint32_t func_id = 0; uint32_t cfunc_id = 0; if (func == "jl_fptr_args") { @@ -872,6 +932,52 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm } data->jl_fvar_map[this_code] = std::make_tuple(func_id, cfunc_id); } + bool Changed = true; + while (Changed) { + Changed = false; + // make sure everything referenced got included though, since some functions aren't + // correctly implemented by staticdata for external use, and so codegen won't emit + // an external reference but expects a private copy here instead + for (auto &def : compiled_functions) { + orc::ThreadSafeModule &M = def.second.TSM; + if (!M) + continue; + jl_llvm_functions_t &decls = def.second.decls; + StringRef func = decls.functionObject; + StringRef cfunc = decls.specFunctionObject; + if (func != "jl_fptr_args" && + func != "jl_fptr_sparam" && + func != "jl_f_opaque_closure_call" && + clone.getModuleUnlocked()->getNamedValue(func)) { + jl_merge_module(L, std::move(M)); + Changed = true; + continue; + } + if (!cfunc.empty() && clone.getModuleUnlocked()->getNamedValue(cfunc)) { + Changed = true; + jl_merge_module(L, std::move(M)); + } + } + } +#ifndef NDEBUG + // make sure we didn't forget anything that we promised to include in here + for (auto &def : compiled_functions) { + jl_llvm_functions_t &decls = def.second.decls; + StringRef func = decls.functionObject; + StringRef cfunc = decls.specFunctionObject; + if (func != "jl_fptr_args" && + func != "jl_fptr_sparam" && + func != "jl_f_opaque_closure_call") { + GlobalValue *F = clone.getModuleUnlocked()->getNamedValue(func); + assert(!F || !F->isDeclaration()); + } + if (!cfunc.empty()) { + GlobalValue *F = clone.getModuleUnlocked()->getNamedValue(cfunc); + assert(!F || !F->isDeclaration()); + } + } +#endif + compiled_functions.clear(); if (params._shared_module) { bool error = L.linkInModule(std::move(params._shared_module)); assert(!error && "Error linking in shared module"); @@ -881,15 +987,35 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm // now get references to the globals in the merged module // and set them to be internalized and initialized at startup + // filter out any gvars that got optimized away + idx = 0; + size_t newoffset = 0; + size_t newidx = 0; for (auto &global : gvars) { //Safe b/c context is locked by params - GlobalVariable *G = cast(clone.getModuleUnlocked()->getNamedValue(global)); - assert(G->hasInitializer()); - G->setLinkage(GlobalValue::InternalLinkage); - G->setDSOLocal(true); - data->jl_sysimg_gvars.push_back(G); + GlobalVariable *G = cast_or_null(clone.getModuleUnlocked()->getNamedValue(global)); + if (G != nullptr) { + assert(!G->hasInitializer()); + G->setInitializer(Constant::getNullValue(G->getValueType())); + G->setLinkage(GlobalValue::InternalLinkage); + G->setDSOLocal(true); + assert(newidx == data->jl_sysimg_gvars.size()); + if (idx < offset) { + data->jl_value_to_llvm[newidx] = data->jl_value_to_llvm[idx]; + newoffset = newidx + 1; + } + else { + data->jl_external_to_llvm[newidx - newoffset] = data->jl_external_to_llvm[idx - offset]; + } + data->jl_sysimg_gvars.push_back(G); + newidx++; + } + idx++; } - CreateNativeGlobals += gvars.size(); + data->jl_value_to_llvm.resize(newoffset); + data->jl_external_to_llvm.resize(newidx - newoffset); + gvars.clear(); + CreateNativeGlobals += idx; data->M = std::move(clone); return (void*)data; @@ -1113,11 +1239,6 @@ struct Partition { size_t weight; }; -static bool canPartition(const Function &F) -{ - return !F.hasFnAttribute(Attribute::AlwaysInline); -} - static inline bool verify_partitioning(const SmallVectorImpl &partitions, const Module &M, DenseMap &fvars, DenseMap &gvars) { bool bad = false; #ifndef JL_NDEBUG @@ -1570,7 +1691,8 @@ static void materializePreserved(Module &M, Partition &partition) { // This just avoids a hashtable lookup. GV->setLinkage(GlobalValue::InternalLinkage); assert(GV->hasDefaultVisibility()); - } else { + } + else { Preserve.insert(GV); } } @@ -2081,11 +2203,6 @@ void jl_dump_native_impl(void *native_code, addComdat(&GA, TheTriple); } - // Wipe the global initializers, we'll reset them at load time - for (auto gv : data->jl_sysimg_gvars) { - cast(gv)->setInitializer(Constant::getNullValue(gv->getValueType())); - } - // add metadata information if (imaging_mode) { multiversioning_preannotate(dataM); @@ -2228,7 +2345,15 @@ void jl_dump_native_impl(void *native_code, "jl_small_typeof"); jl_small_typeof_copy->setVisibility(GlobalValue::HiddenVisibility); jl_small_typeof_copy->setDSOLocal(true); - AT = ArrayType::get(T_psize, 5); + + // Create CPU target string constant + auto cpu_target_str = jl_options.cpu_target ? jl_options.cpu_target : "native"; + auto cpu_target_data = ConstantDataArray::getString(Context, cpu_target_str, true); + auto cpu_target_global = new GlobalVariable(metadataM, cpu_target_data->getType(), true, + GlobalVariable::InternalLinkage, + cpu_target_data, "jl_cpu_target_string"); + + AT = ArrayType::get(T_psize, 6); auto pointers = new GlobalVariable(metadataM, AT, false, GlobalVariable::ExternalLinkage, ConstantArray::get(AT, { @@ -2236,7 +2361,8 @@ void jl_dump_native_impl(void *native_code, ConstantExpr::getBitCast(shards, T_psize), ConstantExpr::getBitCast(ptls, T_psize), ConstantExpr::getBitCast(jl_small_typeof_copy, T_psize), - ConstantExpr::getBitCast(target_ids, T_psize) + ConstantExpr::getBitCast(target_ids, T_psize), + ConstantExpr::getBitCast(cpu_target_global, T_psize) }), "jl_image_pointers"); addComdat(pointers, TheTriple); @@ -2346,17 +2472,16 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ dump->TSM = nullptr; if (src && jl_is_code_info(src)) { auto ctx = jl_ExecutionEngine->makeContext(); - orc::ThreadSafeModule m = jl_create_ts_module(name_from_method_instance(mi), ctx); + const auto &DL = jl_ExecutionEngine->getDataLayout(); + const auto &TT = jl_ExecutionEngine->getTargetTriple(); + orc::ThreadSafeModule m = jl_create_ts_module(name_from_method_instance(mi), ctx, DL, TT); Function *F = nullptr; { uint64_t compiler_start_time = 0; uint8_t measure_compile_time_enabled = jl_atomic_load_relaxed(&jl_measure_compile_time_enabled); if (measure_compile_time_enabled) compiler_start_time = jl_hrtime(); - auto target_info = m.withModuleDo([&](Module &M) { - return std::make_pair(M.getDataLayout(), Triple(M.getTargetTriple())); - }); - jl_codegen_params_t output(ctx, std::move(target_info.first), std::move(target_info.second)); + jl_codegen_params_t output(ctx, DL, TT); output.params = ¶ms; output.imaging_mode = jl_options.image_codegen; output.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); @@ -2368,15 +2493,15 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ jl_compiled_functions_t compiled_functions; size_t latestworld = jl_atomic_load_acquire(&jl_world_counter); for (cfunc_decl_t &cfunc : output.cfuncs) { - jl_value_t *sigt = jl_pinned_ref_get(cfunc.sigt); + jl_value_t *sigt = jl_pinned_ref_get(cfunc.abi.sigt); JL_GC_PROMISE_ROOTED(sigt); jl_method_instance_t *mi = jl_get_specialization1((jl_tupletype_t*)sigt, latestworld, 0); if (mi == nullptr) continue; - jl_code_instance_t *codeinst = jl_type_infer(mi, latestworld, SOURCE_MODE_NOT_REQUIRED); + jl_code_instance_t *codeinst = jl_type_infer(mi, latestworld, SOURCE_MODE_NOT_REQUIRED, jl_options.trim); if (codeinst == nullptr || compiled_functions.count(codeinst)) continue; - orc::ThreadSafeModule decl_m = jl_create_ts_module("extern", ctx); + orc::ThreadSafeModule decl_m = jl_create_ts_module("extern", ctx, DL, TT); jl_llvm_functions_t decls; if (jl_atomic_load_relaxed(&codeinst->invoke) == jl_fptr_const_return_addr) decls.functionObject = "jl_fptr_const_return"; @@ -2385,6 +2510,8 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ compiled_functions[codeinst] = {std::move(decl_m), std::move(decls)}; } generate_cfunc_thunks(output, compiled_functions); + emit_always_inline(m, output); + output.workqueue.clear(); compiled_functions.clear(); output.temporary_roots = nullptr; JL_GC_POP(); // GC the global_targets array contents now since reflection doesn't need it @@ -2399,7 +2526,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t *dump, jl_method_instance_t *mi, jl_ } else { auto p = literal_static_pointer_val(global.first, global.second->getValueType()); - Type *elty = PointerType::get(output.getContext(), 0); + Type *elty = PointerType::get(p->getContext(), 0); // For pretty printing, when LLVM inlines the global initializer into its loads auto alias = GlobalAlias::create(elty, 0, GlobalValue::PrivateLinkage, global.second->getName() + ".jit", p, global.second->getParent()); global.second->setInitializer(ConstantExpr::getBitCast(alias, global.second->getValueType())); diff --git a/src/ast.c b/src/ast.c index aeb061f15bc33..a0458657651bd 100644 --- a/src/ast.c +++ b/src/ast.c @@ -30,7 +30,6 @@ JL_DLLEXPORT jl_sym_t *jl_module_sym; JL_DLLEXPORT jl_sym_t *jl_slot_sym; JL_DLLEXPORT jl_sym_t *jl_export_sym; JL_DLLEXPORT jl_sym_t *jl_public_sym; -JL_DLLEXPORT jl_sym_t *jl_import_sym; JL_DLLEXPORT jl_sym_t *jl_toplevel_sym; JL_DLLEXPORT jl_sym_t *jl_quote_sym; JL_DLLEXPORT jl_sym_t *jl_line_sym; @@ -51,7 +50,6 @@ JL_DLLEXPORT jl_sym_t *jl_pop_exception_sym; JL_DLLEXPORT jl_sym_t *jl_exc_sym; JL_DLLEXPORT jl_sym_t *jl_error_sym; JL_DLLEXPORT jl_sym_t *jl_new_sym; -JL_DLLEXPORT jl_sym_t *jl_using_sym; JL_DLLEXPORT jl_sym_t *jl_splatnew_sym; JL_DLLEXPORT jl_sym_t *jl_block_sym; JL_DLLEXPORT jl_sym_t *jl_new_opaque_closure_sym; @@ -367,8 +365,6 @@ void jl_init_common_symbols(void) jl_module_sym = jl_symbol("module"); jl_export_sym = jl_symbol("export"); jl_public_sym = jl_symbol("public"); - jl_import_sym = jl_symbol("import"); - jl_using_sym = jl_symbol("using"); jl_assign_sym = jl_symbol("="); jl_method_sym = jl_symbol("method"); jl_exc_sym = jl_symbol("the_exception"); @@ -1076,14 +1072,15 @@ int jl_has_meta(jl_array_t *body, jl_sym_t *sym) JL_NOTSAFEPOINT // Utility function to return whether `e` is any of the special AST types or // will always evaluate to itself exactly unchanged. This corresponds to -// `is_self_quoting` in Core.Compiler utilities. -int jl_is_ast_node(jl_value_t *e) JL_NOTSAFEPOINT +// `isa_ast_node` in Core.Compiler utilities. +int jl_isa_ast_node(jl_value_t *e) JL_NOTSAFEPOINT { return jl_is_newvarnode(e) || jl_is_code_info(e) || jl_is_linenode(e) || jl_is_gotonode(e) || jl_is_gotoifnot(e) + || jl_is_enternode(e) || jl_is_returnnode(e) || jl_is_ssavalue(e) || jl_is_slotnumber(e) @@ -1098,9 +1095,10 @@ int jl_is_ast_node(jl_value_t *e) JL_NOTSAFEPOINT || jl_is_expr(e); } -static int is_self_quoting_expr(jl_expr_t *e) JL_NOTSAFEPOINT +static int is_self_escaping_expr(jl_expr_t *e) JL_NOTSAFEPOINT { return (e->head == jl_inert_sym || + e->head == jl_leave_sym || e->head == jl_core_sym || e->head == jl_line_sym || e->head == jl_lineinfo_sym || @@ -1118,12 +1116,13 @@ int need_esc_node(jl_value_t *e) JL_NOTSAFEPOINT || jl_is_ssavalue(e) || jl_is_slotnumber(e) || jl_is_argument(e) + || jl_is_enternode(e) || jl_is_quotenode(e)) return 0; if (jl_is_expr(e)) - return !is_self_quoting_expr((jl_expr_t*)e); + return !is_self_escaping_expr((jl_expr_t*)e); // note: jl_is_globalref(e) is not included here, since we care a little about about having a line number for it - return jl_is_ast_node(e); + return jl_isa_ast_node(e); } static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule, jl_module_t **ctx, jl_value_t **lineinfo, size_t world, int throw_load_error) @@ -1300,98 +1299,87 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule return expr; } -// Lower an expression tree into Julia's intermediate-representation. -JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule) -{ - return jl_expand_with_loc(expr, inmodule, "none", 0); -} - -// Lowering, with starting program location specified -JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line) -{ - return jl_expand_in_world(expr, inmodule, file, line, ~(size_t)0); -} - -// Lowering, with starting program location and worldage specified -JL_DLLEXPORT jl_value_t *jl_expand_in_world(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world) -{ - JL_TIMING(LOWERING, LOWERING); - jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); - JL_GC_PUSH1(&expr); - expr = jl_copy_ast(expr); - expr = jl_expand_macros(expr, inmodule, NULL, 0, world, 1); - expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk", expr, inmodule, file, line); - JL_GC_POP(); - return expr; -} - -// Same as the above, but printing warnings when applicable -JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line) +// warn: Print any lowering warnings returned; otherwise ignore +JL_DLLEXPORT jl_value_t *jl_fl_lower(jl_value_t *expr, jl_module_t *inmodule, + const char *filename, int line, size_t world, bool_t warn) { JL_TIMING(LOWERING, LOWERING); - jl_timing_show_location(file, line, inmodule, JL_TIMING_DEFAULT_BLOCK); + jl_timing_show_location(filename, line, inmodule, JL_TIMING_DEFAULT_BLOCK); jl_array_t *kwargs = NULL; - JL_GC_PUSH2(&expr, &kwargs); + JL_GC_PUSH3(&expr, &kwargs, &inmodule); expr = jl_copy_ast(expr); - expr = jl_expand_macros(expr, inmodule, NULL, 0, ~(size_t)0, 1); + expr = jl_expand_macros(expr, inmodule, NULL, 0, world, 1); jl_ast_context_t *ctx = jl_ast_ctx_enter(inmodule); value_t arg = julia_to_scm(ctx, expr); fl_context_t *fl_ctx = &ctx->fl; - value_t e = fl_applyn(fl_ctx, 4, symbol_value(symbol(fl_ctx, "jl-expand-to-thunk-warn")), arg, - symbol(fl_ctx, file), fixnum(line), fl_ctx->F); - expr = scm_to_julia(fl_ctx, e, inmodule); + value_t e = fl_applyn(fl_ctx, 3, symbol_value(symbol(fl_ctx, "jl-lower-to-thunk")), arg, + symbol(fl_ctx, filename), fixnum(line)); + value_t lwr = car_(e); + value_t warnings = car_(cdr_(e)); + expr = scm_to_julia(fl_ctx, lwr, inmodule); jl_ast_ctx_leave(ctx); jl_sym_t *warn_sym = jl_symbol("warn"); - if (jl_is_expr(expr) && ((jl_expr_t*)expr)->head == warn_sym) { - size_t nargs = jl_expr_nargs(expr); - for (int i = 0; i < nargs - 1; i++) { - jl_value_t *warning = jl_exprarg(expr, i); - size_t nargs = 0; - if (jl_is_expr(warning) && ((jl_expr_t*)warning)->head == warn_sym) - nargs = jl_expr_nargs(warning); - int kwargs_len = (int)nargs - 6; - if (nargs < 6 || kwargs_len % 2 != 0) { - jl_error("julia-logmsg: bad argument list - expected " - ":warn level (symbol) group (symbol) id file line msg . kwargs"); - } - jl_value_t *level = jl_exprarg(warning, 0); - jl_value_t *group = jl_exprarg(warning, 1); - jl_value_t *id = jl_exprarg(warning, 2); - jl_value_t *file = jl_exprarg(warning, 3); - jl_value_t *line = jl_exprarg(warning, 4); - jl_value_t *msg = jl_exprarg(warning, 5); - kwargs = jl_alloc_vec_any(kwargs_len); - for (int i = 0; i < kwargs_len; ++i) { - jl_array_ptr_set(kwargs, i, jl_exprarg(warning, i + 6)); - } - JL_TYPECHK(logmsg, long, level); - jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg); + for (; warn && iscons(warnings); warnings = cdr_(warnings)) { + jl_value_t *warning = scm_to_julia(fl_ctx, car_(warnings), inmodule); + size_t nargs = 0; + if (jl_is_expr(warning) && ((jl_expr_t*)warning)->head == warn_sym) + nargs = jl_expr_nargs(warning); + int kwargs_len = (int)nargs - 6; + if (nargs < 6 || kwargs_len % 2 != 0) { + jl_error("julia-logmsg: bad argument list - expected " + ":warn level (symbol) group (symbol) id file line msg . kwargs"); } - expr = jl_exprarg(expr, nargs - 1); + JL_GC_PUSH1(&warning); + jl_value_t *level = jl_exprarg(warning, 0); + jl_value_t *group = jl_exprarg(warning, 1); + jl_value_t *id = jl_exprarg(warning, 2); + jl_value_t *file = jl_exprarg(warning, 3); + jl_value_t *line = jl_exprarg(warning, 4); + jl_value_t *msg = jl_exprarg(warning, 5); + kwargs = jl_alloc_vec_any(kwargs_len); + for (int i = 0; i < kwargs_len; ++i) { + jl_array_ptr_set(kwargs, i, jl_exprarg(warning, i + 6)); + } + JL_TYPECHK(logmsg, long, level); + jl_log(jl_unbox_long(level), NULL, group, id, file, line, (jl_value_t*)kwargs, msg); + JL_GC_POP(); } + jl_value_t *result = (jl_value_t *)jl_svec1(expr); JL_GC_POP(); - return expr; + return result; } -// expand in a context where the expression value is unused -JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line) +// Main C entry point to lowering. Calls jl_fl_lower during bootstrap, and +// Core._lower otherwise (this is also jl_fl_lower unless we have JuliaLowering) +JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, + const char *filename, int line, size_t world, bool_t warn) { - JL_TIMING(LOWERING, LOWERING); - JL_GC_PUSH1(&expr); - expr = jl_copy_ast(expr); - expr = jl_expand_macros(expr, inmodule, NULL, 0, ~(size_t)0, 1); - expr = jl_call_scm_on_ast_and_loc("jl-expand-to-thunk-stmt", expr, inmodule, file, line); + jl_value_t *core_lower = NULL; + if (jl_core_module) + core_lower = jl_get_global_value(jl_core_module, jl_symbol("_lower"), jl_current_task->world_age); + if (!core_lower || core_lower == jl_nothing) { + return jl_fl_lower(expr, inmodule, filename, line, world, warn); + } + jl_value_t **args; + JL_GC_PUSHARGS(args, 7); + args[0] = core_lower; + args[1] = expr; + args[2] = (jl_value_t*)inmodule; + args[3] = jl_cstr_to_string(filename); + args[4] = jl_box_ulong(line); + args[5] = jl_box_ulong(world); + args[6] = warn ? jl_true : jl_false; + jl_task_t *ct = jl_current_task; + size_t last_age = ct->world_age; + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + jl_value_t *result = jl_apply(args, 7); + ct->world_age = last_age; + args[0] = result; // root during error check below + JL_TYPECHK(parse, simplevector, result); + if (jl_svec_len(result) < 1) + jl_error("Result from lowering should be `svec(a::Any, x::Any...)`"); JL_GC_POP(); - return expr; -} - -JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule) -{ - return jl_expand_stmt_with_loc(expr, inmodule, "none", 0); + return result; } jl_code_info_t *jl_outer_ctor_body(jl_value_t *thistype, size_t nfields, size_t nsparams, jl_module_t *inmodule, const char *file, int line) diff --git a/src/ast.scm b/src/ast.scm index a20f8f87f955c..ea538e0aede4e 100644 --- a/src/ast.scm +++ b/src/ast.scm @@ -466,6 +466,7 @@ (define (make-assignment l r) `(= ,l ,r)) (define (assignment? e) (and (pair? e) (eq? (car e) '=))) (define (return? e) (and (pair? e) (eq? (car e) 'return))) +(define (thisfunction? e) (and (pair? e) (eq? (car e) 'thisfunction))) (define (tuple-call? e) (and (length> e 1) @@ -493,6 +494,7 @@ (define (vinfo:never-undef v) (< 0 (logand (caddr v) 4))) (define (vinfo:read v) (< 0 (logand (caddr v) 8))) (define (vinfo:sa v) (< 0 (logand (caddr v) 16))) +(define (vinfo:nospecialize v) (< 0 (logand (caddr v) 128))) (define (set-bit x b val) (if val (logior x b) (logand x (lognot b)))) ;; record whether var is captured (define (vinfo:set-capt! v c) (set-car! (cddr v) (set-bit (caddr v) 1 c))) @@ -507,6 +509,7 @@ ;; occurs undef: mask 32 ;; whether var is called (occurs in function call head position) (define (vinfo:set-called! v a) (set-car! (cddr v) (set-bit (caddr v) 64 a))) +(define (vinfo:set-nospecialize! v c) (set-car! (cddr v) (set-bit (caddr v) 128 c))) (define var-info-for assq) diff --git a/src/builtin_proto.h b/src/builtin_proto.h index c82ec77414129..607106f35bac0 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -8,82 +8,90 @@ extern "C" { #endif // declarations for julia-callable builtin functions +#define JL_BUILTIN_FUNCTIONS(XX) \ + XX(_abstracttype,"_abstracttype") \ + XX(_apply_iterate,"_apply_iterate") \ + XX(_call_in_world_total,"_call_in_world_total") \ + XX(_compute_sparams,"_compute_sparams") \ + XX(_defaultctors,"_defaultctors") \ + XX(_equiv_typedef,"_equiv_typedef") \ + XX(_expr,"_expr") \ + XX(_import, "_import") \ + XX(_primitivetype,"_primitivetype") \ + XX(_setsuper,"_setsuper!") \ + XX(_structtype,"_structtype") \ + XX(_svec_ref,"_svec_ref") \ + XX(_typebody,"_typebody!") \ + XX(_typevar,"_typevar") \ + XX(_using, "_using") \ + XX(applicable,"applicable") \ + XX(apply_type,"apply_type") \ + XX(compilerbarrier,"compilerbarrier") \ + XX(current_scope,"current_scope") \ + XX(donotdelete,"donotdelete") \ + XX(fieldtype,"fieldtype") \ + XX(finalizer,"finalizer") \ + XX(get_binding_type,"get_binding_type") \ + XX(getfield,"getfield") \ + XX(getglobal,"getglobal") \ + XX(ifelse,"ifelse") \ + XX(intrinsic_call,"intrinsic_call") \ + XX(invoke,"invoke") \ + XX(invoke_in_world,"invoke_in_world") \ + XX(invokelatest,"invokelatest") \ + XX(is,"===") \ + XX(isa,"isa") \ + XX(isdefined,"isdefined") \ + XX(isdefinedglobal,"isdefinedglobal") \ + XX(issubtype,"<:") \ + XX(memorynew,"memorynew") \ + XX(memoryrefnew,"memoryrefnew") \ + XX(memoryref_isassigned,"memoryref_isassigned") \ + XX(memoryrefget,"memoryrefget") \ + XX(memoryrefmodify,"memoryrefmodify!") \ + XX(memoryrefoffset,"memoryrefoffset") \ + XX(memoryrefreplace,"memoryrefreplace!") \ + XX(memoryrefset,"memoryrefset!") \ + XX(memoryrefsetonce,"memoryrefsetonce!") \ + XX(memoryrefswap,"memoryrefswap!") \ + XX(modifyfield,"modifyfield!") \ + XX(modifyglobal,"modifyglobal!") \ + XX(nfields,"nfields") \ + XX(opaque_closure_call,"opaque_closure_call") \ + XX(replacefield,"replacefield!") \ + XX(replaceglobal,"replaceglobal!") \ + XX(setfield,"setfield!") \ + XX(setfieldonce,"setfieldonce!") \ + XX(setglobal,"setglobal!") \ + XX(setglobalonce,"setglobalonce!") \ + XX(sizeof,"sizeof") \ + XX(svec,"svec") \ + XX(swapfield,"swapfield!") \ + XX(swapglobal,"swapglobal!") \ + XX(throw,"throw") \ + XX(throw_methoderror,"throw_methoderror") \ + XX(tuple,"tuple") \ + XX(typeassert,"typeassert") \ + XX(typeof,"typeof") \ -#ifdef DEFINE_BUILTIN_GLOBALS -#define DECLARE_BUILTIN(name) \ - JL_CALLABLE(jl_f_##name); \ - JL_DLLEXPORT jl_value_t *jl_builtin_##name; \ - JL_DLLEXPORT jl_fptr_args_t jl_f_##name##_addr = &jl_f_##name -#else -#define DECLARE_BUILTIN(name) \ - JL_CALLABLE(jl_f_##name); \ - JL_DLLEXPORT extern jl_value_t *jl_builtin_##name; \ - JL_DLLEXPORT extern jl_fptr_args_t jl_f_##name##_addr -#endif +#define DECLARE_BUILTIN(cname,jlname) \ + JL_CALLABLE(jl_f_##cname); +JL_BUILTIN_FUNCTIONS(DECLARE_BUILTIN) +#undef DECLARE_BUILTIN + +#define BUILTIN(cname) (jl_builtin_instances[jl_builtin_id_##cname]) + +enum jl_builtin_ids { +#define BUILTIN_IDS(cname,jlname) jl_builtin_id_##cname, +JL_BUILTIN_FUNCTIONS(BUILTIN_IDS) +#undef BUILTIN_IDS + jl_n_builtins +}; -DECLARE_BUILTIN(_apply_iterate); -DECLARE_BUILTIN(invoke_in_world); -DECLARE_BUILTIN(_call_in_world_total); -DECLARE_BUILTIN(invokelatest); -DECLARE_BUILTIN(_compute_sparams); -DECLARE_BUILTIN(_expr); -DECLARE_BUILTIN(_svec_ref); -DECLARE_BUILTIN(_typebody); -DECLARE_BUILTIN(_typevar); -DECLARE_BUILTIN(applicable); -DECLARE_BUILTIN(apply_type); -DECLARE_BUILTIN(compilerbarrier); -DECLARE_BUILTIN(current_scope); -DECLARE_BUILTIN(donotdelete); -DECLARE_BUILTIN(fieldtype); -DECLARE_BUILTIN(finalizer); -DECLARE_BUILTIN(getfield); -DECLARE_BUILTIN(getglobal); -DECLARE_BUILTIN(ifelse); -DECLARE_BUILTIN(invoke); -DECLARE_BUILTIN(is); -DECLARE_BUILTIN(isa); -DECLARE_BUILTIN(isdefined); -DECLARE_BUILTIN(isdefinedglobal); -DECLARE_BUILTIN(issubtype); -DECLARE_BUILTIN(memorynew); -DECLARE_BUILTIN(memoryref); -DECLARE_BUILTIN(memoryref_isassigned); -DECLARE_BUILTIN(memoryrefget); -DECLARE_BUILTIN(memoryrefmodify); -DECLARE_BUILTIN(memoryrefoffset); -DECLARE_BUILTIN(memoryrefreplace); -DECLARE_BUILTIN(memoryrefset); -DECLARE_BUILTIN(memoryrefsetonce); -DECLARE_BUILTIN(memoryrefswap); -DECLARE_BUILTIN(modifyfield); -DECLARE_BUILTIN(modifyglobal); -DECLARE_BUILTIN(nfields); -DECLARE_BUILTIN(replacefield); -DECLARE_BUILTIN(replaceglobal); -DECLARE_BUILTIN(setfield); -DECLARE_BUILTIN(setfieldonce); -DECLARE_BUILTIN(setglobal); -DECLARE_BUILTIN(setglobalonce); -DECLARE_BUILTIN(sizeof); -DECLARE_BUILTIN(svec); -DECLARE_BUILTIN(swapfield); -DECLARE_BUILTIN(swapglobal); -DECLARE_BUILTIN(throw); -DECLARE_BUILTIN(throw_methoderror); -DECLARE_BUILTIN(tuple); -DECLARE_BUILTIN(typeassert); -DECLARE_BUILTIN(typeof); +JL_DLLEXPORT extern jl_fptr_args_t const jl_builtin_f_addrs[]; +JL_DLLEXPORT extern const char *const jl_builtin_f_names[]; +JL_DLLEXPORT extern jl_value_t *jl_builtin_instances[]; -JL_CALLABLE(jl_f__structtype); -JL_CALLABLE(jl_f__abstracttype); -JL_CALLABLE(jl_f__primitivetype); -JL_CALLABLE(jl_f__setsuper); -JL_CALLABLE(jl_f__defaultctors); -JL_CALLABLE(jl_f__equiv_typedef); -JL_CALLABLE(jl_f_get_binding_type); -JL_CALLABLE(jl_f__compute_sparams); -JL_CALLABLE(jl_f__svec_ref); #ifdef __cplusplus } #endif diff --git a/src/builtins.c b/src/builtins.c index a1b485eee1e94..684ca3a691f7d 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -3,6 +3,8 @@ /* implementations of built-in functions */ +#include "dtypes.h" +#include "julia_atomics.h" #include "platform.h" #include @@ -30,6 +32,27 @@ extern "C" { #endif +jl_fptr_args_t const jl_builtin_f_addrs[jl_n_builtins] = { +#define BUILTIN_ADDRS(cname,jlname) &jl_f_##cname, +JL_BUILTIN_FUNCTIONS(BUILTIN_ADDRS) +#undef BUILTIN_ADDRS +}; + +const char *const jl_builtin_f_names[jl_n_builtins] = { +#define BUILTIN_F_NAMES(cname,jlname) XSTR(jl_f_##cname), +JL_BUILTIN_FUNCTIONS(BUILTIN_F_NAMES) +#undef BUILTIN_F_NAMES +}; + +jl_value_t *jl_builtin_instances[jl_n_builtins]; + +static const char *const jl_builtin_names[jl_n_builtins] = { +#define BUILTIN_NAMES(cname,jlname) jlname, +JL_BUILTIN_FUNCTIONS(BUILTIN_NAMES) +#undef BUILTIN_NAMES +}; + + // egal and object_id --------------------------------------------------------- static int bits_equal(const void *a, const void *b, int sz) JL_NOTSAFEPOINT @@ -649,7 +672,7 @@ JL_CALLABLE(jl_f__apply_iterate) nargs -= 1; if (nargs == 2) { // some common simple cases - if (f == jl_builtin_svec) { + if (f == BUILTIN(svec)) { if (jl_is_svec(args[1])) return args[1]; if (jl_is_genericmemory(args[1])) { @@ -674,7 +697,7 @@ JL_CALLABLE(jl_f__apply_iterate) return (jl_value_t*)t; } } - else if (f == jl_builtin_tuple && jl_is_tuple(args[1])) { + else if (f == BUILTIN(tuple) && jl_is_tuple(args[1])) { return args[1]; } } @@ -848,6 +871,7 @@ JL_CALLABLE(jl_f__apply_iterate) // this is like a regular call, but always runs in the newest world JL_CALLABLE(jl_f_invokelatest) { + JL_NARGSV(invokelatest, 1); jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; if (!ct->ptls->in_pure_callback) @@ -861,10 +885,10 @@ JL_CALLABLE(jl_f_invokelatest) // If world > jl_atomic_load_acquire(&jl_world_counter), run in the latest world. JL_CALLABLE(jl_f_invoke_in_world) { - JL_NARGSV(_apply_in_world, 2); + JL_NARGSV(invoke_in_world, 2); jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; - JL_TYPECHK(_apply_in_world, ulong, args[0]); + JL_TYPECHK(invoke_in_world, ulong, args[0]); size_t world = jl_unbox_ulong(args[0]); if (!ct->ptls->in_pure_callback) { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); @@ -879,7 +903,7 @@ JL_CALLABLE(jl_f_invoke_in_world) JL_CALLABLE(jl_f__call_in_world_total) { JL_NARGSV(_call_in_world_total, 2); - JL_TYPECHK(_apply_in_world, ulong, args[0]); + JL_TYPECHK(_call_in_world_total, ulong, args[0]); jl_task_t *ct = jl_current_task; int last_in = ct->ptls->in_pure_callback; jl_value_t *ret = NULL; @@ -1318,7 +1342,7 @@ JL_CALLABLE(jl_f_getglobal) jl_atomic_error("getglobal: module binding cannot be read non-atomically"); else if (order >= jl_memory_order_seq_cst) jl_fence(); - jl_value_t *v = jl_eval_global_var(mod, sym); // relaxed load + jl_value_t *v = jl_eval_global_var(mod, sym, jl_current_task->world_age); // relaxed load if (order >= jl_memory_order_acquire) jl_fence(); return v; @@ -1484,7 +1508,44 @@ JL_CALLABLE(jl_f_setglobalonce) return old == NULL ? jl_true : jl_false; } +// import, using -------------------------------------------------------------- + +// Import binding `from.sym` as `asname` into `to`: +// _import(to::Module, from::Module, asname::Symbol, sym::Symbol, imported::Bool) +// +// Create const binding to `mod` in `to` with name `asname`: +// _import(to::Module, mod::Module, asname::Symbol) +JL_CALLABLE(jl_f__import) +{ + JL_NARGS(_import, 3, 5); + JL_TYPECHK(_import, module, args[0]); + JL_TYPECHK(_import, module, args[1]); + JL_TYPECHK(_import, symbol, args[2]); + if (nargs == 3) { + jl_import_module(jl_current_task, (jl_module_t *)args[0], (jl_module_t *)args[1], + (jl_sym_t *)args[2]); + } + else if (nargs == 4) { + jl_too_few_args("_import", 5); + } + else if (nargs == 5) { + JL_TYPECHK(_import, symbol, args[3]); + JL_TYPECHK(_import, bool, args[4]); + jl_module_import(jl_current_task, (jl_module_t *)args[0], (jl_module_t *)args[1], + (jl_sym_t *)args[2], (jl_sym_t *)args[3], args[4] == jl_true); + } + return jl_nothing; +} +// _using(to::Module, from::Module) +JL_CALLABLE(jl_f__using) +{ + JL_NARGS(_using, 2, 2); + JL_TYPECHK(_using, module, args[0]); + JL_TYPECHK(_using, module, args[1]); + jl_module_using((jl_module_t *)args[0], (jl_module_t *)args[1]); + return jl_nothing; +} // apply_type ----------------------------------------------------------------- @@ -1692,11 +1753,11 @@ JL_CALLABLE(jl_f_memorynew) return (jl_value_t*)jl_alloc_genericmemory(args[0], nel); } -JL_CALLABLE(jl_f_memoryref) +JL_CALLABLE(jl_f_memoryrefnew) { - JL_NARGS(memoryref, 1, 3); + JL_NARGS(memoryrefnew, 1, 3); if (nargs == 1) { - JL_TYPECHK(memoryref, genericmemory, args[0]); + JL_TYPECHK(memoryrefnew, genericmemory, args[0]); jl_genericmemory_t *m = (jl_genericmemory_t*)args[0]; jl_value_t *typ = jl_apply_type((jl_value_t*)jl_genericmemoryref_type, jl_svec_data(((jl_datatype_t*)jl_typetagof(m))->parameters), 3); JL_GC_PROMISE_ROOTED(typ); // it is a concrete type @@ -1706,37 +1767,52 @@ JL_CALLABLE(jl_f_memoryref) return (jl_value_t*)jl_new_memoryref(typ, m, m->ptr); } else { - JL_TYPECHK(memoryref, genericmemoryref, args[0]); - JL_TYPECHK(memoryref, long, args[1]); + JL_TYPECHK(memoryrefnew, long, args[1]); if (nargs == 3) - JL_TYPECHK(memoryref, bool, args[2]); + JL_TYPECHK(memoryrefnew, bool, args[2]); + size_t i = (size_t) jl_unbox_long(args[1]) - 1; + char *data; + if (jl_is_genericmemory(args[0])) { + jl_genericmemory_t *m = (jl_genericmemory_t*)args[0]; + jl_value_t *typ = jl_apply_type((jl_value_t*)jl_genericmemoryref_type, jl_svec_data(((jl_datatype_t*)jl_typetagof(m))->parameters), 3); + JL_GC_PROMISE_ROOTED(typ); // it is a concrete type + if (i >= m->length) + jl_bounds_error((jl_value_t*)m, args[1]); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; + if (layout->flags.arrayelem_isunion || layout->size == 0) + return (jl_value_t*)jl_new_memoryref(typ, m, (char*)i); + else if (layout->flags.arrayelem_isboxed) + return (jl_value_t*)jl_new_memoryref(typ, m, (char*)m->ptr + sizeof(jl_value_t*)*i); + return (jl_value_t*)jl_new_memoryref(typ, m, (char*)m->ptr + layout->size*i); + } + JL_TYPECHK(memoryrefnew, genericmemoryref, args[0]); jl_genericmemoryref_t *m = (jl_genericmemoryref_t*)args[0]; - size_t i = jl_unbox_long(args[1]) - 1; - const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m->mem))->layout; - char *data = (char*)m->ptr_or_offset; + jl_genericmemory_t *mem = m->mem; + data = (char*)m->ptr_or_offset; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(mem))->layout; if (layout->flags.arrayelem_isboxed) { - if (((data - (char*)m->mem->ptr) / sizeof(jl_value_t*)) + i >= m->mem->length) + if (((data - (char*)mem->ptr) / sizeof(jl_value_t*)) + i >= mem->length) jl_bounds_error((jl_value_t*)m, args[1]); data += sizeof(jl_value_t*) * i; } else if (layout->flags.arrayelem_isunion || layout->size == 0) { - if ((size_t)data + i >= m->mem->length) + if ((size_t)data + i >= mem->length) jl_bounds_error((jl_value_t*)m, args[1]); data += i; } else { - if (((data - (char*)m->mem->ptr) / layout->size) + i >= m->mem->length) + if (((data - (char*)mem->ptr) / layout->size) + i >= mem->length) jl_bounds_error((jl_value_t*)m, args[1]); data += layout->size * i; } - return (jl_value_t*)jl_new_memoryref((jl_value_t*)jl_typetagof(m), m->mem, data); + return (jl_value_t*)jl_new_memoryref((jl_value_t*)jl_typetagof(m), mem, data); } } JL_CALLABLE(jl_f_memoryrefoffset) { JL_NARGS(memoryrefoffset, 1, 1); - JL_TYPECHK(memoryref, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefoffest, genericmemoryref, args[0]); jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; size_t offset; @@ -2398,8 +2474,7 @@ static void add_intrinsic(jl_module_t *inm, const char *name, enum intrinsic f) { jl_value_t *i = jl_permbox32(jl_intrinsic_type, 0, (int32_t)f); jl_sym_t *sym = jl_symbol(name); - jl_set_const(inm, sym, i); - jl_module_public(inm, sym, 1); + jl_set_initial_const(inm, sym, i, 1); } void jl_init_intrinsic_properties(void) JL_GC_DISABLED @@ -2415,18 +2490,14 @@ void jl_init_intrinsic_properties(void) JL_GC_DISABLED void jl_init_intrinsic_functions(void) JL_GC_DISABLED { - jl_module_t *inm = jl_new_module(jl_symbol("Intrinsics"), NULL); - inm->parent = jl_core_module; - jl_set_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm); - jl_mk_builtin_func(jl_intrinsic_type, "IntrinsicFunction", jl_f_intrinsic_call); - jl_mk_builtin_func( - (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_opaque_closure_type), - "OpaqueClosure", jl_f_opaque_closure_call); + jl_module_t *inm = jl_new_module_(jl_symbol("Intrinsics"), jl_core_module, 0, 1); + jl_set_initial_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm, 0); + jl_mk_builtin_func(jl_intrinsic_type, jl_symbol("IntrinsicFunction"), jl_f_intrinsic_call); + jl_datatype_t *oc = (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_opaque_closure_type); // Save a reference to the just created OpaqueClosure method, so we can provide special // codegen for it later. - jl_opaque_closure_method = (jl_method_t*)jl_methtable_lookup(jl_opaque_closure_typename->mt, - (jl_value_t*)jl_anytuple_type, 1); + jl_opaque_closure_method = jl_mk_builtin_func(oc, jl_symbol("OpaqueClosure"), jl_f_opaque_closure_call); // TODO: awkwardly not actually declared a Builtin, even though it relies on being handled by the special cases for Builtin everywhere else #define ADD_I(name, nargs) add_intrinsic(inm, #name, name); #define ADD_HIDDEN(name, nargs) @@ -2439,96 +2510,24 @@ void jl_init_intrinsic_functions(void) JL_GC_DISABLED static void add_builtin(const char *name, jl_value_t *v) { - jl_set_const(jl_core_module, jl_symbol(name), v); -} - -jl_fptr_args_t jl_get_builtin_fptr(jl_datatype_t *dt) -{ - assert(jl_subtype((jl_value_t*)dt, (jl_value_t*)jl_builtin_type)); - jl_typemap_entry_t *entry = (jl_typemap_entry_t*)jl_atomic_load_relaxed(&dt->name->mt->defs); - jl_method_instance_t *mi = jl_atomic_load_relaxed(&entry->func.method->unspecialized); - jl_code_instance_t *ci = jl_atomic_load_relaxed(&mi->cache); - assert(ci->owner == jl_nothing); - return jl_atomic_load_relaxed(&ci->specptr.fptr1); -} - -static jl_value_t *add_builtin_func(const char *name, jl_fptr_args_t fptr) -{ - return jl_mk_builtin_func(NULL, name, fptr)->instance; + jl_set_initial_const(jl_core_module, jl_symbol(name), v, 0); } void jl_init_primitives(void) JL_GC_DISABLED { - jl_builtin_is = add_builtin_func("===", jl_f_is); - jl_builtin_typeof = add_builtin_func("typeof", jl_f_typeof); - jl_builtin_sizeof = add_builtin_func("sizeof", jl_f_sizeof); - jl_builtin_issubtype = add_builtin_func("<:", jl_f_issubtype); - jl_builtin_isa = add_builtin_func("isa", jl_f_isa); - jl_builtin_typeassert = add_builtin_func("typeassert", jl_f_typeassert); - jl_builtin_throw = add_builtin_func("throw", jl_f_throw); - jl_builtin_tuple = add_builtin_func("tuple", jl_f_tuple); - jl_builtin_ifelse = add_builtin_func("ifelse", jl_f_ifelse); - - // field access - jl_builtin_getfield = add_builtin_func("getfield", jl_f_getfield); - jl_builtin_setfield = add_builtin_func("setfield!", jl_f_setfield); - jl_builtin_setfieldonce = add_builtin_func("setfieldonce!", jl_f_setfieldonce); - jl_builtin_swapfield = add_builtin_func("swapfield!", jl_f_swapfield); - jl_builtin_modifyfield = add_builtin_func("modifyfield!", jl_f_modifyfield); - jl_builtin_replacefield = add_builtin_func("replacefield!", jl_f_replacefield); - jl_builtin_fieldtype = add_builtin_func("fieldtype", jl_f_fieldtype); - jl_builtin_nfields = add_builtin_func("nfields", jl_f_nfields); - jl_builtin_isdefined = add_builtin_func("isdefined", jl_f_isdefined); - - // module bindings - jl_builtin_getglobal = add_builtin_func("getglobal", jl_f_getglobal); - jl_builtin_setglobal = add_builtin_func("setglobal!", jl_f_setglobal); - jl_builtin_isdefinedglobal = add_builtin_func("isdefinedglobal", jl_f_isdefinedglobal); - add_builtin_func("get_binding_type", jl_f_get_binding_type); - jl_builtin_swapglobal = add_builtin_func("swapglobal!", jl_f_swapglobal); - jl_builtin_replaceglobal = add_builtin_func("replaceglobal!", jl_f_replaceglobal); - jl_builtin_modifyglobal = add_builtin_func("modifyglobal!", jl_f_modifyglobal); - jl_builtin_setglobalonce = add_builtin_func("setglobalonce!", jl_f_setglobalonce); - - // memory primitives - jl_builtin_memorynew = add_builtin_func("memorynew", jl_f_memorynew); - jl_builtin_memoryref = add_builtin_func("memoryrefnew", jl_f_memoryref); - jl_builtin_memoryrefoffset = add_builtin_func("memoryrefoffset", jl_f_memoryrefoffset); - jl_builtin_memoryrefget = add_builtin_func("memoryrefget", jl_f_memoryrefget); - jl_builtin_memoryrefset = add_builtin_func("memoryrefset!", jl_f_memoryrefset); - jl_builtin_memoryref_isassigned = add_builtin_func("memoryref_isassigned", jl_f_memoryref_isassigned); - jl_builtin_memoryrefswap = add_builtin_func("memoryrefswap!", jl_f_memoryrefswap); - jl_builtin_memoryrefreplace = add_builtin_func("memoryrefreplace!", jl_f_memoryrefreplace); - jl_builtin_memoryrefmodify = add_builtin_func("memoryrefmodify!", jl_f_memoryrefmodify); - jl_builtin_memoryrefsetonce = add_builtin_func("memoryrefsetonce!", jl_f_memoryrefsetonce); - - // method table utils - jl_builtin_applicable = add_builtin_func("applicable", jl_f_applicable); - jl_builtin_invoke = add_builtin_func("invoke", jl_f_invoke); - - // internal functions - jl_builtin_apply_type = add_builtin_func("apply_type", jl_f_apply_type); - jl_builtin__apply_iterate = add_builtin_func("_apply_iterate", jl_f__apply_iterate); - jl_builtin__expr = add_builtin_func("_expr", jl_f__expr); - jl_builtin_svec = add_builtin_func("svec", jl_f_svec); - add_builtin_func("invokelatest", jl_f_invokelatest); - add_builtin_func("invoke_in_world", jl_f_invoke_in_world); - add_builtin_func("_call_in_world_total", jl_f__call_in_world_total); - add_builtin_func("_typevar", jl_f__typevar); - add_builtin_func("_structtype", jl_f__structtype); - add_builtin_func("_abstracttype", jl_f__abstracttype); - add_builtin_func("_primitivetype", jl_f__primitivetype); - add_builtin_func("_setsuper!", jl_f__setsuper); - add_builtin_func("_defaultctors", jl_f__defaultctors); - jl_builtin__typebody = add_builtin_func("_typebody!", jl_f__typebody); - add_builtin_func("_equiv_typedef", jl_f__equiv_typedef); - jl_builtin_donotdelete = add_builtin_func("donotdelete", jl_f_donotdelete); - jl_builtin_compilerbarrier = add_builtin_func("compilerbarrier", jl_f_compilerbarrier); - add_builtin_func("finalizer", jl_f_finalizer); - add_builtin_func("_compute_sparams", jl_f__compute_sparams); - add_builtin_func("_svec_ref", jl_f__svec_ref); - jl_builtin_current_scope = add_builtin_func("current_scope", jl_f_current_scope); - add_builtin_func("throw_methoderror", jl_f_throw_methoderror); + // Builtins are specially considered available from world 0 + for (int i = 0; i < jl_n_builtins; i++) { + if (i == jl_builtin_id_intrinsic_call || + i == jl_builtin_id_opaque_closure_call) + continue; + jl_sym_t *sname = jl_symbol(jl_builtin_names[i]); + jl_value_t *builtin = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0); + jl_set_initial_const(jl_core_module, sname, builtin, 0); + jl_mk_builtin_func((jl_datatype_t*)jl_typeof(builtin), sname, jl_builtin_f_addrs[i]); + jl_builtin_instances[i] = builtin; + } + add_builtin("OpaqueClosure", (jl_value_t*)jl_opaque_closure_type); + add_builtin("IntrinsicFunction", (jl_value_t*)jl_intrinsic_type); // builtin types add_builtin("Any", (jl_value_t*)jl_any_type); @@ -2548,6 +2547,8 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("Module", (jl_value_t*)jl_module_type); add_builtin("MethodTable", (jl_value_t*)jl_methtable_type); + add_builtin("methodtable", (jl_value_t*)jl_method_table); + add_builtin("MethodCache", (jl_value_t*)jl_methcache_type); add_builtin("Method", (jl_value_t*)jl_method_type); add_builtin("CodeInstance", (jl_value_t*)jl_code_instance_type); add_builtin("TypeMapEntry", (jl_value_t*)jl_typemap_entry_type); @@ -2561,14 +2562,12 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("PartialOpaque", (jl_value_t*)jl_partial_opaque_type); add_builtin("InterConditional", (jl_value_t*)jl_interconditional_type); add_builtin("MethodMatch", (jl_value_t*)jl_method_match_type); - add_builtin("IntrinsicFunction", (jl_value_t*)jl_intrinsic_type); add_builtin("Function", (jl_value_t*)jl_function_type); add_builtin("Builtin", (jl_value_t*)jl_builtin_type); add_builtin("MethodInstance", (jl_value_t*)jl_method_instance_type); add_builtin("CodeInfo", (jl_value_t*)jl_code_info_type); add_builtin("LLVMPtr", (jl_value_t*)jl_llvmpointer_type); add_builtin("Task", (jl_value_t*)jl_task_type); - add_builtin("OpaqueClosure", (jl_value_t*)jl_opaque_closure_type); add_builtin("AddrSpace", (jl_value_t*)jl_addrspace_type); add_builtin("Ref", (jl_value_t*)jl_ref_type); @@ -2614,6 +2613,26 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("AbstractString", (jl_value_t*)jl_abstractstring_type); add_builtin("String", (jl_value_t*)jl_string_type); + + // ensure that primitive types are fully allocated (since jl_init_types is incomplete) + assert(jl_atomic_load_relaxed(&jl_world_counter) == 1); + jl_module_t *core = jl_core_module; + jl_svec_t *bindings = jl_atomic_load_relaxed(&core->bindings); + jl_value_t **table = jl_svec_data(bindings); + for (size_t i = 0; i < jl_svec_len(bindings); i++) { + if (table[i] != jl_nothing) { + jl_binding_t *b = (jl_binding_t*)table[i]; + jl_value_t *v = jl_get_binding_value_in_world(b, 1); + if (v) { + if (jl_is_unionall(v)) + v = jl_unwrap_unionall(v); + if (jl_is_datatype(v)) { + jl_datatype_t *tt = (jl_datatype_t*)v; + tt->name->module = core; + } + } + } + } } #ifdef __cplusplus diff --git a/src/cgutils.cpp b/src/cgutils.cpp index baf9bc046d6ce..52316e4652012 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -341,11 +341,11 @@ static void find_perm_offsets(jl_datatype_t *typ, SmallVectorImpl &res } // load a pointer to N inlined_roots into registers (as a SmallVector) -static llvm::SmallVector load_gc_roots(jl_codectx_t &ctx, Value *inline_roots_ptr, size_t npointers, bool isVolatile=false) +static llvm::SmallVector load_gc_roots(jl_codectx_t &ctx, Value *inline_roots_ptr, size_t npointers, MDNode *tbaa, bool isVolatile=false) { SmallVector gcroots(npointers); Type *T_prjlvalue = ctx.types().T_prjlvalue; - auto roots_ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); + auto roots_ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); for (size_t i = 0; i < npointers; i++) { auto *ptr = ctx.builder.CreateAlignedLoad(T_prjlvalue, emit_ptrgep(ctx, inline_roots_ptr, i * sizeof(jl_value_t*)), Align(sizeof(void*)), isVolatile); roots_ai.decorateInst(ptr); @@ -1481,6 +1481,7 @@ static Value *emit_sizeof(jl_codectx_t &ctx, const jl_cgval_t &p) return dyn_size; } } +*/ static Value *emit_datatype_mutabl(jl_codectx_t &ctx, Value *dt) { @@ -1495,7 +1496,6 @@ static Value *emit_datatype_mutabl(jl_codectx_t &ctx, Value *dt) mutabl = ctx.builder.CreateLShr(mutabl, 1); return ctx.builder.CreateTrunc(mutabl, getInt1Ty(ctx.builder.getContext())); } -*/ static Value *emit_datatype_isprimitivetype(jl_codectx_t &ctx, Value *typ) { @@ -1603,10 +1603,17 @@ static void undef_var_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *name, ctx.builder.SetInsertPoint(ifok); } +// ctx.builder.CreateIsNotNull(v) lowers incorrectly in non-standard +// address spaces where null is not zero +// TODO: adapt to https://github.com/llvm/llvm-project/pull/131557 once merged static Value *null_pointer_cmp(jl_codectx_t &ctx, Value *v) { ++EmittedNullchecks; - return ctx.builder.CreateIsNotNull(v); + Type *T = v->getType(); + return ctx.builder.CreateICmpNE( + v, + ctx.builder.CreateAddrSpaceCast( + Constant::getNullValue(ctx.builder.getPtrTy(0)), T)); } @@ -2267,8 +2274,10 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j return mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); } +static Function *emit_modifyhelper(jl_codectx_t &ctx2, const jl_cgval_t &op, const jl_cgval_t &modifyop, jl_value_t *jltype, Type *elty, jl_cgval_t rhs, const Twine &fname, bool gcstack_arg); + static jl_cgval_t typed_store(jl_codectx_t &ctx, - Value *ptr, jl_cgval_t rhs, jl_cgval_t cmp, + Value *ptr, jl_cgval_t rhs, jl_cgval_t cmpop, jl_value_t *jltype, MDNode *tbaa, MDNode *aliasscope, Value *parent, // for the write barrier, NULL if no barrier needed bool isboxed, AtomicOrdering Order, AtomicOrdering FailOrder, unsigned alignment, @@ -2277,10 +2286,10 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *var) { auto newval = [&](const jl_cgval_t &lhs) { - const jl_cgval_t argv[3] = { cmp, lhs, rhs }; + const jl_cgval_t argv[3] = { cmpop, lhs, rhs }; jl_cgval_t ret; if (modifyop) { - ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, true); } else { Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); @@ -2304,7 +2313,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, return rhs; } else if (isreplacefield) { - Value *Success = emit_f_is(ctx, cmp, ghostValue(ctx, jltype)); + Value *Success = emit_f_is(ctx, cmpop, ghostValue(ctx, jltype)); Success = ctx.builder.CreateZExt(Success, getInt8Ty(ctx.builder.getContext())); const jl_cgval_t argv[2] = {ghostValue(ctx, jltype), mark_julia_type(ctx, Success, false, jl_bool_type)}; jl_datatype_t *rettyp = jl_apply_cmpswap_type(jltype); @@ -2405,6 +2414,46 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ai.decorateInst(store); instr = store; } + else if (ismodifyfield && modifyop && !needlock && Order != AtomicOrdering::NotAtomic && !isboxed && realelty == elty && !intcast && elty->isIntegerTy() && !jl_type_hasptr(jltype)) { + // emit this only if we have a possibility of optimizing it + if (Order == AtomicOrdering::Unordered) + Order = AtomicOrdering::Monotonic; + if (jl_is_pointerfree(jl_pinned_ref_get(rhs.typ)) && !rhs.isghost && (rhs.constant || rhs.isboxed || rhs.ispointer())) { + // if this value can be loaded from memory, do that now so that it is sequenced before the atomicmodify + // and the IR is less dependent on what was emitted before now to create this rhs. + // Inlining should do okay to clean this up later if there are parts we don't need. + rhs = jl_cgval_t(emit_unbox(ctx, julia_type_to_llvm(ctx, jl_pinned_ref_get(rhs.typ)), rhs, jl_pinned_ref_get(rhs.typ)), jl_pinned_ref_get(rhs.typ), NULL); + } + bool gcstack_arg = JL_FEAT_TEST(ctx,gcstack_arg); + Function *op = emit_modifyhelper(ctx, cmpop, *modifyop, jltype, elty, rhs, fname, gcstack_arg); + std::string intr_name = "julia.atomicmodify.i"; + intr_name += utostr(cast(elty)->getBitWidth()); + intr_name += ".p"; + intr_name += utostr(ptr->getType()->getPointerAddressSpace()); + FunctionCallee intr = jl_Module->getOrInsertFunction(intr_name, + FunctionType::get(StructType::get(elty, elty), {ptr->getType(), ctx.builder.getPtrTy(), ctx.builder.getInt8Ty(), ctx.builder.getInt8Ty()}, true), + AttributeList::get(elty->getContext(), + Attributes(elty->getContext(), {Attribute::NoMerge}), // prevent llvm from merging calls to different functions + AttributeSet(), + None)); + SmallVector Args = {ptr, op, ctx.builder.getInt8((unsigned)Order), ctx.builder.getInt8(SyncScope::System)}; + if (rhs.V) + Args.push_back(rhs.V); + if (rhs.Vboxed) + Args.push_back(rhs.Vboxed); + if (rhs.TIndex) + Args.push_back(rhs.TIndex); + Args.append(rhs.inline_roots); + if (gcstack_arg) + Args.push_back(ctx.pgcstack); + auto oldnew = ctx.builder.CreateCall(intr, Args); + oldnew->addParamAttr(0, Attribute::getWithAlignment(oldnew->getContext(), Align(alignment))); + //jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + //ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + //ai.decorateInst(oldnew); + oldval = mark_julia_type(ctx, ctx.builder.CreateExtractValue(oldnew, 0), isboxed, jltype); + rhs = mark_julia_type(ctx, ctx.builder.CreateExtractValue(oldnew, 1), isboxed, jltype); + } else { // replacefield, modifyfield, swapfield, setfieldonce (isboxed && atomic) DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg", ctx.f); @@ -2418,7 +2467,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, assert(jl_is_concrete_type(jltype)); needloop = ((jl_datatype_t*)jltype)->layout->flags.haspadding || !((jl_datatype_t*)jltype)->layout->flags.isbitsegal; - Value *SameType = emit_isa(ctx, cmp, jltype, Twine()).first; + Value *SameType = emit_isa(ctx, cmpop, jltype, Twine()).first; if (SameType != ConstantInt::getTrue(ctx.builder.getContext())) { BasicBlock *SkipBB = BasicBlock::Create(ctx.builder.getContext(), "skip_xchg", ctx.f); BasicBlock *BB = BasicBlock::Create(ctx.builder.getContext(), "ok_xchg", ctx.f); @@ -2438,22 +2487,22 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Current->addIncoming(instr, SkipBB); ctx.builder.SetInsertPoint(BB); } - cmp = update_julia_type(ctx, cmp, jltype); + cmpop = update_julia_type(ctx, cmpop, jltype); if (intcast) { - emit_unbox_store(ctx, cmp, intcast, ctx.tbaa().tbaa_stack, MaybeAlign(), intcast->getAlign()); + emit_unbox_store(ctx, cmpop, intcast, ctx.tbaa().tbaa_stack, MaybeAlign(), intcast->getAlign()); Compare = ctx.builder.CreateLoad(realelty, intcast); } else { - Compare = emit_unbox(ctx, realelty, cmp, jltype); + Compare = emit_unbox(ctx, realelty, cmpop, jltype); } if (realelty != elty) Compare = ctx.builder.CreateZExt(Compare, elty); } - else if (cmp.isboxed || cmp.constant || jl_pointer_egal(jltype)) { - Compare = boxed(ctx, cmp); - needloop = !jl_pointer_egal(jltype) && !jl_pointer_egal(jl_pinned_ref_get(cmp.typ)); - if (needloop && !cmp.isboxed) // try to use the same box in the compare now and later - cmp = mark_julia_type(ctx, Compare, true, jl_pinned_ref_get(cmp.typ)); + else if (cmpop.isboxed || cmpop.constant || jl_pointer_egal(jltype)) { + Compare = boxed(ctx, cmpop); + needloop = !jl_pointer_egal(jltype) && !jl_pointer_egal(jl_pinned_ref_get(cmpop.typ)); + if (needloop && !cmpop.isboxed) // try to use the same box in the compare now and later + cmpop = mark_julia_type(ctx, Compare, true, jl_pinned_ref_get(cmpop.typ)); } else { Compare = Constant::getNullValue(ctx.types().T_prjlvalue); // TODO: does this need to be an invalid bit pattern? @@ -2487,7 +2536,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } if (ismodifyfield) { if (needlock) - emit_lockstate_value(ctx, needlock, false); + emit_lockstate_value(ctx, needlock, false); // unlock Value *realCompare = Compare; if (realelty != elty) realCompare = ctx.builder.CreateTrunc(realCompare, realelty); @@ -2522,8 +2571,8 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (realelty != elty) r = ctx.builder.CreateZExt(r, elty); if (needlock) - emit_lockstate_value(ctx, needlock, true); - cmp = oldval; + emit_lockstate_value(ctx, needlock, true); // relock + cmpop = oldval; } Value *Done; if (Order == AtomicOrdering::NotAtomic) { @@ -2543,7 +2592,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (issetfieldonce) Success = ctx.builder.CreateIsNull(first_ptr); else - Success = emit_f_is(ctx, oldval, cmp, first_ptr, nullptr); + Success = emit_f_is(ctx, oldval, cmpop, first_ptr, nullptr); if (needloop && ismodifyfield) CmpPhi->addIncoming(load, ctx.builder.GetInsertBlock()); assert(Succ == nullptr); @@ -2601,12 +2650,12 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Done = ctx.builder.CreateIsNotNull(first_ptr); } else { - // Done = !(!Success && (first_ptr != NULL && oldval == cmp)) + // Done = !(!Success && (first_ptr != NULL && oldval == cmpop)) Done = emit_guarded_test(ctx, ctx.builder.CreateNot(Success), false, [&] { Value *first_ptr = nullptr; if (maybe_null_if_boxed) first_ptr = isboxed ? realinstr : extract_first_ptr(ctx, realinstr); - return emit_f_is(ctx, oldval, cmp, first_ptr, nullptr); + return emit_f_is(ctx, oldval, cmpop, first_ptr, nullptr); }); Done = ctx.builder.CreateNot(Done); } @@ -2643,6 +2692,11 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.CreateStore(r, intcast); r = ctx.builder.CreateLoad(intcast_eltyp, intcast); } + else if (!isboxed && intcast_eltyp) { + assert(issetfield); + // issetfield doesn't use intcast, so need to reload rhs with the correct type + r = emit_unbox(ctx, intcast_eltyp, rhs, jltype); + } if (!isboxed) emit_write_multibarrier(ctx, parent, r, jl_pinned_ref_get(rhs.typ)); else @@ -3247,7 +3301,7 @@ static Value *emit_genericmemoryelsize(jl_codectx_t &ctx, Value *v, jl_value_t * if (jl_is_genericmemoryref_type(sty)) sty = (jl_datatype_t*)jl_field_type_concrete(sty, 1); size_t sz = sty->layout->size; - if (sty->layout->flags.arrayelem_isunion) + if (sty->layout->flags.arrayelem_isunion && add_isunion) sz++; auto elsize = ConstantInt::get(ctx.types().T_size, sz); return elsize; @@ -4026,7 +4080,7 @@ static jl_cgval_t union_store(jl_codectx_t &ctx, emit_lockstate_value(ctx, needlock, false); const jl_cgval_t argv[3] = { cmp, oldval, rhs }; if (modifyop) { - rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); + rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type, true); } else { Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); @@ -4101,7 +4155,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, size_t fsz1 = jl_field_size(sty, idx0) - 1; Value *ptindex = emit_ptrgep(ctx, addr, fsz1); setNameWithField(ctx.emission_context, ptindex, get_objname, sty, idx0, Twine(".tindex_ptr")); - return union_store(ctx, addr, ptindex, rhs, cmp, jfty, tbaa, ctx.tbaa().tbaa_unionselbyte, + return union_store(ctx, addr, ptindex, rhs, cmp, jfty, tbaa, strct.tbaa, Order, FailOrder, needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, issetfieldonce, modifyop, fname); @@ -4369,7 +4423,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg undef_derived_strct(ctx, strct, sty, strctinfo.tbaa); for (size_t i = nargs; i < nf; i++) { if (!jl_field_isptr(sty, i) && jl_is_uniontype(jl_field_type(sty, i))) { - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, strctinfo.tbaa); ai.decorateInst(ctx.builder.CreateAlignedStore( ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), emit_ptrgep(ctx, strct, jl_field_offset(sty, i) + jl_field_size(sty, i) - 1), @@ -4632,6 +4686,45 @@ static jl_cgval_t _emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &mem, cons return _emit_memoryref(ctx, boxed(ctx, mem), data, layout, typ); } +static jl_cgval_t emit_memoryref_direct(jl_codectx_t &ctx, const jl_cgval_t &mem, jl_cgval_t idx, jl_value_t *typ, jl_value_t *inbounds, const jl_datatype_layout_t *layout) +{ + bool isboxed = layout->flags.arrayelem_isboxed; + bool isunion = layout->flags.arrayelem_isunion; + bool isghost = layout->size == 0; + Value *boxmem = boxed(ctx, mem); + Value *i = emit_unbox(ctx, ctx.types().T_size, idx, (jl_value_t*)jl_long_type); + Value *idx0 = ctx.builder.CreateSub(i, ConstantInt::get(ctx.types().T_size, 1)); + bool bc = bounds_check_enabled(ctx, inbounds); + if (bc) { + BasicBlock *failBB, *endBB; + failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); + endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); + Value *mlen = emit_genericmemorylen(ctx, boxmem, typ); + Value *inbound = ctx.builder.CreateICmpULT(idx0, mlen); + setName(ctx.emission_context, inbound, "memoryref_isinbounds"); + ctx.builder.CreateCondBr(inbound, endBB, failBB); + failBB->insertInto(ctx.f); + ctx.builder.SetInsertPoint(failBB); + ctx.builder.CreateCall(prepare_call(jlboundserror_func), + { mark_callee_rooted(ctx, boxmem), i }); + ctx.builder.CreateUnreachable(); + endBB->insertInto(ctx.f); + ctx.builder.SetInsertPoint(endBB); + } + Value *data; + + if ((!isboxed && isunion) || isghost) { + data = idx0; + + } else { + data = emit_genericmemoryptr(ctx, boxmem, layout, 0); + idx0 = ctx.builder.CreateMul(idx0, emit_genericmemoryelsize(ctx, boxmem, jl_pinned_ref_get(mem.typ), false), "", true, true); + data = ctx.builder.CreatePtrAdd(data, idx0); + } + + return _emit_memoryref(ctx, boxmem, data, layout, typ); +} + static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, const jl_datatype_layout_t *layout) { if (!ref.inline_roots.empty()) { @@ -4664,7 +4757,7 @@ static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, const static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cgval_t idx, jl_value_t *inbounds, const jl_datatype_layout_t *layout) { ++EmittedArrayNdIndex; - emit_typecheck(ctx, idx, (jl_value_t*)jl_long_type, "memoryref"); + emit_typecheck(ctx, idx, (jl_value_t*)jl_long_type, "memoryrefnew"); idx = update_julia_type(ctx, idx, (jl_value_t*)jl_long_type); if (idx.typ == jl_bottom_type) return jl_cgval_t(); @@ -4729,9 +4822,10 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg setName(ctx.emission_context, ovflw, "memoryref_ovflw"); } #endif - Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jl_tparam1(jl_pinned_ref_get(ref.typ))); - newdata = ctx.builder.CreateGEP(elty, data, offset); - setName(ctx.emission_context, newdata, "memoryref_data_offset"); + boffset = ctx.builder.CreateMul(offset, elsz); + setName(ctx.emission_context, boffset, "memoryref_byteoffset"); + newdata = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), data, boffset); + setName(ctx.emission_context, newdata, "memoryref_data_byteoffset"); (void)boffset; // LLVM is very bad at handling GEP with types different from the load if (bc) { BasicBlock *failBB, *endBB; diff --git a/src/clangsa/GCChecker.cpp b/src/clangsa/GCChecker.cpp index af07ca2227839..09a034a9549d8 100644 --- a/src/clangsa/GCChecker.cpp +++ b/src/clangsa/GCChecker.cpp @@ -836,6 +836,7 @@ bool GCChecker::isGCTrackedType(QualType QT) { Name.ends_with_insensitive("jl_typemap_t") || Name.ends_with_insensitive("jl_unionall_t") || Name.ends_with_insensitive("jl_methtable_t") || + Name.ends_with_insensitive("jl_methcache_t") || Name.ends_with_insensitive("jl_cgval_t") || Name.ends_with_insensitive("jl_codectx_t") || Name.ends_with_insensitive("jl_ast_context_t") || diff --git a/src/codegen-stubs.c b/src/codegen-stubs.c index 04f38fb9091be..e083d1d90eeba 100644 --- a/src/codegen-stubs.c +++ b/src/codegen-stubs.c @@ -14,6 +14,7 @@ JL_DLLEXPORT void jl_dump_native_fallback(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, ios_t *z, ios_t *s) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_gvs_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE +JL_DLLEXPORT void jl_get_llvm_gv_inits_fallback(void *native_code, arraylist_t *inits) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_external_fns_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_mis_fallback(void *native_code, arraylist_t* MIs) UNAVAILABLE diff --git a/src/codegen.cpp b/src/codegen.cpp index 23a4587baaf8c..ac3cda587880a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -239,13 +239,13 @@ extern void __stack_chk_fail(); #ifdef _OS_WINDOWS_ #if defined(_CPU_X86_64_) -#if defined(_COMPILER_GCC_) +#if defined(__MINGW32__) extern void ___chkstk_ms(void); #else extern void __chkstk(void); #endif #else -#if defined(_COMPILER_GCC_) +#if defined(__MINGW32__) #undef _alloca extern void _alloca(void); #else @@ -341,7 +341,7 @@ struct jl_tbaacache_t { MDNode *tbaa_ptrarraybuf; // Data in an array of boxed values MDNode *tbaa_arraybuf; // Data in an array of POD MDNode *tbaa_array; // jl_array_t or jl_genericmemory_t - MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t (to memoryref) + MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t (to a memoryref) MDNode *tbaa_arraysize; // A size in a jl_array_t MDNode *tbaa_arrayselbyte; // a selector byte in a isbits Union jl_genericmemory_t MDNode *tbaa_memoryptr; // The pointer inside a jl_genericmemory_t @@ -555,7 +555,14 @@ FunctionType *invoke_type(TypeFnContextAndTriple f, Module &M) template struct JuliaFunction { public: - llvm::StringLiteral name; + template + constexpr JuliaFunction(const char (&cname)[N], TypeFn_t _type, llvm::AttributeList (*_attrs)(llvm::LLVMContext &C)) + : name(StringRef(cname, N-1)), _type(_type), _attrs(_attrs) {} + JuliaFunction(StringRef cname, TypeFn_t _type, llvm::AttributeList (*_attrs)(llvm::LLVMContext &C)) + : name(cname), _type(_type), _attrs(_attrs) {} + JuliaFunction(char *cname, TypeFn_t _type, llvm::AttributeList (*_attrs)(llvm::LLVMContext &C)) = delete; + + llvm::StringRef name; TypeFn_t _type; llvm::AttributeList (*_attrs)(llvm::LLVMContext &C); @@ -604,10 +611,6 @@ static FunctionType *get_func_sig(LLVMContext &C) { return JuliaType::get_jlfunc static FunctionType *get_func2_sig(LLVMContext &C) { return JuliaType::get_jlfunc2_ty(C); } static FunctionType *get_func3_sig(LLVMContext &C) { return JuliaType::get_jlfunc3_ty(C); } -static FunctionType *get_donotdelete_sig(LLVMContext &C) { - return FunctionType::get(getVoidTy(C), true); -} - static AttributeList get_func_attrs(LLVMContext &C) { return AttributeList::get(C, @@ -617,18 +620,6 @@ static AttributeList get_func_attrs(LLVMContext &C) Attributes(C, {Attribute::NoAlias, Attribute::ReadOnly, Attribute::NoCapture, Attribute::NoUndef})}); } -static AttributeList get_donotdelete_func_attrs(LLVMContext &C) -{ - AttrBuilder FnAttrs(C); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); - FnAttrs.addAttribute(Attribute::WillReturn); - FnAttrs.addAttribute(Attribute::NoUnwind); - return AttributeList::get(C, - AttributeSet::get(C, FnAttrs), - Attributes(C, {}), - None); -} - static AttributeList get_attrs_noreturn(LLVMContext &C) { return AttributeList::get(C, @@ -650,7 +641,7 @@ static AttributeList get_attrs_box_float(LLVMContext &C, unsigned nbytes) auto FnAttrs = AttrBuilder(C); FnAttrs.addAttribute(Attribute::WillReturn); FnAttrs.addAttribute(Attribute::NoUnwind); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly() | MemoryEffects::readOnly()); auto RetAttrs = AttrBuilder(C); RetAttrs.addAttribute(Attribute::NonNull); RetAttrs.addDereferenceableAttr(nbytes); @@ -666,7 +657,7 @@ static AttributeList get_attrs_box_sext(LLVMContext &C, unsigned nbytes) auto FnAttrs = AttrBuilder(C); FnAttrs.addAttribute(Attribute::WillReturn); FnAttrs.addAttribute(Attribute::NoUnwind); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly() | MemoryEffects::readOnly()); auto RetAttrs = AttrBuilder(C); RetAttrs.addAttribute(Attribute::NonNull); RetAttrs.addAttribute(Attribute::getWithDereferenceableBytes(C, nbytes)); @@ -683,7 +674,7 @@ static AttributeList get_attrs_box_zext(LLVMContext &C, unsigned nbytes) auto FnAttrs = AttrBuilder(C); FnAttrs.addAttribute(Attribute::WillReturn); FnAttrs.addAttribute(Attribute::NoUnwind); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly() | MemoryEffects::readOnly()); auto RetAttrs = AttrBuilder(C); RetAttrs.addAttribute(Attribute::NonNull); RetAttrs.addDereferenceableAttr(nbytes); @@ -1134,7 +1125,7 @@ static const auto jl_alloc_obj_func = new JuliaFunction{ auto FnAttrs = AttrBuilder(C); FnAttrs.addAllocSizeAttr(1, None); // returns %1 bytes FnAttrs.addAllocKindAttr(AllocFnKind::Alloc); - FnAttrs.addMemoryAttr(MemoryEffects::argMemOnly(ModRefInfo::Ref) | MemoryEffects::inaccessibleMemOnly(ModRefInfo::ModRef)); + FnAttrs.addMemoryAttr(MemoryEffects::argMemOnly(ModRefInfo::Ref) | MemoryEffects::inaccessibleMemOnly()); FnAttrs.addAttribute(Attribute::WillReturn); FnAttrs.addAttribute(Attribute::NoUnwind); auto RetAttrs = AttrBuilder(C); @@ -1158,7 +1149,7 @@ static const auto jl_alloc_genericmemory_unchecked_func = new JuliaFunction{ @@ -1403,7 +1394,7 @@ static const auto jl_allocgenericmemory = new JuliaFunction{ }, get_attrs_basic, }; -static const auto jlgetbuiltinfptr_func = new JuliaFunction<>{ - XSTR(jl_get_builtin_fptr), - [](LLVMContext &C) { return FunctionType::get(getPointerTy(C), - {JuliaType::get_prjlvalue_ty(C)}, false); }, - nullptr, +static const auto jldnd_func = new JuliaFunction<>{ + XSTR(jl_f_donotdelete), + [](LLVMContext &C) { + return FunctionType::get(getVoidTy(C), true); + }, + [](LLVMContext &C) { + AttrBuilder FnAttrs(C); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly() | MemoryEffects::readOnly()); + FnAttrs.addAttribute(Attribute::WillReturn); + FnAttrs.addAttribute(Attribute::NoUnwind); + return AttributeList::get(C, + AttributeSet::get(C, FnAttrs), + Attributes(C, {}), + None); + }, }; // placeholder functions @@ -1558,56 +1559,23 @@ static const auto julia_call3 = new JuliaFunction<>{ static const auto jltuple_func = new JuliaFunction<>{XSTR(jl_f_tuple), get_func_sig, get_func_attrs}; static const auto jlintrinsic_func = new JuliaFunction<>{XSTR(jl_f_intrinsic_call), get_func3_sig, get_func_attrs}; +static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction<>{XSTR(jl_new_opaque_closure_jlcall), get_func_sig, get_func_attrs}; + +static const auto mk_builtin_func_map() { + auto builtin_addrs = new DenseMap*>(); + for (int i = 0; i < jl_n_builtins; i++) { + jl_value_t *builtin = jl_builtin_instances[i]; + if (builtin) // a couple do not have instances (e.g. IntrinsicFunction) + (*builtin_addrs)[builtin] = new JuliaFunction<>{StringRef(jl_builtin_f_names[i]), get_func_sig, get_func_attrs}; + } + return builtin_addrs; +} static const auto &builtin_func_map() { - static auto builtins = new DenseMap*> { - { jl_f_is_addr, new JuliaFunction<>{XSTR(jl_f_is), get_func_sig, get_func_attrs} }, - { jl_f_typeof_addr, new JuliaFunction<>{XSTR(jl_f_typeof), get_func_sig, get_func_attrs} }, - { jl_f_sizeof_addr, new JuliaFunction<>{XSTR(jl_f_sizeof), get_func_sig, get_func_attrs} }, - { jl_f_issubtype_addr, new JuliaFunction<>{XSTR(jl_f_issubtype), get_func_sig, get_func_attrs} }, - { jl_f_isa_addr, new JuliaFunction<>{XSTR(jl_f_isa), get_func_sig, get_func_attrs} }, - { jl_f_typeassert_addr, new JuliaFunction<>{XSTR(jl_f_typeassert), get_func_sig, get_func_attrs} }, - { jl_f_ifelse_addr, new JuliaFunction<>{XSTR(jl_f_ifelse), get_func_sig, get_func_attrs} }, - { jl_f__apply_iterate_addr, new JuliaFunction<>{XSTR(jl_f__apply_iterate), get_func_sig, get_func_attrs} }, - { jl_f_invokelatest_addr, new JuliaFunction<>{XSTR(jl_f_invokelatest), get_func_sig, get_func_attrs} }, - { jl_f_invoke_in_world_addr, new JuliaFunction<>{XSTR(jl_f_invoke_in_world), get_func_sig, get_func_attrs} }, - { jl_f__call_in_world_total_addr, new JuliaFunction<>{XSTR(jl_f__call_in_world_total), get_func_sig, get_func_attrs} }, - { jl_f_throw_addr, new JuliaFunction<>{XSTR(jl_f_throw), get_func_sig, get_func_attrs} }, - { jl_f_throw_methoderror_addr, new JuliaFunction<>{XSTR(jl_f_throw_methoderror), get_func_sig, get_func_attrs} }, - { jl_f_tuple_addr, jltuple_func }, - { jl_f_svec_addr, new JuliaFunction<>{XSTR(jl_f_svec), get_func_sig, get_func_attrs} }, - { jl_f_applicable_addr, new JuliaFunction<>{XSTR(jl_f_applicable), get_func_sig, get_func_attrs} }, - { jl_f_invoke_addr, new JuliaFunction<>{XSTR(jl_f_invoke), get_func_sig, get_func_attrs} }, - { jl_f_isdefined_addr, new JuliaFunction<>{XSTR(jl_f_isdefined), get_func_sig, get_func_attrs} }, - { jl_f_getfield_addr, new JuliaFunction<>{XSTR(jl_f_getfield), get_func_sig, get_func_attrs} }, - { jl_f_setfield_addr, new JuliaFunction<>{XSTR(jl_f_setfield), get_func_sig, get_func_attrs} }, - { jl_f_swapfield_addr, new JuliaFunction<>{XSTR(jl_f_swapfield), get_func_sig, get_func_attrs} }, - { jl_f_modifyfield_addr, new JuliaFunction<>{XSTR(jl_f_modifyfield), get_func_sig, get_func_attrs} }, - { jl_f_fieldtype_addr, new JuliaFunction<>{XSTR(jl_f_fieldtype), get_func_sig, get_func_attrs} }, - { jl_f_nfields_addr, new JuliaFunction<>{XSTR(jl_f_nfields), get_func_sig, get_func_attrs} }, - { jl_f__expr_addr, new JuliaFunction<>{XSTR(jl_f__expr), get_func_sig, get_func_attrs} }, - { jl_f__typevar_addr, new JuliaFunction<>{XSTR(jl_f__typevar), get_func_sig, get_func_attrs} }, - { jl_f_memorynew_addr, new JuliaFunction<>{XSTR(jl_f_memorynew), get_func_sig, get_func_attrs} }, - { jl_f_memoryref_addr, new JuliaFunction<>{XSTR(jl_f_memoryref), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefoffset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefoffset), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefset), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefswap_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefswap), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefreplace_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefreplace), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefmodify_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefmodify), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefsetonce_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefsetonce), get_func_sig, get_func_attrs} }, - { jl_f_memoryref_isassigned_addr,new JuliaFunction<>{XSTR(jl_f_memoryref_isassigned), get_func_sig, get_func_attrs} }, - { jl_f_apply_type_addr, new JuliaFunction<>{XSTR(jl_f_apply_type), get_func_sig, get_func_attrs} }, - { jl_f_donotdelete_addr, new JuliaFunction<>{XSTR(jl_f_donotdelete), get_donotdelete_sig, get_donotdelete_func_attrs} }, - { jl_f_compilerbarrier_addr, new JuliaFunction<>{XSTR(jl_f_compilerbarrier), get_func_sig, get_func_attrs} }, - { jl_f_finalizer_addr, new JuliaFunction<>{XSTR(jl_f_finalizer), get_func_sig, get_func_attrs} }, - { jl_f__svec_ref_addr, new JuliaFunction<>{XSTR(jl_f__svec_ref), get_func_sig, get_func_attrs} }, - { jl_f_current_scope_addr, new JuliaFunction<>{XSTR(jl_f_current_scope), get_func_sig, get_func_attrs} }, - }; + static auto builtins = mk_builtin_func_map(); return *builtins; } -static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction<>{XSTR(jl_new_opaque_closure_jlcall), get_func_sig, get_func_attrs}; - static _Atomic(uint64_t) globalUniqueGeneratedNames{1}; // --- code generation --- @@ -1632,7 +1600,7 @@ static MDNode *best_tbaa(jl_tbaacache_t &tbaa_cache, jl_value_t *jt) { // note that this includes jl_isbits, although codegen should work regardless static bool jl_is_concrete_immutable(jl_value_t* t) { - return jl_is_immutable_datatype(t) && ((jl_datatype_t*)t)->isconcretetype; + return jl_may_be_immutable_datatype(t) && ((jl_datatype_t*)t)->isconcretetype; } static bool jl_is_pointerfree(jl_value_t* t) @@ -2083,7 +2051,7 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction<> *theFptr, Value static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2, Value *nullcheck1 = nullptr, Value *nullcheck2 = nullptr); static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t nargs, ArrayRef argv, bool is_promotable=false); -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt); +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, bool always_inline); static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static unsigned julia_alignment(jl_value_t *jt); @@ -2908,7 +2876,7 @@ static jl_value_t *static_apply_type(jl_codectx_t &ctx, ArrayRef arg return NULL; v[i] = jl_pinned_ref_get(args[i].constant); } - assert(v[0] == jl_builtin_apply_type); + assert(v[0] == BUILTIN(apply_type)); size_t last_age = jl_current_task->world_age; // call apply_type, but ignore errors. we know that will work in world 1. jl_current_task->world_age = 1; @@ -2978,7 +2946,7 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) if (e->head == jl_call_sym) { jl_value_t *f = static_eval(ctx, jl_exprarg(e, 0)); if (f) { - if (jl_array_dim0(e->args) == 3 && (f == jl_builtin_getfield || f == jl_builtin_getglobal)) { + if (jl_array_dim0(e->args) == 3 && (f == BUILTIN(getfield) || f == BUILTIN(getglobal))) { m = (jl_module_t*)static_eval(ctx, jl_exprarg(e, 1)); // Check the tag before evaluating `s` so that a value of random // type won't be corrupted. @@ -2997,10 +2965,11 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) } } } - else if (f==jl_builtin_tuple || f==jl_builtin_apply_type) { + else if (f==BUILTIN(tuple) || f==BUILTIN(apply_type)) { size_t i; size_t n = jl_array_dim0(e->args)-1; - if (n==0 && f==jl_builtin_tuple) return (jl_value_t*)jl_emptytuple; + if (n==0 && f==BUILTIN(tuple)) + return (jl_value_t*)jl_emptytuple; jl_value_t **v; JL_GC_PUSHARGS(v, n+1); v[0] = f; @@ -3208,11 +3177,10 @@ static void jl_temporary_root(jl_codegen_params_t &ctx, jl_value_t *val) { if (!jl_is_globally_rooted(val)) { jl_array_t *roots = ctx.temporary_roots; - for (size_t i = 0; i < jl_array_dim0(roots); i++) { - if (jl_array_ptr_ref(roots, i) == val) - return; - } + if (ctx.temporary_roots_set.find(val) != ctx.temporary_roots_set.end()) + return; jl_array_ptr_1d_push(roots, val); + ctx.temporary_roots_set.insert(val); } } static void jl_temporary_root(jl_codectx_t &ctx, jl_value_t *val) @@ -3237,7 +3205,7 @@ static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t * if (!jl_get_binding_leaf_partitions_restriction_kind(bnd, &rkp, ctx.min_world, ctx.max_world)) { return emit_globalref_runtime(ctx, bnd, mod, name); } - if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != PARTITION_KIND_BACKDATED_CONST) { + if (jl_bkind_is_real_constant(rkp.kind) || rkp.kind == PARTITION_KIND_UNDEF_CONST) { if (rkp.maybe_depwarn) { Value *bp = julia_binding_gv(ctx, bnd); ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); @@ -3632,11 +3600,11 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva static bool emit_f_opglobal(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { - bool issetglobal = f == jl_builtin_setglobal; - bool isreplaceglobal = f == jl_builtin_replaceglobal; - bool isswapglobal = f == jl_builtin_swapglobal; - bool ismodifyglobal = f == jl_builtin_modifyglobal; - bool issetglobalonce = f == jl_builtin_setglobalonce; + bool issetglobal = f == BUILTIN(setglobal); + bool isreplaceglobal = f == BUILTIN(replaceglobal); + bool isswapglobal = f == BUILTIN(swapglobal); + bool ismodifyglobal = f == BUILTIN(modifyglobal); + bool issetglobalonce = f == BUILTIN(setglobalonce); const jl_cgval_t undefval; const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; @@ -3705,11 +3673,11 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { ++EmittedOpfields; - bool issetfield = f == jl_builtin_setfield; - bool isreplacefield = f == jl_builtin_replacefield; - bool isswapfield = f == jl_builtin_swapfield; - bool ismodifyfield = f == jl_builtin_modifyfield; - bool issetfieldonce = f == jl_builtin_setfieldonce; + bool issetfield = f == BUILTIN(setfield); + bool isreplacefield = f == BUILTIN(replacefield); + bool isswapfield = f == BUILTIN(swapfield); + bool ismodifyfield = f == BUILTIN(modifyfield); + bool issetfieldonce = f == BUILTIN(setfieldonce); const jl_cgval_t undefval; const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; @@ -3827,7 +3795,7 @@ static jl_cgval_t emit_isdefinedglobal(jl_codectx_t &ctx, jl_module_t *modu, jl_ jl_binding_t *bnd = allow_import ? jl_get_binding(modu, name) : jl_get_module_binding(modu, name, 0); struct restriction_kind_pair rkp = { NULL, NULL, PARTITION_KIND_GUARD, 0 }; if (allow_import && jl_get_binding_leaf_partitions_restriction_kind(bnd, &rkp, ctx.min_world, ctx.max_world)) { - if (jl_bkind_is_some_constant(rkp.kind) && rkp.restriction) + if (jl_bkind_is_real_constant(rkp.kind)) return mark_julia_const(ctx, jl_true); if (rkp.kind == PARTITION_KIND_GLOBAL) { Value *bp = julia_binding_gv(ctx, rkp.binding_if_global); @@ -3852,11 +3820,11 @@ static jl_cgval_t emit_isdefinedglobal(jl_codectx_t &ctx, jl_module_t *modu, jl_ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { - bool issetmemory = f == jl_builtin_memoryrefset; - bool isreplacememory = f == jl_builtin_memoryrefreplace; - bool isswapmemory = f == jl_builtin_memoryrefswap; - bool ismodifymemory = f == jl_builtin_memoryrefmodify; - bool issetmemoryonce = f == jl_builtin_memoryrefsetonce; + bool issetmemory = f == BUILTIN(memoryrefset); + bool isreplacememory = f == BUILTIN(memoryrefreplace); + bool isswapmemory = f == BUILTIN(memoryrefswap); + bool ismodifymemory = f == BUILTIN(memoryrefmodify); + bool issetmemoryonce = f == BUILTIN(memoryrefsetonce); const jl_cgval_t undefval; const jl_cgval_t &ref = argv[1]; @@ -3901,8 +3869,8 @@ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; bool isboxed = layout->flags.arrayelem_isboxed; bool isunion = layout->flags.arrayelem_isunion; - bool isatomic = kind == (jl_value_t*)jl_atomic_sym; - bool needlock = isatomic && layout->size > MAX_ATOMIC_SIZE; + bool isatomic = layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked; + bool needlock = layout->flags.arrayelem_islocked; size_t elsz = layout->size; size_t al = layout->alignment; if (al > JL_HEAP_ALIGNMENT) @@ -4042,14 +4010,19 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // returns true if the call has been handled { ++EmittedBuiltinCalls; - if (f == jl_builtin_is && nargs == 2) { + if (f == BUILTIN(is) && nargs == 2) { // emit comparison test Value *ans = emit_f_is(ctx, argv[1], argv[2]); *ret = mark_julia_type(ctx, ans, false, jl_bool_type); return true; } - else if (f == jl_builtin_typeof && nargs == 1) { + else if (f == BUILTIN(ifelse) && nargs == 3) { + *ret = emit_ifelse(ctx, argv[1], argv[2], argv[3], rt); + return true; + } + + else if (f == BUILTIN(typeof) && nargs == 1) { const jl_cgval_t &p = argv[1]; if (p.constant) *ret = mark_julia_const(ctx, jl_typeof(jl_pinned_ref_get(p.constant))); @@ -4060,7 +4033,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_typeassert && nargs == 2) { + else if (f == BUILTIN(typeassert) && nargs == 2) { const jl_cgval_t &arg = argv[1]; const jl_cgval_t &ty = argv[2]; if (jl_is_type_type(jl_pinned_ref_get(ty.typ)) && !jl_has_free_typevars(jl_pinned_ref_get(ty.typ))) { @@ -4078,7 +4051,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_isa && nargs == 2) { + else if (f == BUILTIN(isa) && nargs == 2) { const jl_cgval_t &arg = argv[1]; const jl_cgval_t &ty = argv[2]; if (jl_is_type_type(jl_pinned_ref_get(ty.typ)) && !jl_has_free_typevars(jl_pinned_ref_get(ty.typ))) { @@ -4089,7 +4062,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_issubtype && nargs == 2) { + else if (f == BUILTIN(issubtype) && nargs == 2) { const jl_cgval_t &ta = argv[1]; const jl_cgval_t &tb = argv[2]; if (jl_is_type_type(jl_pinned_ref_get(ta.typ)) && !jl_has_free_typevars(jl_pinned_ref_get(ta.typ)) && @@ -4100,7 +4073,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if ((f == jl_builtin__apply_iterate && nargs == 3) && ctx.vaSlot > 0) { + else if ((f == BUILTIN(_apply_iterate) && nargs == 3) && ctx.vaSlot > 0) { // turn Core._apply_iterate(iter, f, Tuple) ==> f(Tuple...) using the jlcall calling convention if Tuple is the va allocation if (LoadInst *load = dyn_cast_or_null(argv[3].V)) { if (load->getPointerOperand() == ctx.slots[ctx.vaSlot].boxroot && ctx.argArray) { @@ -4117,7 +4090,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_tuple) { + else if (f == BUILTIN(tuple)) { if (nargs == 0) { *ret = ghostValue(ctx, jl_emptytuple_type); return true; @@ -4128,14 +4101,14 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_throw && nargs == 1) { + else if (f == BUILTIN(throw) && nargs == 1) { Value *arg1 = boxed(ctx, argv[1]); raise_exception(ctx, arg1); *ret = jl_cgval_t(); return true; } - else if (f == jl_builtin_memorynew && (nargs == 2)) { + else if (f == BUILTIN(memorynew) && (nargs == 2)) { const jl_cgval_t &memty = argv[1]; if (!memty.constant) return false; @@ -4159,7 +4132,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_memoryref && nargs == 1) { + else if (f == BUILTIN(memoryrefnew) && nargs == 1) { const jl_cgval_t &mem = argv[1]; jl_datatype_t *mty_dt = (jl_datatype_t*)jl_unwrap_unionall(jl_pinned_ref_get(mem.typ)); if (jl_is_genericmemory_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { @@ -4170,21 +4143,30 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_memoryref && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(memoryrefnew) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &ref = argv[1]; - jl_value_t *mty_dt = jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); - if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { - mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); - const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + jl_datatype_t *mty_dt = (jl_datatype_t*)jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); + if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { + mty_dt = (jl_datatype_t*)jl_field_type_concrete(mty_dt, 1); + const jl_datatype_layout_t *layout = mty_dt->layout; jl_value_t *boundscheck = nargs == 3 ? jl_pinned_ref_get(argv[3].constant) : nullptr; if (nargs == 3) - emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefnew"); *ret = emit_memoryref(ctx, ref, argv[2], boundscheck, layout); return true; } + if (jl_is_genericmemory_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { + const jl_datatype_layout_t *layout = mty_dt->layout; + jl_value_t *boundscheck = nargs == 3 ? jl_pinned_ref_get(argv[3].constant) : nullptr; + if (nargs == 3) + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefnew"); + jl_value_t *typ = jl_apply_type((jl_value_t*)jl_genericmemoryref_type, jl_svec_data(mty_dt->parameters), jl_svec_len(mty_dt->parameters)); + *ret = emit_memoryref_direct(ctx, ref, argv[2], typ, boundscheck, layout); + return true; + } } - else if (f == jl_builtin_memoryrefoffset && nargs == 1) { + else if (f == BUILTIN(memoryrefoffset) && nargs == 1) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4195,7 +4177,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_memoryrefget && nargs == 3) { + else if (f == BUILTIN(memoryrefget) && nargs == 3) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4234,7 +4216,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, order = isatomic ? jl_memory_order_unordered : jl_memory_order_notatomic; } jl_value_t *boundscheck = jl_pinned_ref_get(argv[3].constant); - emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefget"); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; Value *mem = emit_memoryref_mem(ctx, ref, layout); Value *mlen = emit_genericmemorylen(ctx, mem, jl_pinned_ref_get(ref.typ)); @@ -4256,7 +4238,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, size_t al = layout->alignment; if (al > JL_HEAP_ALIGNMENT) al = JL_HEAP_ALIGNMENT; - bool needlock = isatomic && !isboxed && elsz > MAX_ATOMIC_SIZE; + bool needlock = layout->flags.arrayelem_islocked; AtomicOrdering Order = (needlock || order <= jl_memory_order_notatomic) ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) : get_llvm_atomic_order(order); @@ -4311,16 +4293,16 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if ((f == jl_builtin_memoryrefset && nargs == 4) || - (f == jl_builtin_memoryrefswap && nargs == 4) || - (f == jl_builtin_memoryrefreplace && nargs == 6) || - (f == jl_builtin_memoryrefmodify && nargs == 5) || - (f == jl_builtin_memoryrefsetonce && nargs == 5)) { + else if ((f == BUILTIN(memoryrefset) && nargs == 4) || + (f == BUILTIN(memoryrefswap) && nargs == 4) || + (f == BUILTIN(memoryrefreplace) && nargs == 6) || + (f == BUILTIN(memoryrefmodify) && nargs == 5) || + (f == BUILTIN(memoryrefsetonce) && nargs == 5)) { return emit_f_opmemory(ctx, ret, f, argv, nargs, nullptr); } - else if (f == jl_builtin_memoryref_isassigned && nargs == 3) { + else if (f == BUILTIN(memoryref_isassigned) && nargs == 3) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(jl_pinned_ref_get(ref.typ)); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4342,7 +4324,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, *ret = jl_cgval_t(); // unreachable return true; } - bool isatomic = kind == (jl_value_t*)jl_atomic_sym; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + bool isatomic = layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked; if (!isatomic && order != jl_memory_order_notatomic && order != jl_memory_order_unspecified) { emit_atomic_error(ctx, "memoryref_isassigned: non-atomic memory cannot be accessed atomically"); *ret = jl_cgval_t(); // unreachable @@ -4358,13 +4341,12 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } jl_value_t *boundscheck = jl_pinned_ref_get(argv[3].constant); emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, fname); - const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; Value *mem = emit_memoryref_mem(ctx, ref, layout); Value *mlen = emit_genericmemorylen(ctx, mem, jl_pinned_ref_get(ref.typ)); Value *oob = bounds_check_enabled(ctx, boundscheck) ? ctx.builder.CreateIsNull(mlen) : nullptr; bool isboxed = layout->flags.arrayelem_isboxed; if (isboxed || layout->first_ptr >= 0) { - bool needlock = isatomic && !isboxed && layout->size > MAX_ATOMIC_SIZE; + bool needlock = layout->flags.arrayelem_islocked; AtomicOrdering Order = (needlock || order <= jl_memory_order_notatomic) ? (isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic) : get_llvm_atomic_order(order); @@ -4384,14 +4366,13 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ctx.builder.SetInsertPoint(passBB); } Value *elem = emit_memoryref_ptr(ctx, ref, layout); - if (needlock) { + if (!isboxed) + elem = emit_ptrgep(ctx, elem, layout->first_ptr * sizeof(void*)); + else if (needlock) // n.b. no actual lock acquire needed, as the check itself only needs to load a single pointer and check for null // elem += sizeof(lock); elem = emit_ptrgep(ctx, elem, LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT)); - } - if (!isboxed) - elem = emit_ptrgep(ctx, elem, layout->first_ptr * sizeof(void*)); - // emit this using the same type as jl_builtin_memoryrefget + // emit this using the same type as BUILTIN(memoryrefget) // so that LLVM may be able to load-load forward them and fold the result auto tbaa = isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf; jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); @@ -4420,7 +4401,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } - else if (f == jl_builtin_getfield && (nargs == 2 || nargs == 3 || nargs == 4)) { + else if (f == BUILTIN(getfield) && (nargs == 2 || nargs == 3 || nargs == 4)) { const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; enum jl_memory_order order = jl_memory_order_unspecified; @@ -4580,7 +4561,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } - else if (f == jl_builtin_getglobal && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(getglobal) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; enum jl_memory_order order = jl_memory_order_unspecified; @@ -4612,23 +4593,23 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } - else if ((f == jl_builtin_setglobal && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_swapglobal && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_replaceglobal && (nargs == 4 || nargs == 5 || nargs == 6)) || - (f == jl_builtin_modifyglobal && (nargs == 4 || nargs == 5)) || - (f == jl_builtin_setglobalonce && (nargs == 3 || nargs == 4 || nargs == 5))) { + else if ((f == BUILTIN(setglobal) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(swapglobal) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(replaceglobal) && (nargs == 4 || nargs == 5 || nargs == 6)) || + (f == BUILTIN(modifyglobal) && (nargs == 4 || nargs == 5)) || + (f == BUILTIN(setglobalonce) && (nargs == 3 || nargs == 4 || nargs == 5))) { return emit_f_opglobal(ctx, ret, f, argv, nargs, nullptr); } - else if ((f == jl_builtin_setfield && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_swapfield && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_replacefield && (nargs == 4 || nargs == 5 || nargs == 6)) || - (f == jl_builtin_modifyfield && (nargs == 4 || nargs == 5)) || - (f == jl_builtin_setfieldonce && (nargs == 3 || nargs == 4 || nargs == 5))) { + else if ((f == BUILTIN(setfield) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(swapfield) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(replacefield) && (nargs == 4 || nargs == 5 || nargs == 6)) || + (f == BUILTIN(modifyfield) && (nargs == 4 || nargs == 5)) || + (f == BUILTIN(setfieldonce) && (nargs == 3 || nargs == 4 || nargs == 5))) { return emit_f_opfield(ctx, ret, f, argv, nargs, nullptr); } - else if (f == jl_builtin_nfields && nargs == 1) { + else if (f == BUILTIN(nfields) && nargs == 1) { const jl_cgval_t &obj = argv[1]; if (ctx.vaSlot > 0) { // optimize VA tuple @@ -4660,7 +4641,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_fieldtype && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(fieldtype) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &typ = argv[1]; const jl_cgval_t &fld = argv[2]; if ((jl_is_type_type(jl_pinned_ref_get(typ.typ)) && jl_is_concrete_type(jl_tparam0(jl_pinned_ref_get(typ.typ)))) || @@ -4685,7 +4666,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_sizeof && nargs == 1) { + else if (f == BUILTIN(sizeof) && nargs == 1) { const jl_cgval_t &obj = argv[1]; jl_datatype_t *sty = (jl_datatype_t*)jl_unwrap_unionall(jl_pinned_ref_get(obj.typ)); assert(jl_string_type->name->mutabl); @@ -4730,7 +4711,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_apply_type && nargs > 0) { + else if (f == BUILTIN(apply_type) && nargs > 0) { if (jl_is_method(ctx.linfo->def.method)) { // don't bother codegen constant-folding for toplevel. jl_value_t *ty = static_apply_type(ctx, argv, nargs + 1); @@ -4744,7 +4725,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_isdefinedglobal && (nargs == 2 || nargs == 3 || nargs == 4)) { + else if (f == BUILTIN(isdefinedglobal) && (nargs == 2 || nargs == 3 || nargs == 4)) { const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; bool allow_import = true; @@ -4780,7 +4761,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_isdefined && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(isdefined) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; jl_datatype_t *stt = (jl_datatype_t*)jl_pinned_ref_get(obj.typ); @@ -4896,7 +4877,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_current_scope && (nargs == 0)) { + else if (f == BUILTIN(current_scope) && (nargs == 0)) { jl_aliasinfo_t scope_ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); Instruction *v = scope_ai.decorateInst( ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, get_scope_field(ctx), ctx.types().alignof_ptr)); @@ -4904,18 +4885,13 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_donotdelete) { + else if (f == BUILTIN(donotdelete)) { // For now we emit this as a vararg call to the builtin // (which doesn't look at the arguments). In the future, // this should be an LLVM builtin. - auto it = builtin_func_map().find(jl_f_donotdelete_addr); - if (it == builtin_func_map().end()) { - return false; - } - *ret = mark_julia_const(ctx, jl_nothing); FunctionType *Fty = FunctionType::get(getVoidTy(ctx.builder.getContext()), true); - Function *dnd = prepare_call(it->second); + Function *dnd = prepare_call(jldnd_func); SmallVector call_args; for (size_t i = 1; i <= nargs; ++i) { @@ -4934,7 +4910,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_compilerbarrier && (nargs == 2)) { + else if (f == BUILTIN(compilerbarrier) && (nargs == 2)) { emit_typecheck(ctx, argv[1], (jl_value_t*)jl_symbol_type, "compilerbarrier"); *ret = argv[2]; return true; @@ -5029,8 +5005,10 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos } jl_value_t *jt = jl_nth_slot_type(specTypes, i); jl_cgval_t arg = update_julia_type(ctx, argv[i], jt); - if (arg.typ == jl_bottom_type) + if (arg.typ == jl_bottom_type) { + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Argument type mismatch in Expr(:invoke)"); return jl_cgval_t(); + } if (is_uniquerep_Type(jt)) { continue; } @@ -5067,7 +5045,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos Value *val = emit_unbox(ctx, et, arg, jt); if (!val) { // There was a type mismatch of some sort - exit early - CreateTrap(ctx.builder); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Argument type mismatch in Expr(:invoke)"); return jl_cgval_t(); } argvals[idx] = val; @@ -5091,7 +5069,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos break; case jl_returninfo_t::SRet: assert(result); - retval = mark_julia_slot(result, jlretty, NULL, ctx.tbaa().tbaa_gcframe, load_gc_roots(ctx, return_roots, returninfo.return_roots)); + retval = mark_julia_slot(result, jlretty, NULL, ctx.tbaa().tbaa_gcframe, load_gc_roots(ctx, return_roots, returninfo.return_roots, ctx.tbaa().tbaa_gcframe)); break; case jl_returninfo_t::Union: { Value *box = ctx.builder.CreateExtractValue(call, 0); @@ -5131,10 +5109,7 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, bool is_opaque_clos namep += cast(TheCallee)->getName(); GlobalVariable *GV = cast_or_null(jl_Module->getNamedValue(namep)); if (GV == nullptr) { - GV = new GlobalVariable(*jl_Module, TheCallee->getType(), false, - GlobalVariable::ExternalLinkage, - Constant::getNullValue(TheCallee->getType()), - namep); + GV = new GlobalVariable(*jl_Module, TheCallee->getType(), false, GlobalVariable::ExternalLinkage, nullptr, namep); ctx.emission_context.external_fns[std::make_tuple(fromexternal, true)] = GV; } jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); @@ -5181,10 +5156,7 @@ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty GlobalVariable *GV = cast_or_null(jl_Module->getNamedValue(namep)); Type *pfunc = PointerType::getUnqual(ctx.builder.getContext()); if (GV == nullptr) { - GV = new GlobalVariable(*jl_Module, pfunc, false, - GlobalVariable::ExternalLinkage, - Constant::getNullValue(pfunc), - namep); + GV = new GlobalVariable(*jl_Module, pfunc, false, GlobalVariable::ExternalLinkage, nullptr, namep); ctx.emission_context.external_fns[std::make_tuple(fromexternal, false)] = GV; } jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); @@ -5213,10 +5185,10 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) if (argv[i].typ == jl_bottom_type) return jl_cgval_t(); } - return emit_invoke(ctx, lival, argv, nargs, rt); + return emit_invoke(ctx, lival, argv, nargs, rt, false); } -static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt) +static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt, bool always_inline) { ++EmittedInvokes; bool handled = false; @@ -5227,10 +5199,12 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR if (jl_is_method_instance(jl_pinned_ref_get(lival.constant))) { mi = (jl_method_instance_t*)jl_pinned_ref_get(lival.constant); } - else { + else if (jl_is_code_instance(jl_pinned_ref_get(lival.constant))) { ci = jl_pinned_ref_get(lival.constant); - assert(jl_is_code_instance(ci)); mi = jl_get_ci_mi((jl_code_instance_t*)ci); + } else { + emit_error(ctx, "(Internal ERROR - IR Validity): Invoke target is not a method instance or code instance"); + return jl_cgval_t(); } assert(jl_is_method_instance(mi)); if (mi == ctx.linfo) { @@ -5272,44 +5246,52 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR std::string name; StringRef protoname; bool need_to_emit = true; - bool cache_valid = ctx.use_cache || ctx.external_linkage; + bool cache_valid = (ctx.use_cache || ctx.external_linkage); bool external = false; // Check if we already queued this up auto it = ctx.call_targets.find(codeinst); - if (need_to_emit && it != ctx.call_targets.end()) { + if (it != ctx.call_targets.end()) { assert(it->second.specsig == specsig); protoname = it->second.decl->getName(); - need_to_emit = cache_valid = false; + if (always_inline) + it->second.private_linkage = true; + else + it->second.external_linkage = true; } - - // Check if it is already compiled (either JIT or externally) - if (need_to_emit && cache_valid) { - // optimization: emit the correct name immediately, if we know it + // Check if it is already compiled (either JIT or externally), and if so, re-use that name if possible + // This is just an optimization to emit the correct name immediately, if we know it, since the JIT and AOT code will be able to do this later also + if (cache_valid) { // TODO: use `emitted` map here too to try to consolidate names? uint8_t specsigflags; jl_callptr_t invoke; void *fptr; jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) { - protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); if (ctx.external_linkage) { // TODO: Add !specsig support to aotcompile.cpp // Check that the codeinst is containing native code if (specsig && (specsigflags & 0b100)) { - external = true; + external = !always_inline; need_to_emit = false; } } else { // ctx.use_cache need_to_emit = false; } + if (!need_to_emit && protoname.empty()) + protoname = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); } } - if (need_to_emit) { + if (it != ctx.call_targets.end()) + need_to_emit = false; + else if (always_inline) + need_to_emit = true; + if (protoname.empty()) { raw_string_ostream(name) << (specsig ? "j_" : "j1_") << name_from_method_instance(mi) << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); protoname = StringRef(name); } + jl_returninfo_t::CallingConv cc = jl_returninfo_t::CallingConv::Boxed; unsigned return_roots = 0; if (specsig) @@ -5318,7 +5300,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, external ? codeinst : nullptr, argv, nargs, rt); if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); - ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig}; + ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig, !always_inline, always_inline}; } } } @@ -5332,7 +5314,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR } if (result.typ == jl_bottom_type) { #ifndef JL_NDEBUG - emit_error(ctx, "(Internal Error - IR Validity): Returned from function we expected not to."); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Returned from function we expected not to."); #endif CreateTrap(ctx.builder); } @@ -5357,22 +5339,22 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ if (f.constant) { jl_cgval_t ret; auto it = builtin_func_map().end(); - if (f.constant == jl_builtin_modifyfield) { - if (emit_f_opfield(ctx, &ret, jl_builtin_modifyfield, argv, nargs - 1, &lival)) + if (f.constant == BUILTIN(modifyfield)) { + if (emit_f_opfield(ctx, &ret, BUILTIN(modifyfield), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(jl_f_modifyfield_addr); + it = builtin_func_map().find(jl_pinned_ref_get(f.constant)); assert(it != builtin_func_map().end()); } - else if (f.constant == jl_builtin_modifyglobal) { - if (emit_f_opglobal(ctx, &ret, jl_builtin_modifyglobal, argv, nargs - 1, &lival)) + else if (f.constant == BUILTIN(modifyglobal)) { + if (emit_f_opglobal(ctx, &ret, BUILTIN(modifyglobal), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(jl_f_modifyglobal_addr); + it = builtin_func_map().find(jl_pinned_ref_get(f.constant)); assert(it != builtin_func_map().end()); } - else if (f.constant == jl_builtin_memoryrefmodify) { - if (emit_f_opmemory(ctx, &ret, jl_builtin_memoryrefmodify, argv, nargs - 1, &lival)) + else if (f.constant == BUILTIN(memoryrefmodify)) { + if (emit_f_opmemory(ctx, &ret, BUILTIN(memoryrefmodify), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(jl_f_memoryrefmodify_addr); + it = builtin_func_map().find(jl_pinned_ref_get(f.constant)); assert(it != builtin_func_map().end()); } else if (jl_typetagis(jl_pinned_ref_get(f.constant), jl_intrinsic_type)) { @@ -5432,15 +5414,15 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo return jl_cgval_t(); } + // a couple intrinsics (really just llvmcall, though partly cglobal too) + // have non-standard (aka invalid) evaluation semantics, so we must handle these first if (f.constant && jl_typetagis(jl_pinned_ref_get(f.constant), jl_intrinsic_type)) { JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(jl_pinned_ref_get(f.constant)); return emit_intrinsic(ctx, fi, args, nargs - 1); } size_t n_generic_args = nargs; - SmallVector argv(n_generic_args); - argv[0] = f; for (size_t i = 1; i < nargs; ++i) { argv[i] = emit_expr(ctx, args[i]); @@ -5448,37 +5430,23 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo return jl_cgval_t(); // anything past here is unreachable } - if (jl_subtype(jl_pinned_ref_get(f.typ), (jl_value_t*)jl_builtin_type)) { - if (f.constant) { - if (f.constant == jl_builtin_ifelse && nargs == 4) - return emit_ifelse(ctx, argv[1], argv[2], argv[3], rt); - jl_cgval_t result; - bool handled = emit_builtin_call(ctx, &result, jl_pinned_ref_get(f.constant), argv, nargs - 1, rt, ex, is_promotable); - if (handled) - return result; - jl_fptr_args_t builtin_fptr = jl_get_builtin_fptr((jl_datatype_t*)jl_typeof(jl_pinned_ref_get(f.constant))); - // special case for some known builtin not handled by emit_builtin_call - auto it = builtin_func_map().find(builtin_fptr); - if (it != builtin_func_map().end()) { - Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); - setName(ctx.emission_context, ret, it->second->name + "_ret"); - return mark_julia_type(ctx, ret, true, rt); - } - } - Value *fptr; - JuliaFunction<> *cc; - if (f.typ == (jl_value_t*)jl_intrinsic_type) { - fptr = prepare_call(jlintrinsic_func); - cc = julia_call3; - } - else { - fptr = ctx.builder.CreateCall(prepare_call(jlgetbuiltinfptr_func), {emit_typeof(ctx, f)}); - cc = julia_call; - } - Value *ret = emit_jlcall(ctx, fptr, nullptr, argv, nargs, cc); + if (f.typ == (jl_value_t*)jl_intrinsic_type) { + Value *ret = emit_jlcall(ctx, prepare_call(jlintrinsic_func), nullptr, argv, nargs, julia_call3); setName(ctx.emission_context, ret, "Builtin_ret"); return mark_julia_type(ctx, ret, true, rt); } + else if (f.constant && jl_isa(jl_pinned_ref_get(f.constant), (jl_value_t*)jl_builtin_type)) { + jl_cgval_t result; + bool handled = emit_builtin_call(ctx, &result, jl_pinned_ref_get(f.constant), argv, nargs - 1, rt, ex, is_promotable); + if (handled) + return result; + auto it = builtin_func_map().find(jl_pinned_ref_get(f.constant)); + if (it != builtin_func_map().end()) { + Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); + setName(ctx.emission_context, ret, it->second->name + "_ret"); + return mark_julia_type(ctx, ret, true, rt); + } + } // handle calling an OpaqueClosure if (jl_is_concrete_type(jl_pinned_ref_get(f.typ)) && jl_subtype(jl_pinned_ref_get(f.typ), (jl_value_t*)jl_opaque_closure_type)) { @@ -5650,7 +5618,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va T_prjlvalue = AT->getElementType(); } assert(T_prjlvalue == ctx.types().T_prjlvalue); - v.inline_roots = load_gc_roots(ctx, varslot, nroots, vi.isVolatile); + v.inline_roots = load_gc_roots(ctx, varslot, nroots, ctx.tbaa().tbaa_gcframe, vi.isVolatile); } if (vi.usedUndef) { assert(vi.defFlag); @@ -5698,7 +5666,7 @@ static jl_cgval_t emit_local(jl_codectx_t &ctx, jl_value_t *slotload) if (sym == jl_unused_sym) { // This shouldn't happen in well-formed input, but let's be robust, // since we otherwise cause undefined behavior here. - emit_error(ctx, "(INTERNAL ERROR): Tried to use `#undef#` argument."); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Tried to use `#undef#` argument."); return jl_cgval_t(); } return emit_varinfo(ctx, vi, sym); @@ -5832,7 +5800,11 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) unsigned nb = jl_datatype_size(phiType); dest = emit_static_alloca(ctx, nb, align); phi = cast(dest->clone()); +#if JL_LLVM_VERSION >= 200000 + phi->insertBefore(dest->getIterator()); +#else phi->insertBefore(dest); +#endif ctx.builder.CreateMemCpy(phi, align, dest, align, nb, false); ctx.builder.CreateLifetimeEnd(dest); } @@ -6267,7 +6239,7 @@ static std::pair get_oc_function(jl_codectx_t &ctx, jl_met } if (need_to_emit) { - ctx.call_targets[ci] = {cc, return_roots, specsig ? specF : F, specsig ? F : nullptr, specsig}; + ctx.call_targets[ci] = {cc, return_roots, specsig ? specF : F, specsig ? F : nullptr, specsig, true, false}; } JL_GC_POP(); @@ -6402,8 +6374,8 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ } else if (head == jl_cfunction_sym) { assert(nargs == 5); - jl_cgval_t fexpr_rt = emit_expr(ctx, args[1]); - return emit_cfunction(ctx, args[0], fexpr_rt, args[2], (jl_svec_t*)args[3]); + jl_cgval_t fexpr_val = emit_expr(ctx, args[1]); + return emit_cfunction(ctx, args[0], fexpr_val, args[2], (jl_svec_t*)args[3]); } else if (head == jl_assign_sym) { assert(nargs == 2); @@ -6544,7 +6516,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ if (source.constant == NULL) { // For now, we require non-constant source to be handled by using // eval. This should probably be a verifier error and an abort here. - emit_error(ctx, "(internal error) invalid IR: opaque closure source must be constant"); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): opaque closure source must be constant"); return jl_cgval_t(); } bool can_optimize = jl_pinned_ref_get(argt.constant) != NULL && jl_pinned_ref_get(lb.constant) != NULL && jl_pinned_ref_get(ub.constant) != NULL && @@ -6790,6 +6762,74 @@ Function *get_or_emit_fptr1(StringRef preal_decl, Module *M) return cast(M->getOrInsertFunction(preal_decl, get_func_sig(M->getContext()), get_func_attrs(M->getContext())).getCallee()); } +static Function *emit_modifyhelper(jl_codectx_t &ctx2, const jl_cgval_t &op, const jl_cgval_t &modifyop, jl_value_t *jltype, Type *elty, jl_cgval_t rhs, const Twine &fname, bool gcstack_arg) +{ + Module *M = ctx2.f->getParent(); + jl_codectx_t ctx(M->getContext(), ctx2.emission_context, ctx2.min_world, ctx2.max_world); + SmallVector ArgTy; + ArgTy.push_back(elty); + if (rhs.V) + ArgTy.push_back(rhs.V->getType()); + if (rhs.Vboxed) + ArgTy.push_back(rhs.Vboxed->getType()); + if (rhs.TIndex) + ArgTy.push_back(rhs.TIndex->getType()); + for (auto &root : rhs.inline_roots) + ArgTy.push_back(root->getType()); + if (gcstack_arg) + ArgTy.push_back(ctx.builder.getPtrTy()); + FunctionType *FT = FunctionType::get(elty, ArgTy, false); + Function *w = Function::Create(FT, GlobalVariable::PrivateLinkage, "", M); + jl_init_function(w, ctx.emission_context.TargetTriple); + w->addFnAttr(Attribute::AlwaysInline); + w->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + Function::arg_iterator AI = w->arg_begin(); + Argument *A = &*AI++; + // rebuild a copy of rhs from the arguments + if (rhs.V) + rhs.V = &*AI++; + if (rhs.Vboxed) + rhs.Vboxed = &*AI++; + if (rhs.TIndex) + rhs.TIndex = &*AI++; + for (size_t i = 0; i < rhs.inline_roots.size(); i++) + rhs.inline_roots[i] = &*AI++; + rhs.promotion_point = nullptr; + rhs.promotion_ssa = -1; + if (gcstack_arg) { + AttrBuilder param(ctx.builder.getContext()); + if (ctx.emission_context.use_swiftcc) { + w->setCallingConv(CallingConv::Swift); + param.addAttribute(Attribute::SwiftSelf); + } + param.addAttribute("gcstack"); + param.addAttribute(Attribute::NonNull); + Argument *gcstackarg = &*AI++; + gcstackarg->addAttrs(param); + gcstackarg->setName("pgcstack_arg"); + ctx.pgcstack = gcstackarg; + } + assert(AI == w->arg_end()); + ctx.f = w; + ctx.rettype = jl_pinned_ref_create(jl_value_t, jltype); + BasicBlock *b0 = BasicBlock::Create(ctx.builder.getContext(), "top", w); + ctx.builder.SetInsertPoint(b0); + DebugLoc noDbg; + ctx.builder.SetCurrentDebugLocation(noDbg); + allocate_gc_frame(ctx, b0); + const jl_cgval_t argv[3] = { op, mark_julia_type(ctx, A, false, jltype), rhs }; + jl_cgval_t ret = emit_invoke(ctx, modifyop, argv, 3, (jl_value_t*)jl_any_type, true); + emit_typecheck(ctx, ret, jltype, fname); + ret = update_julia_type(ctx, ret, jltype); + ctx.builder.CreateRet(emit_unbox(ctx, elty, ret, jltype)); + if (ctx.topalloca->use_empty()) { + ctx.topalloca->eraseFromParent(); + ctx.topalloca = nullptr; + } + return w; +} + + Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Value *theFunc, Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT { ++EmittedToJLInvokes; @@ -6908,7 +6948,7 @@ static void emit_specsig_to_specsig( auto tracked = CountTrackedPointers(et); SmallVector roots; if (tracked.count && !tracked.all) { - roots = load_gc_roots(ctx, &*AI, tracked.count); + roots = load_gc_roots(ctx, &*AI, tracked.count, ctx.tbaa().tbaa_const); ++AI; } myargs[i] = mark_julia_slot(arg_v, jt, NULL, ctx.tbaa().tbaa_const, roots); @@ -7046,7 +7086,7 @@ static void emit_specsig_to_specsig( emit_specsig_to_specsig(gf_thunk, returninfo.cc, returninfo.return_roots, calltype, rettype, is_for_opaque_closure, nargs, params, target, targetsig, targetrt, targetspec, rettype_const); } -std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *target, bool target_specsig) +std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_code_instance_t *codeinst, Value *target, bool target_specsig) { // this builds a method that calls a method with the same arguments but a different specsig // build a specsig -> specsig converter thunk @@ -7054,36 +7094,35 @@ std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_value_ // build a args1 -> specsig converter thunk (gen_invoke_wrapper) // build a args1 -> args1 converter thunk (to add typeassert on result) bool needsparams = false; - bool is_opaque_closure = false; + bool target_is_opaque_closure = false; jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - std::string gf_thunk_name = get_function_name(specsig, needsparams, name_from_method_instance(mi), params.TargetTriple); + std::string gf_thunk_name = get_function_name(from_abi.specsig, needsparams, name_from_method_instance(mi), params.TargetTriple); gf_thunk_name += "_gfthunk"; if (target_specsig) { jl_value_t *abi = get_ci_abi(codeinst); - jl_returninfo_t targetspec = get_specsig_function(params, M, target, "", abi, codeinst->rettype, is_opaque_closure); - if (specsig) - emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, + jl_returninfo_t targetspec = get_specsig_function(params, M, target, "", abi, codeinst->rettype, target_is_opaque_closure); + if (from_abi.specsig) + emit_specsig_to_specsig(M, gf_thunk_name, jl_pinned_ref_get(from_abi.sigt), jl_pinned_ref_get(from_abi.rt), from_abi.is_opaque_closure, from_abi.nargs, params, target, mi->specTypes, codeinst->rettype, &targetspec, nullptr); else - gen_invoke_wrapper(mi, abi, codeinst->rettype, declrt, targetspec, nargs, -1, is_opaque_closure, gf_thunk_name, M, params); + gen_invoke_wrapper(mi, abi, codeinst->rettype, jl_pinned_ref_get(from_abi.rt), targetspec, from_abi.nargs, -1, from_abi.is_opaque_closure, gf_thunk_name, M, params); } else { - if (specsig) - emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, + if (from_abi.specsig) + emit_specsig_to_specsig(M, gf_thunk_name, jl_pinned_ref_get(from_abi.sigt), jl_pinned_ref_get(from_abi.rt), from_abi.is_opaque_closure, from_abi.nargs, params, target, mi->specTypes, codeinst->rettype, nullptr, nullptr); else - emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, declrt, codeinst->rettype, params); + emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, jl_pinned_ref_get(from_abi.rt), codeinst->rettype, params); } return gf_thunk_name; } -std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *invoke) +std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_code_instance_t *codeinst, Value *invoke) { // this builds a method that calls a method with the same arguments but a different specsig // build a specsig -> args1 (apply_generic) or invoke (emit_tojlinvoke) call // build a args1 -> args1 call (emit_fptr1_wrapper) // build a args1 -> invoke call (emit_tojlinvoke) - bool is_opaque_closure = false; Value *target; if (!codeinst) target = prepare_call_in(M, jlapplygeneric_func); @@ -7095,33 +7134,40 @@ std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_value else raw_string_ostream(gf_thunk_name) << "j_"; raw_string_ostream(gf_thunk_name) << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1) << "_gfthunk"; - if (specsig) - emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, - target, sigt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, nullptr, nullptr); + if (from_abi.specsig) + emit_specsig_to_specsig(M, gf_thunk_name, jl_pinned_ref_get(from_abi.sigt), jl_pinned_ref_get(from_abi.rt), from_abi.is_opaque_closure, from_abi.nargs, params, + target, jl_pinned_ref_get(from_abi.sigt), codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, nullptr, nullptr); else - emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, declrt, codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, params); + emit_fptr1_wrapper(M, gf_thunk_name, target, nullptr, jl_pinned_ref_get(from_abi.rt), codeinst ? codeinst->rettype : (jl_value_t*)jl_any_type, params); return gf_thunk_name; } -std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_value_t *rettype_const) +std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_value_t *rettype_const) { - bool is_opaque_closure = false; std::string gf_thunk_name; raw_string_ostream(gf_thunk_name) << "jconst_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); - if (specsig) { - emit_specsig_to_specsig(M, gf_thunk_name, sigt, declrt, is_opaque_closure, nargs, params, - nullptr, sigt, jl_typeof(rettype_const), nullptr, rettype_const); + if (from_abi.specsig) { + emit_specsig_to_specsig(M, gf_thunk_name, jl_pinned_ref_get(from_abi.sigt), jl_pinned_ref_get(from_abi.rt), from_abi.is_opaque_closure, from_abi.nargs, params, + nullptr, jl_pinned_ref_get(from_abi.sigt), jl_typeof(rettype_const), nullptr, rettype_const); } else { - emit_fptr1_wrapper(M, gf_thunk_name, nullptr, rettype_const, declrt, jl_typeof(rettype_const), params); + emit_fptr1_wrapper(M, gf_thunk_name, nullptr, rettype_const, jl_pinned_ref_get(from_abi.rt), jl_typeof(rettype_const), params); } return gf_thunk_name; } std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, bool specsig, jl_code_instance_t *codeinst) { - jl_value_t *abi = get_ci_abi(codeinst); - return emit_abi_constreturn(M, params, codeinst->rettype, abi, specsig ? jl_nparams(abi) : 0, specsig, codeinst->rettype_const); + jl_value_t *sigt = get_ci_abi(codeinst); + jl_value_t *rt = codeinst->rettype; + + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; + + size_t nargs = specsig ? jl_nparams(sigt) : 0; + jl_abi_t abi = {jl_pinned_ref_create(jl_value_t, sigt), jl_pinned_ref_create(jl_value_t, rt), nargs, specsig, is_opaque_closure}; + + return emit_abi_constreturn(M, params, abi, codeinst->rettype_const); } // release jl_world_counter @@ -7148,26 +7194,23 @@ static jl_cgval_t emit_abi_call(jl_codectx_t &ctx, jl_value_t *declrt, jl_value_ Type *T_size = ctx.types().T_size; Constant *Vnull = ConstantPointerNull::get(T_ptr); Module *M = jl_Module; - GlobalVariable *theFptr = new GlobalVariable(*M, T_ptr, false, - GlobalVariable::PrivateLinkage, - Vnull); - GlobalVariable *last_world_p = new GlobalVariable(*M, T_size, false, - GlobalVariable::PrivateLinkage, - ConstantInt::get(T_size, 0)); - ArrayType *T_cfuncdata = ArrayType::get(T_ptr, 6); + ArrayType *T_cfuncdata = ArrayType::get(T_ptr, 8); size_t flags = specsig; GlobalVariable *cfuncdata = new GlobalVariable(*M, T_cfuncdata, false, GlobalVariable::PrivateLinkage, ConstantArray::get(T_cfuncdata, { + Vnull, + Vnull, Vnull, Vnull, Vnull, literal_pointer_val_slot(ctx.emission_context, M, declrt), literal_pointer_val_slot(ctx.emission_context, M, sigt), literal_static_pointer_val((void*)flags, T_ptr)})); + Value *last_world_p = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_size, cfuncdata, 1); LoadInst *last_world_v = ctx.builder.CreateAlignedLoad(T_size, last_world_p, ctx.types().alignof_ptr); last_world_v->setOrdering(AtomicOrdering::Acquire); - LoadInst *callee = ctx.builder.CreateAlignedLoad(T_ptr, theFptr, ctx.types().alignof_ptr); + LoadInst *callee = ctx.builder.CreateAlignedLoad(T_ptr, cfuncdata, ctx.types().alignof_ptr); callee->setOrdering(AtomicOrdering::Monotonic); LoadInst *world_v = ctx.builder.CreateAlignedLoad(ctx.types().T_size, prepare_global_in(M, jlgetworld_global), ctx.types().alignof_ptr); @@ -7176,15 +7219,12 @@ static jl_cgval_t emit_abi_call(jl_codectx_t &ctx, jl_value_t *declrt, jl_value_ Value *age_not_ok = ctx.builder.CreateICmpNE(last_world_v, world_v); Value *target = emit_guarded_test(ctx, age_not_ok, callee, [&] { Function *getcaller = prepare_call(jlgetabiconverter_func); - CallInst *cw = ctx.builder.CreateCall(getcaller, { - get_current_task(ctx), - theFptr, - last_world_p, - cfuncdata}); + CallInst *cw = ctx.builder.CreateCall(getcaller, {get_current_task(ctx), cfuncdata}); cw->setAttributes(getcaller->getAttributes()); return cw; }); - ctx.emission_context.cfuncs.push_back({jl_pinned_ref_create(jl_value_t, declrt), jl_pinned_ref_create(jl_value_t, sigt), nargs, specsig, theFptr, cfuncdata}); + jl_abi_t cfuncabi = {jl_pinned_ref_create(jl_value_t, sigt), jl_pinned_ref_create(jl_value_t, declrt), nargs, specsig, is_opaque_closure}; + ctx.emission_context.cfuncs.push_back({cfuncabi, cfuncdata}); if (specsig) { // TODO: could we force this to guarantee passing a box for `f` here (since we // know we had it here) and on the receiver end (emit_abi_converter / @@ -7369,8 +7409,8 @@ static Function *gen_cfun_wrapper( inputarg = mark_julia_type(ctx, val, false, jargty); } } - else if (static_at || (!jl_is_typevar(jargty) && !jl_is_immutable_datatype(jargty))) { - // must be a jl_value_t* (because it's mutable or contains gc roots) + else if (static_at || (!jl_is_typevar(jargty) && (!jl_is_datatype(jargty) || jl_is_abstracttype(jargty) || jl_is_mutable_datatype(jargty)))) { + // must be a jl_value_t* (because it is mutable or abstract) inputarg = mark_julia_type(ctx, maybe_decay_untracked(ctx, val), true, jargty_proper); } else { @@ -7384,31 +7424,36 @@ static Function *gen_cfun_wrapper( emit_ptrgep(ctx, nestPtr, jl_array_nrows(*closure_types) * ctx.types().sizeof_ptr), Align(sizeof(void*))); BasicBlock *boxedBB = BasicBlock::Create(ctx.builder.getContext(), "isboxed", cw); - BasicBlock *loadBB = BasicBlock::Create(ctx.builder.getContext(), "need-load", cw); + BasicBlock *notanyBB = BasicBlock::Create(ctx.builder.getContext(), "not-any", cw); BasicBlock *unboxedBB = BasicBlock::Create(ctx.builder.getContext(), "maybe-unboxed", cw); BasicBlock *isanyBB = BasicBlock::Create(ctx.builder.getContext(), "any", cw); BasicBlock *afterBB = BasicBlock::Create(ctx.builder.getContext(), "after", cw); - Value *isrtboxed = ctx.builder.CreateIsNull(val); // XXX: this is the wrong condition and should be inspecting runtime_dt instead - ctx.builder.CreateCondBr(isrtboxed, boxedBB, loadBB); - ctx.builder.SetInsertPoint(boxedBB); - Value *p1 = val; - p1 = track_pjlvalue(ctx, p1); - ctx.builder.CreateBr(afterBB); - ctx.builder.SetInsertPoint(loadBB); Value *isrtany = ctx.builder.CreateICmpEQ( - literal_pointer_val(ctx, (jl_value_t*)jl_any_type), val); - ctx.builder.CreateCondBr(isrtany, isanyBB, unboxedBB); + track_pjlvalue(ctx,literal_pointer_val(ctx, (jl_value_t*)jl_any_type)), runtime_dt); + ctx.builder.CreateCondBr(isrtany, isanyBB, notanyBB); ctx.builder.SetInsertPoint(isanyBB); - Value *p2 = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, val, Align(sizeof(void*))); + Value *p1 = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, val, Align(sizeof(void*))); + ctx.builder.CreateBr(afterBB); + isanyBB = ctx.builder.GetInsertBlock(); // could have changed + ctx.builder.SetInsertPoint(notanyBB); + jl_cgval_t runtime_dt_val = mark_julia_type(ctx, runtime_dt, true, jl_any_type); + Value *isrtboxed = // (!jl_is_datatype(runtime_dt) || !jl_is_concrete_datatype(runtime_dt) || jl_is_mutable_datatype(runtime_dt)) + emit_guarded_test(ctx, emit_exactly_isa(ctx, runtime_dt_val, jl_datatype_type), true, [&] { + return ctx.builder.CreateOr(ctx.builder.CreateNot(emit_isconcrete(ctx, runtime_dt)), emit_datatype_mutabl(ctx, runtime_dt)); + }); + ctx.builder.CreateCondBr(isrtboxed, boxedBB, unboxedBB); + ctx.builder.SetInsertPoint(boxedBB); + Value *p2 = track_pjlvalue(ctx, val); ctx.builder.CreateBr(afterBB); + boxedBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.SetInsertPoint(unboxedBB); Value *p3 = emit_new_bits(ctx, runtime_dt, val); unboxedBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.CreateBr(afterBB); ctx.builder.SetInsertPoint(afterBB); PHINode *p = ctx.builder.CreatePHI(ctx.types().T_prjlvalue, 3); - p->addIncoming(p1, boxedBB); - p->addIncoming(p2, isanyBB); + p->addIncoming(p1, isanyBB); + p->addIncoming(p2, boxedBB); p->addIncoming(p3, unboxedBB); inputarg = mark_julia_type(ctx, p, true, jargty_proper); } @@ -7539,16 +7584,11 @@ static const char *derive_sigt_name(jl_value_t *jargty) jl_datatype_t *dt = (jl_datatype_t*)jl_argument_datatype(jargty); if ((jl_value_t*)dt == jl_nothing) return NULL; - jl_sym_t *name = dt->name->name; - // if we have a kwcall, use that as the name anyways - jl_methtable_t *mt = dt->name->mt; - if (mt == jl_type_type_mt || mt == jl_nonfunction_mt || mt == NULL) { - // our value for `name` from MethodTable is not good, try to come up with something better - if (jl_is_type_type((jl_value_t*)dt)) { - dt = (jl_datatype_t*)jl_argument_datatype(jl_tparam0(dt)); - if ((jl_value_t*)dt != jl_nothing) { - name = dt->name->name; - } + jl_sym_t *name = dt->name->singletonname; + if (jl_is_type_type((jl_value_t*)dt)) { + dt = (jl_datatype_t*)jl_argument_datatype(jl_tparam0(dt)); + if ((jl_value_t*)dt != jl_nothing) { + name = dt->name->singletonname; } } return jl_symbol_name(name); @@ -7557,7 +7597,7 @@ static const char *derive_sigt_name(jl_value_t *jargty) // Get the LLVM Function* for the C-callable entry point for a certain function // and argument types. // here argt does not include the leading function type argument -static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, const jl_cgval_t &fexpr_rt, jl_value_t *declrt, jl_svec_t *argt) +static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, const jl_cgval_t &fexpr_val, jl_value_t *declrt, jl_svec_t *argt) { jl_unionall_t *unionall_env = (jl_is_method(ctx.linfo->def.method) && jl_is_unionall(ctx.linfo->def.method->sig)) ? (jl_unionall_t*)ctx.linfo->def.method->sig @@ -7615,8 +7655,8 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con // compute+verify the dispatch signature, and see if it depends on the environment sparams bool approx = false; sigt = (jl_value_t*)jl_alloc_svec(nargt + 1); - jl_svecset(sigt, 0, fexpr_rt.typ); - if (!fexpr_rt.constant && (!jl_is_concrete_type(jl_pinned_ref_get(fexpr_rt.typ)) || jl_is_kind(jl_pinned_ref_get(fexpr_rt.typ)))) + jl_svecset(sigt, 0, fexpr_val.typ); + if (!fexpr_val.constant && (!jl_is_concrete_type(jl_pinned_ref_get(fexpr_val.typ)) || jl_is_kind(jl_pinned_ref_get(fexpr_val.typ)))) approx = true; for (size_t i = 0; i < nargt; i++) { jl_value_t *jargty = jl_svecref(argt, i); @@ -7645,7 +7685,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con unionall_env = NULL; } - bool nest = (!fexpr_rt.constant || unionall_env); + bool nest = (!fexpr_val.constant || unionall_env); if (ctx.emission_context.TargetTriple.isAArch64() || ctx.emission_context.TargetTriple.isARM() || ctx.emission_context.TargetTriple.isPPC64()) { if (nest) { emit_error(ctx, "cfunction: closures are not supported on this platform"); @@ -7653,17 +7693,17 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con return jl_cgval_t(); } } - const char *name = derive_sigt_name(jl_pinned_ref_get(fexpr_rt.typ)); + const char *name = derive_sigt_name(jl_pinned_ref_get(fexpr_val.typ)); Value *F = gen_cfun_wrapper( jl_Module, ctx.emission_context, - sig, jl_pinned_ref_get(fexpr_rt.constant), name, + sig, jl_pinned_ref_get(fexpr_val.constant), name, declrt, sigt, unionall_env, sparam_vals, &closure_types); bool outboxed; if (nest) { // F is actually an init_trampoline function that returns the real address // Now fill in the nest parameters - Value *fobj = boxed(ctx, fexpr_rt); + Value *fobj = boxed(ctx, fexpr_val); jl_svec_t *fill = jl_emptysvec; if (closure_types) { assert(ctx.spvals_ptr); @@ -7703,7 +7743,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); ai.decorateInst(ctx.builder.CreateStore(F, derived_strct)); ai.decorateInst(ctx.builder.CreateStore( - ctx.builder.CreatePtrToInt(literal_pointer_val(ctx, jl_pinned_ref_get(fexpr_rt.constant)), ctx.types().T_size), + ctx.builder.CreatePtrToInt(literal_pointer_val(ctx, jl_pinned_ref_get(fexpr_val.constant)), ctx.types().T_size), ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_size, derived_strct, 1))); ai.decorateInst(ctx.builder.CreateStore(Constant::getNullValue(ctx.types().T_size), ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_size, derived_strct, 2))); @@ -7725,7 +7765,7 @@ const char *jl_generate_ccallable(Module *llvmmod, jl_value_t *nameval, jl_value assert(jl_is_datatype(ft)); jl_value_t *ff = ft->instance; assert(ff); - const char *name = !jl_is_string(nameval) ? jl_symbol_name(ft->name->mt->name) : jl_string_data(nameval); + const char *name = !jl_is_string(nameval) ? jl_symbol_name(ft->name->singletonname) : jl_string_data(nameval); jl_value_t *crt = declrt; if (jl_is_abstract_ref_type(declrt)) { declrt = jl_tparam0(declrt); @@ -7938,6 +7978,7 @@ static jl_returninfo_t get_specsig_function(jl_codegen_params_t ¶ms, Module AttrBuilder param(M->getContext()); if (params.use_swiftcc) param.addAttribute(Attribute::SwiftSelf); + param.addAttribute("gcstack"); param.addAttribute(Attribute::NonNull); attrs.push_back(AttributeSet::get(M->getContext(), param)); fsig.push_back(PointerType::get(M->getContext(), 0)); @@ -7964,7 +8005,7 @@ static jl_returninfo_t get_specsig_function(jl_codegen_params_t ¶ms, Module param.addAttribute(Attribute::ReadOnly); ty = PointerType::get(M->getContext(), AddressSpace::Derived); } - else if (isboxed && jl_is_immutable_datatype(jt)) { + else if (isboxed && jl_may_be_immutable_datatype(jt) && !jl_is_abstracttype(jt)) { param.addAttribute(Attribute::ReadOnly); } else if (jl_is_primitivetype(jt) && ty->isIntegerTy()) { @@ -8492,7 +8533,7 @@ static jl_llvm_functions_t ctx.spvals_ptr = &*AI++; } } - // step 6. set up GC frame and special arguments + // step 6a. set up special arguments and attributes Function::arg_iterator AI = f->arg_begin(); SmallVector attrs(f->arg_size()); // function declaration attributes @@ -8539,7 +8580,11 @@ static jl_llvm_functions_t attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); } + // step 6b. Setup the GC frame and entry safepoint before any loads allocate_gc_frame(ctx, b0); + if (params.safepoint_on_entry && JL_FEAT_TEST(ctx, safepoint_on_entry)) + emit_gc_safepoint(ctx.builder, ctx.types().T_size, get_current_ptls(ctx), ctx.tbaa().tbaa_const); + Value *last_age = NULL; Value *world_age_field = NULL; if (ctx.is_opaque_closure) { @@ -8697,7 +8742,14 @@ static jl_llvm_functions_t SmallVector roots; auto tracked = CountTrackedPointers(llvmArgType); if (tracked.count && !tracked.all) { - roots = load_gc_roots(ctx, &*AI, tracked.count); + Argument *RootArg = &*AI; + roots = load_gc_roots(ctx, RootArg, tracked.count, ctx.tbaa().tbaa_const); + AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); + param.addAttribute(Attribute::NonNull); + param.addAttribute(Attribute::NoUndef); + param.addDereferenceableAttr(tracked.count * sizeof(void*)); + param.addAlignmentAttr(alignof(void*)); + attrs[RootArg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); ++AI; } theArg = mark_julia_slot(Arg, argType, NULL, ctx.tbaa().tbaa_const, roots); // this argument is by-pointer @@ -9007,11 +9059,7 @@ static jl_llvm_functions_t Instruction &prologue_end = ctx.builder.GetInsertBlock()->back(); - // step 11a. Emit the entry safepoint - if (JL_FEAT_TEST(ctx, safepoint_on_entry)) - emit_gc_safepoint(ctx.builder, ctx.types().T_size, get_current_ptls(ctx), ctx.tbaa().tbaa_const); - - // step 11b. Do codegen in control flow order + // step 11. Do codegen in control flow order SmallVector workstack; DenseMap BB; DenseMap come_from_bb; @@ -9360,7 +9408,7 @@ static jl_llvm_functions_t // Probably dead code, but let's be loud about it in case it isn't, so we fail // at the point of the miscompile, rather than later when something attempts to // read the scope. - emit_error(ctx, "(INTERNAL ERROR): Attempted to execute EnterNode with bad scope"); + emit_error(ctx, "(INTERNAL ERROR - IR Validity): Attempted to execute EnterNode with bad scope"); find_next_stmt(-1); continue; } @@ -9832,6 +9880,85 @@ jl_llvm_functions_t jl_emit_codeinst( return decls; } +/// Stolen from IRMover.cpp, since it is needlessly private there +void linkFunctionBody(Function &Dst, Function &Src) +{ + assert(Dst.isDeclaration() && !Src.isDeclaration()); + + // Link in the operands without remapping. + if (Src.hasPrefixData()) + Dst.setPrefixData(Src.getPrefixData()); + if (Src.hasPrologueData()) + Dst.setPrologueData(Src.getPrologueData()); + if (Src.hasPersonalityFn()) + Dst.setPersonalityFn(Src.getPersonalityFn()); + if (Src.hasPersonalityFn()) + Dst.setPersonalityFn(Src.getPersonalityFn()); + assert(Src.IsNewDbgInfoFormat == Dst.IsNewDbgInfoFormat); + + // Copy over the metadata attachments without remapping. + Dst.copyMetadata(&Src, 0); + + // Steal arguments and splice the body of Src into Dst. + Dst.stealArgumentListFrom(Src); + Dst.splice(Dst.end(), &Src); +} + +void emit_always_inline(orc::ThreadSafeModule &result_m, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER +{ + while (true) { + SmallVector always_inline; + for (auto &it : params.workqueue) { + if (it.second.private_linkage && it.second.decl->isDeclaration()) + always_inline.push_back(it); + it.second.private_linkage = false; + } + if (always_inline.empty()) + return; + jl_task_t *ct = jl_current_task; + int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) + jl_code_info_t *src = nullptr; + params.safepoint_on_entry = false; + params.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); + JL_GC_PUSH2(¶ms.temporary_roots, &src); + for (auto &it : always_inline) { + jl_code_instance_t *codeinst = it.first; + auto &proto = it.second; + Function *decl = proto.decl; + if (decl->isDeclaration()) { + src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + jl_method_t *def = mi->def.method; + if (src && jl_is_string((jl_value_t*)src) && jl_is_method(def) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) + src = jl_uncompress_ir(def, codeinst, (jl_value_t*)src); + if (src && jl_is_code_info(src) && jl_ir_inlining_cost((jl_value_t*)src) < UINT16_MAX) { + jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params); // contains safepoints + if (!result_m) + break; + // TODO: jl_optimize_roots(params, mi, *result_m.getModuleUnlocked()); // contains safepoints + Module &M = *result_m.getModuleUnlocked(); + if (decls.functionObject != "jl_fptr_args" && + decls.functionObject != "jl_fptr_sparam" && + decls.functionObject != "jl_f_opaque_closure_call") { + Function *F = M.getFunction(decls.functionObject); + F->eraseFromParent(); + } + if (!decls.specFunctionObject.empty()) { + Function *specF = M.getFunction(decls.specFunctionObject); + linkFunctionBody(*decl, *specF); + decl->addFnAttr(Attribute::InlineHint); + decl->setLinkage(proto.external_linkage ? GlobalValue::AvailableExternallyLinkage : GlobalValue::PrivateLinkage); + specF->eraseFromParent(); + } + } + } + } + params.temporary_roots = nullptr; + JL_GC_POP(); + jl_gc_unsafe_leave(ct->ptls, gc_state); + } +} + // --- initialization --- static auto gv_for_global = new SmallVector, 0>(); static void global_jlvalue_to_llvm(JuliaVariable *var, jl_value_t **addr) @@ -9851,7 +9978,6 @@ static void init_jit_functions(void) { add_named_global("jl_fptr_args", jl_fptr_args_addr); add_named_global("jl_fptr_sparam", jl_fptr_sparam_addr); - add_named_global("jl_f_opaque_closure_call", &jl_f_opaque_closure_call); add_named_global(jl_small_typeof_var, &jl_small_typeof); add_named_global(jlstack_chk_guard_var, &__stack_chk_guard); add_named_global(jlRTLD_DEFAULT_var, &jl_RTLD_DEFAULT_handle); @@ -9888,10 +10014,8 @@ static void init_jit_functions(void) add_named_global(jlcheckassign_func, &jl_checked_assignment); add_named_global(jlcheckbpwritable_func, &jl_check_binding_currently_writable); add_named_global(jlboundp_func, &jl_boundp); - for (auto it : builtin_func_map()) - add_named_global(it.second, it.first); - add_named_global(jlintrinsic_func, &jl_f_intrinsic_call); - add_named_global(jlgetbuiltinfptr_func, &jl_get_builtin_fptr); + for (int i = 0; i < jl_n_builtins; i++) + add_named_global(jl_builtin_f_names[i], jl_builtin_f_addrs[i]); add_named_global(jlapplygeneric_func, &jl_apply_generic); add_named_global(jlinvoke_func, &jl_invoke); add_named_global(jltopeval_func, &jl_toplevel_eval); @@ -9939,13 +10063,13 @@ static void init_jit_functions(void) #ifdef _OS_WINDOWS_ #if defined(_CPU_X86_64_) add_named_global("__julia_personality", &__julia_personality); -#if defined(_COMPILER_GCC_) +#if defined(__MINGW32__) add_named_global("___chkstk_ms", &___chkstk_ms); #else add_named_global("__chkstk", &__chkstk); #endif #else -#if defined(_COMPILER_GCC_) +#if defined(__MINGW32__) add_named_global("_alloca", &_alloca); #else add_named_global("_chkstk", &_chkstk); diff --git a/src/common_symbols2.inc b/src/common_symbols2.inc index 2a6990bac52ff..e9c070ee8da6a 100644 --- a/src/common_symbols2.inc +++ b/src/common_symbols2.inc @@ -97,7 +97,7 @@ jl_symbol("pointerref"), jl_symbol("multidimensional.jl"), jl_symbol("Generator"), jl_symbol("leave"), -jl_symbol("memoryref"), +jl_symbol("memoryrefnew"), jl_symbol("show.jl"), jl_symbol("pointer_from_objref"), jl_symbol("memoryrefget"), diff --git a/src/datatype.c b/src/datatype.c index f27d0eceac321..71673aa693fe0 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -39,22 +39,31 @@ static jl_sym_t *jl_demangle_typename(jl_sym_t *s) JL_NOTSAFEPOINT return _jl_symbol(&n[1], len); } +JL_DLLEXPORT jl_methcache_t *jl_new_method_cache(void) +{ + jl_task_t *ct = jl_current_task; + jl_methcache_t *mc = + (jl_methcache_t*)jl_gc_alloc(ct->ptls, sizeof(jl_methcache_t), + jl_methcache_type); + jl_atomic_store_relaxed(&mc->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); + jl_atomic_store_relaxed(&mc->cache, jl_nothing); + JL_MUTEX_INIT(&mc->writelock, "methodtable->writelock"); + return mc; +} + JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module) { + jl_methcache_t *mc = jl_new_method_cache(); + JL_GC_PUSH1(&mc); jl_task_t *ct = jl_current_task; jl_methtable_t *mt = - (jl_methtable_t*)jl_gc_alloc(ct->ptls, sizeof(jl_methtable_t), - jl_methtable_type); - mt->name = jl_demangle_typename(name); - mt->module = module; + (jl_methtable_t*)jl_gc_alloc(ct->ptls, sizeof(jl_methtable_t), jl_methtable_type); jl_atomic_store_relaxed(&mt->defs, jl_nothing); - jl_atomic_store_relaxed(&mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); - jl_atomic_store_relaxed(&mt->cache, jl_nothing); - jl_atomic_store_relaxed(&mt->max_args, 0); - mt->backedges = NULL; - JL_MUTEX_INIT(&mt->writelock, "methodtable->writelock"); - mt->offs = 0; - mt->frozen = 0; + mt->cache = mc; + mt->name = name; + mt->module = module; + mt->backedges = (jl_genericmemory_t*)jl_an_empty_memory_any; + JL_GC_POP(); return mt; } @@ -67,21 +76,23 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu tn->name = name; tn->module = module; tn->wrapper = NULL; + tn->singletonname = jl_demangle_typename(name); jl_atomic_store_relaxed(&tn->Typeofwrapper, NULL); jl_atomic_store_relaxed(&tn->cache, jl_emptysvec); jl_atomic_store_relaxed(&tn->linearcache, jl_emptysvec); tn->names = NULL; tn->hash = bitmix(bitmix(module ? module->build_id.lo : 0, name->hash), 0xa1ada1da); - tn->_reserved = 0; + tn->_unused = 0; tn->abstract = abstract; tn->mutabl = mutabl; tn->mayinlinealloc = 0; - tn->mt = NULL; tn->partial = NULL; tn->atomicfields = NULL; tn->constfields = NULL; tn->hiddenptrfields = NULL; tn->max_methods = 0; + jl_atomic_store_relaxed(&tn->max_args, 0); + jl_atomic_store_relaxed(&tn->cache_entry_count, 0); tn->constprop_heustic = 0; return tn; } @@ -251,8 +262,10 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, flddesc->flags.haspadding = haspadding; flddesc->flags.isbitsegal = isbitsegal; flddesc->flags.fielddesc_type = fielddesc_type; - flddesc->flags.arrayelem_isboxed = arrayelem == 1; - flddesc->flags.arrayelem_isunion = arrayelem == 2; + flddesc->flags.arrayelem_isboxed = (arrayelem & 1) != 0; + flddesc->flags.arrayelem_isunion = (arrayelem & 2) != 0; + flddesc->flags.arrayelem_isatomic = (arrayelem & 4) != 0; + flddesc->flags.arrayelem_islocked = (arrayelem & 8) != 0; flddesc->flags.padding = 0; flddesc->npointers = npointers; flddesc->first_ptr = first_ptr; @@ -545,7 +558,13 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) jl_value_t *kind = jl_tparam0(st); jl_value_t *eltype = jl_tparam1(st); jl_value_t *addrspace = jl_tparam2(st); - if (!jl_is_typevar(eltype) && !jl_is_type(eltype)) { + if (!st->isconcretetype) { + // Since parent dt has an opaque layout, we may end up here being asked to copy that layout to subtypes, + // but we don't actually want to do that unless this object is constructable (or at least has a layout). + // The real layout is stored only on the wrapper. + return; + } + if (!jl_is_type(eltype)) { // this is expected to have a layout, but since it is not constructable, we don't care too much what it is static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, 0, -1, -1, sizeof(void*), {0}}; st->layout = &opaque_ptr_layout; @@ -565,6 +584,7 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) uint32_t *pointers = &first_ptr; int needlock = 0; + const jl_datatype_layout_t *el_layout = NULL; if (isunboxed) { elsz = LLT_ALIGN(elsz, al); if (kind == (jl_value_t*)jl_atomic_sym) { @@ -579,12 +599,12 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) else { assert(jl_is_datatype(eltype)); zi = ((jl_datatype_t*)eltype)->zeroinit; - const jl_datatype_layout_t *layout = ((jl_datatype_t*)eltype)->layout; - if (layout->first_ptr >= 0) { - first_ptr = layout->first_ptr; - npointers = layout->npointers; - if (layout->flags.fielddesc_type == 2) { - pointers = (uint32_t*)jl_dt_layout_ptrs(layout); + el_layout = ((jl_datatype_t*)eltype)->layout; + if (el_layout->first_ptr >= 0) { + first_ptr = el_layout->first_ptr; + npointers = el_layout->npointers; + if (el_layout->flags.fielddesc_type == 2 && !needlock) { + pointers = (uint32_t*)jl_dt_layout_ptrs(el_layout); } else { pointers = (uint32_t*)alloca(npointers * sizeof(uint32_t)); @@ -596,10 +616,22 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) } if (needlock) { assert(al <= JL_SMALL_BYTE_ALIGNMENT); - size_t offset = LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT); - elsz += offset; + size_t lock_offset = LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT); + elsz += lock_offset; + if (al < sizeof(void*)) { + al = sizeof(void*); + elsz = LLT_ALIGN(elsz, al); + } haspadding = 1; zi = 1; + // Adjust pointer offsets to account for the lock at the beginning + if (first_ptr != -1) { + uint32_t lock_offset_words = lock_offset / sizeof(void*); + first_ptr += lock_offset_words; + for (int j = 0; j < npointers; j++) { + pointers[j] += lock_offset_words; + } + } } } else { @@ -608,13 +640,17 @@ void jl_get_genericmemory_layout(jl_datatype_t *st) zi = 1; } - int arrayelem; + // arrayelem is a bitfield: 1=isboxed, 2=isunion, 4=isatomic, 8=islocked + int arrayelem = 0; if (!isunboxed) - arrayelem = 1; - else if (isunion) - arrayelem = 2; - else - arrayelem = 0; + arrayelem |= 1; // arrayelem_isboxed + if (isunion) + arrayelem |= 2; // arrayelem_isunion + if (kind == (jl_value_t*)jl_atomic_sym) { + arrayelem |= 4; // arrayelem_isatomic + if (needlock) + arrayelem |= 8; // arrayelem_islocked + } assert(!st->layout); st->layout = jl_get_layout(elsz, nfields, npointers, 0, al, haspadding, isbitsegal, arrayelem, NULL, pointers, NULL); st->zeroinit = zi; @@ -675,17 +711,17 @@ void jl_compute_field_offsets(jl_datatype_t *st) // if we have no fields, we can trivially skip the rest if (st == jl_symbol_type || st == jl_string_type) { // opaque layout - heap-allocated blob - static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, 0, -1, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; + static const jl_datatype_layout_t opaque_byte_layout = {0, 0, 1, 0, -1, -1, 1, { .isbitsegal=1 }}; st->layout = &opaque_byte_layout; return; } else if (st == jl_simplevector_type || st == jl_module_type) { - static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, 0, -1, -1, sizeof(void*), { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; + static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, 0, -1, -1, sizeof(void*), { .isbitsegal=1 }}; st->layout = &opaque_ptr_layout; return; } else { - static const jl_datatype_layout_t singleton_layout = {0, 0, 0, 0, -1, -1, 1, { .haspadding = 0, .fielddesc_type=0, .isbitsegal=1, .arrayelem_isboxed=0, .arrayelem_isunion=0 }}; + static const jl_datatype_layout_t singleton_layout = {0, 0, 0, 0, -1, -1, 1, { .isbitsegal=1 }}; st->layout = &singleton_layout; } } @@ -751,6 +787,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) // Should never happen throw_ovf(should_malloc, desc, st, fsz); desc[i].isptr = 0; + if (jl_is_uniontype(fld)) { fsz += 1; // selector byte zeroinit = 1; @@ -758,6 +795,11 @@ void jl_compute_field_offsets(jl_datatype_t *st) isbitsegal = 0; } else { + if (fsz > jl_datatype_size(fld)) { + // We have to pad the size to integer size class, but it means this has some padding + isbitsegal = 0; + haspadding = 1; + } uint32_t fld_npointers = ((jl_datatype_t*)fld)->layout->npointers; if (((jl_datatype_t*)fld)->layout->flags.haspadding) haspadding = 1; @@ -942,18 +984,6 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype_with_hiddenptrs( } else { tn = jl_new_typename_in((jl_sym_t*)name, module, abstract, mutabl); - if (super == jl_function_type || super == jl_builtin_type || is_anonfn_typename(jl_symbol_name(name))) { - // Callable objects (including compiler-generated closures) get independent method tables - // as an optimization - tn->mt = jl_new_method_table(name, module); - jl_gc_wb(tn, tn->mt); - if (jl_svec_len(parameters) == 0 && !abstract) - tn->mt->offs = 1; - } - else { - // Everything else, gets to use the unified table - tn->mt = jl_nonfunction_mt; - } } t->name = tn; jl_gc_wb(t, t->name); @@ -1101,6 +1131,8 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name, layout->flags.padding = 0; layout->flags.arrayelem_isboxed = 0; layout->flags.arrayelem_isunion = 0; + layout->flags.arrayelem_isatomic = 0; + layout->flags.arrayelem_islocked = 0; jl_fielddescdyn_t * desc = (jl_fielddescdyn_t *) ((char *)layout + sizeof(*layout)); desc->markfunc = markfunc; diff --git a/src/debug-registry.h b/src/debug-registry.h index 72189c60d3d40..00e3445200361 100644 --- a/src/debug-registry.h +++ b/src/debug-registry.h @@ -12,7 +12,8 @@ typedef struct { const llvm::object::ObjectFile *obj; llvm::DIContext *ctx; int64_t slide; -} objfileentry_t; + std::map> *symbolmap; +} jl_object_file_entry_t; // Central registry for resolving function addresses to `jl_code_instance_t`s and // originating `ObjectFile`s (for the DWARF debug info). @@ -121,7 +122,7 @@ class JITDebugInfoRegistry using rev_map = std::map>; typedef rev_map objectmap_t; - typedef rev_map objfilemap_t; + typedef rev_map objfilemap_t; objectmap_t objectmap{}; rev_map> cimap{}; @@ -152,4 +153,6 @@ class JITDebugInfoRegistry void add_image_info(image_info_t info) JL_NOTSAFEPOINT; bool get_image_info(uint64_t base, image_info_t *info) const JL_NOTSAFEPOINT; Locked::LockT get_objfile_map() JL_NOTSAFEPOINT; + + std::shared_mutex symbol_mutex; }; diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 6db6a07c917b1..4adfd3398e13d 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include #ifdef _OS_DARWIN_ #include @@ -145,8 +147,8 @@ struct unw_table_entry template static void jl_profile_atomic(T f) JL_NOTSAFEPOINT { - assert(0 == jl_lock_profile_rd_held()); - jl_lock_profile_wr(); + int havelock = jl_lock_profile_wr(); + assert(havelock); #ifndef _OS_WINDOWS_ sigset_t sset; sigset_t oset; @@ -157,7 +159,8 @@ static void jl_profile_atomic(T f) JL_NOTSAFEPOINT #ifndef _OS_WINDOWS_ pthread_sigmask(SIG_SETMASK, &oset, NULL); #endif - jl_unlock_profile_wr(); + if (havelock) + jl_unlock_profile_wr(); } @@ -335,9 +338,18 @@ void JITDebugInfoRegistry::registerJITObject(const object::ObjectFile &Object, #endif // defined(_OS_WINDOWS_) SmallVector packed; - compression::zlib::compress(ArrayRef((uint8_t*)Object.getData().data(), Object.getData().size()), packed, compression::zlib::DefaultCompression); - jl_jit_add_bytes(packed.size()); - auto ObjectCopy = new LazyObjectInfo{packed, Object.getData().size()}; // intentionally leaked so that we don't need to ref-count it, intentionally copied so that we exact-size the allocation (since no shrink_to_fit function) + ArrayRef unpacked = arrayRefFromStringRef(Object.getData()); + std::optional F; + if (compression::zstd::isAvailable()) + F = compression::Format::Zstd; + else if (compression::zlib::isAvailable()) + F = compression::Format::Zlib; + if (F) + compression::compress(*F, unpacked, packed); + // intentionally leak this so that we don't need to ref-count it + // intentionally copy the input so that we exact-size the allocation (since no shrink_to_fit function) + auto ObjectCopy = new LazyObjectInfo{SmallVector(F ? ArrayRef(packed) : unpacked), F ? Object.getData().size() : 0}; + jl_jit_add_bytes(ObjectCopy->data.size()); auto symbols = object::computeSymbolSizes(Object); bool hassection = false; for (const auto &sym_size : symbols) { @@ -464,8 +476,8 @@ static int lookup_pointer( // DWARFContext/DWARFUnit update some internal tables during these queries, so // a lock is needed. - assert(0 == jl_lock_profile_rd_held()); - jl_lock_profile_wr(); + if (!jl_lock_profile_wr()) + return lookup_pointer(object::SectionRef(), NULL, frames, pointer, slide, demangle, noInline); auto inlineInfo = context->getInliningInfoForAddress(makeAddress(Section, pointer + slide), infoSpec); jl_unlock_profile_wr(); @@ -490,7 +502,8 @@ static int lookup_pointer( info = inlineInfo.getFrame(i); } else { - jl_lock_profile_wr(); + int havelock = jl_lock_profile_wr(); + assert(havelock); (void)havelock; info = context->getLineInfoForAddress(makeAddress(Section, pointer + slide), infoSpec); jl_unlock_profile_wr(); } @@ -568,7 +581,7 @@ void JITDebugInfoRegistry::libc_frames_t::libc_deregister_frame(const char *Entr } #endif -static bool getObjUUID(llvm::object::MachOObjectFile *obj, uint8_t uuid[16]) JL_NOTSAFEPOINT +static bool getObjUUID(const object::MachOObjectFile *obj, uint8_t uuid[16]) JL_NOTSAFEPOINT { for (auto Load : obj->load_commands()) { @@ -704,7 +717,8 @@ static inline void ignoreError(T &err) JL_NOTSAFEPOINT #endif } -static void get_function_name_and_base(llvm::object::SectionRef Section, size_t pointer, int64_t slide, bool inimage, +static void get_function_name_and_base(llvm::object::SectionRef Section, std::map> *symbolmap, + size_t pointer, int64_t slide, bool inimage, void **saddr, char **name, bool untrusted_dladdr) JL_NOTSAFEPOINT { bool needs_saddr = saddr && (!*saddr || untrusted_dladdr); @@ -730,59 +744,73 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t #endif } if (Section.getObject() && (needs_saddr || needs_name)) { - size_t distance = (size_t)-1; - object::SymbolRef sym_found; - for (auto sym : Section.getObject()->symbols()) { - if (!Section.containsSymbol(sym)) - continue; - auto addr = sym.getAddress(); - if (!addr) - continue; - size_t symptr = addr.get(); - if (symptr > pointer + slide) - continue; - size_t new_dist = pointer + slide - symptr; - if (new_dist > distance) - continue; - distance = new_dist; - sym_found = sym; - } - if (distance != (size_t)-1) { - if (needs_saddr) { - uintptr_t addr = cantFail(sym_found.getAddress()); - *saddr = (void*)(addr - slide); - needs_saddr = false; + uintptr_t addr = 0; + StringRef nameref{}; + { + std::shared_lock read_lock(getJITDebugRegistry().symbol_mutex); + if (symbolmap->empty()) { + read_lock.unlock(); + { + // symbol map hasn't been generated yet, so fill it in now + std::unique_lock write_lock(getJITDebugRegistry().symbol_mutex); + if (symbolmap->empty()) { + for (auto sym : Section.getObject()->symbols()) { + if (!Section.containsSymbol(sym)) + continue; + + auto maybe_addr = sym.getAddress(); + if (!maybe_addr) + continue; + size_t addr = maybe_addr.get(); + + auto maybe_nameref = sym.getName(); + StringRef nameref{}; + if (maybe_nameref) + nameref = maybe_nameref.get(); + + symbolmap->emplace(addr, nameref); + } + } + } + read_lock.lock(); + } + auto fit = symbolmap->lower_bound(pointer + slide); + if (fit != symbolmap->end()) { + addr = fit->first; + nameref = fit->second; } - if (needs_name) { - if (auto name_or_err = sym_found.getName()) { - auto nameref = name_or_err.get(); - const char globalPrefix = // == DataLayout::getGlobalPrefix + } + std::string namerefstr = nameref.str(); + if (needs_saddr && addr != 0) { + *saddr = (void*)(addr - slide); + needs_saddr = false; + } + if (needs_name && !nameref.empty()) { + const char globalPrefix = // == DataLayout::getGlobalPrefix #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) - '_'; + '_'; #elif defined(_OS_DARWIN_) - '_'; + '_'; #else - '\0'; + '\0'; #endif - if (globalPrefix) { - if (nameref[0] == globalPrefix) - nameref = nameref.drop_front(); + if (globalPrefix) { + if (nameref[0] == globalPrefix) + nameref = nameref.drop_front(); #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) - else if (nameref[0] == '@') // X86_VectorCall - nameref = nameref.drop_front(); + else if (nameref[0] == '@') // X86_VectorCall + nameref = nameref.drop_front(); #endif - // else VectorCall, Assembly, Internal, etc. - } + // else VectorCall, Assembly, Internal, etc. + } #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) - nameref = nameref.split('@').first; + nameref = nameref.split('@').first; #endif - size_t len = nameref.size(); - *name = (char*)realloc_s(*name, len + 1); - memcpy(*name, nameref.data(), len); - (*name)[len] = 0; - needs_name = false; - } - } + size_t len = nameref.size(); + *name = (char*)realloc_s(*name, len + 1); + memcpy(*name, nameref.data(), len); + (*name)[len] = 0; + needs_name = false; } } #ifdef _OS_WINDOWS_ @@ -806,7 +834,7 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t #endif } -static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSAFEPOINT +static jl_object_file_entry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSAFEPOINT { int isdarwin = 0, islinux = 0, iswindows = 0; #if defined(_OS_DARWIN_) @@ -819,7 +847,7 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA (void)iswindows; // GOAL: Read debuginfo from file - objfileentry_t entry{nullptr, nullptr, 0}; + jl_object_file_entry_t entry{nullptr, nullptr, 0, nullptr}; auto success = getJITDebugRegistry().get_objfile_map()->emplace(fbase, entry); if (!success.second) // Return cached value @@ -830,9 +858,6 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA std::string debuginfopath; uint8_t uuid[16], uuid2[16]; if (isdarwin) { - // Hide Darwin symbols (e.g. CoreFoundation) from non-Darwin systems. -#ifdef _OS_DARWIN_ - size_t msize = (size_t)(((uint64_t)-1) - fbase); std::unique_ptr membuf = MemoryBuffer::getMemBuffer( StringRef((const char *)fbase, msize), "", false); @@ -843,14 +868,18 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA return entry; } - llvm::object::MachOObjectFile *morigobj = (llvm::object::MachOObjectFile*) - origerrorobj.get().get(); + const object::MachOObjectFile *morigobj = dyn_cast( + origerrorobj.get().get()); // First find the uuid of the object file (we'll use this to make sure we find the // correct debug symbol file). - if (!getObjUUID(morigobj, uuid)) + if (!morigobj || !getObjUUID(morigobj, uuid)) return entry; + // Hide Darwin symbols (e.g. CoreFoundation) from non-Darwin systems. +#ifndef _OS_DARWIN_ + return entry; +#else // On macOS, debug symbols are not contained in the dynamic library. // Use DBGCopyFullDSYMURLForUUID from the private DebugSymbols framework // to make use of spotlight to find the dSYM file. If that fails, lookup @@ -906,6 +935,7 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA if (dsfmwkbundle) { CFRelease(dsfmwkbundle); } +#endif if (objpath.empty()) { // Fall back to simple path relative to the dynamic library. @@ -915,7 +945,6 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA debuginfopath += fname.substr(sep + 1); objpath = debuginfopath; } -#endif } else { // On Linux systems we need to mmap another copy because of the permissions on the mmap'ed shared library. @@ -974,15 +1003,17 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA if (isdarwin) { // verify the UUID matches - if (!getObjUUID((llvm::object::MachOObjectFile*)debugobj, uuid2) || - memcmp(uuid, uuid2, sizeof(uuid)) != 0) { + if (!isa(debugobj) || + !getObjUUID(cast(debugobj), uuid2) || + memcmp(uuid, uuid2, sizeof(uuid)) != 0) { return entry; } } int64_t slide = 0; if (auto *OF = dyn_cast(debugobj)) { - assert(iswindows); + if (!iswindows) // the COFF parser accepts some garbage inputs (like empty files) that the other parsers correctly reject, so we can end up here even when we should not + return entry; slide = OF->getImageBase() - fbase; } else { @@ -993,7 +1024,8 @@ static objfileentry_t find_object_file(uint64_t fbase, StringRef fname) JL_NOTSA auto binary = errorobj->takeBinary(); binary.first.release(); binary.second.release(); - entry = {debugobj, context, slide}; + + entry = {debugobj, context, slide, new std::map>()}; // update cache (*getJITDebugRegistry().get_objfile_map())[fbase] = entry; } @@ -1123,7 +1155,7 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * jl_copy_str(filename, dlinfo.dli_fname); fname = dlinfo.dli_fname; #endif // ifdef _OS_WINDOWS_ - auto entry = find_object_file(fbase, fname); + jl_object_file_entry_t entry = find_object_file(fbase, fname); *slide = entry.slide; *context = entry.ctx; if (entry.obj) @@ -1131,7 +1163,7 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * // Assume we only need base address for sysimg for now if (!inimage || 0 == image_info.fptrs.nptrs) saddr = nullptr; - get_function_name_and_base(*Section, pointer, entry.slide, inimage, saddr, name, untrusted_dladdr); + get_function_name_and_base(*Section, entry.symbolmap, pointer, entry.slide, inimage, saddr, name, untrusted_dladdr); return true; } @@ -1195,8 +1227,8 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, object::SectionRef *Section, llvm::DIContext **context) JL_NOTSAFEPOINT { int found = 0; - assert(0 == jl_lock_profile_rd_held()); - jl_lock_profile_wr(); + if (!jl_lock_profile_wr()) + return 0; if (symsize) *symsize = 0; @@ -1209,7 +1241,8 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, if (!lazyobject->object && !lazyobject->data.empty()) { if (lazyobject->uncompressedsize) { SmallVector unpacked; - Error E = compression::zlib::decompress(lazyobject->data, unpacked, lazyobject->uncompressedsize); + compression::Format F = compression::zstd::isAvailable() ? compression::Format::Zstd : compression::Format::Zlib; + Error E = compression::decompress(F, lazyobject->data, unpacked, lazyobject->uncompressedsize); if (E) lazyobject->data.clear(); else diff --git a/src/dlload.c b/src/dlload.c index 2c7ee08229394..8f56ab65c37d9 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -68,8 +68,6 @@ const char *jl_crtdll_name = CRTDLL_BASENAME ".dll"; #undef CRTDLL_BASENAME #endif -#define PATHBUF 4096 - #ifdef _OS_WINDOWS_ void win32_formatmessage(DWORD code, char *reason, int len) JL_NOTSAFEPOINT { @@ -272,7 +270,7 @@ void *jl_find_dynamic_library_by_addr(void *symbol, int throw_err) { JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, int throw_err) { - char path[PATHBUF], relocated[PATHBUF]; + ios_t path, relocated; int i; #ifdef _OS_WINDOWS_ int err; @@ -284,7 +282,6 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, // number of extensions to try — if modname already ends with the // standard extension, then we don't try adding additional extensions int n_extensions = endswith_extension(modname) ? 1 : N_EXTENSIONS; - int ret; // modname == NULL is a sentinel value requesting the handle of libjulia-internal if (modname == NULL) @@ -309,6 +306,9 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, } #endif + ios_mem(&path, IOS_INLSIZE); + ios_mem(&relocated, IOS_INLSIZE); + /* this branch permutes all base paths in DL_LOAD_PATH with all extensions note: skip when !jl_base_module to avoid UndefVarError(:DL_LOAD_PATH), @@ -325,43 +325,41 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, size_t j; for (j = 0; j < jl_array_nrows(DL_LOAD_PATH); j++) { char *dl_path = jl_string_data(jl_array_ptr_data(DL_LOAD_PATH)[j]); - size_t len = strlen(dl_path); - if (len == 0) + if (*dl_path == 0) continue; + ios_trunc(&relocated, 0); + // Is this entry supposed to be relative to the bindir? - if (len >= 16 && strncmp(dl_path, "@executable_path", 16) == 0) { - snprintf(relocated, PATHBUF, "%s%s", jl_options.julia_bindir, dl_path + 16); - len = len - 16 + strlen(jl_options.julia_bindir); + if (strncmp(dl_path, "@executable_path", 16) == 0) { + ios_printf(&relocated, "%s%s", jl_options.julia_bindir, dl_path + 16); } else { - strncpy(relocated, dl_path, PATHBUF); - relocated[PATHBUF-1] = '\0'; + ios_puts(dl_path, &relocated); } + ios_putc(0, &relocated); for (i = 0; i < n_extensions; i++) { + ios_trunc(&path, 0); const char *ext = extensions[i]; - path[0] = '\0'; - if (relocated[len-1] == PATHSEPSTRING[0]) - snprintf(path, PATHBUF, "%s%s%s", relocated, modname, ext); - else { - ret = snprintf(path, PATHBUF, "%s" PATHSEPSTRING "%s%s", relocated, modname, ext); - if (ret < 0) - jl_errorf("path is longer than %d\n", PATHBUF); - } + if (relocated.buf[relocated.bpos - 2] == PATHSEPSTRING[0]) + ios_printf(&path, "%s%s%s", relocated.buf, modname, ext); + else + ios_printf(&path, "%s" PATHSEPSTRING "%s%s", relocated.buf, modname, ext); + ios_putc(0, &path); #ifdef _OS_WINDOWS_ if (i == 0) { // LoadLibrary already tested the extensions, we just need to check the `stat` result #endif - handle = jl_dlopen(path, flags); + handle = jl_dlopen(path.buf, flags); if (handle && !(flags & JL_RTLD_NOLOAD)) jl_timing_puts(JL_TIMING_DEFAULT_BLOCK, jl_pathname_for_handle(handle)); if (handle) - return handle; + goto success; #ifdef _OS_WINDOWS_ err = GetLastError(); } #endif // bail out and show the error if file actually exists - if (jl_stat(path, (char*)&stbuf) == 0) + if (jl_stat(path.buf, (char*)&stbuf) == 0) goto notfound; } } @@ -370,20 +368,21 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, // now fall back and look in default library paths, for all extensions for (i = 0; i < n_extensions; i++) { + ios_trunc(&path, 0); const char *ext = extensions[i]; - path[0] = '\0'; - snprintf(path, PATHBUF, "%s%s", modname, ext); - handle = jl_dlopen(path, flags); + ios_printf(&path, "%s%s", modname, ext); + ios_putc(0, &path); + handle = jl_dlopen(path.buf, flags); if (handle && !(flags & JL_RTLD_NOLOAD)) jl_timing_puts(JL_TIMING_DEFAULT_BLOCK, jl_pathname_for_handle(handle)); if (handle) - return handle; + goto success; #ifdef _OS_WINDOWS_ err = GetLastError(); break; // LoadLibrary already tested the rest #else // bail out and show the error if file actually exists - if (jl_stat(path, (char*)&stbuf) == 0) + if (jl_stat(path.buf, (char*)&stbuf) == 0) break; #endif } @@ -396,10 +395,15 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, #else const char *reason = dlerror(); #endif + ios_close(&relocated); + ios_close(&path); jl_errorf("could not load library \"%s\"\n%s", modname, reason); } handle = NULL; +success: + ios_close(&relocated); + ios_close(&path); return handle; } diff --git a/src/flisp/Makefile b/src/flisp/Makefile index eca1de86e588a..2e902a31dbeed 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -117,7 +117,7 @@ $(BUILDDIR)/host/Makefile: @printf "%s\n" 'include $(SRCDIR)/Makefile' >> $@ $(BUILDDIR)/host/$(EXENAME): $(BUILDDIR)/host/Makefile | ${BUILDDIR}/host/flisp.boot - make -C $(BUILDDIR)/host $(EXENAME) + $(MAKE) -C $(BUILDDIR)/host $(EXENAME) $(BUILDDIR)/host/flisp.boot: $(SRCDIR)/flisp.boot | $(BUILDDIR)/host/Makefile diff --git a/src/flisp/julia_extensions.c b/src/flisp/julia_extensions.c index 07d074e1fb80b..c39c2edfe0f37 100644 --- a/src/flisp/julia_extensions.c +++ b/src/flisp/julia_extensions.c @@ -130,6 +130,9 @@ JL_DLLEXPORT int jl_id_start_char(uint32_t wc) return 1; if (wc < 0xA1 || wc > 0x10ffff) return 0; + // "Rightwards Arrow with Lower Hook" + if (wc == 0x1f8b2) + return 1; return is_wc_cat_id_start(wc, utf8proc_category((utf8proc_int32_t) wc)); } @@ -147,7 +150,9 @@ JL_DLLEXPORT int jl_id_char(uint32_t wc) cat == UTF8PROC_CATEGORY_SK || cat == UTF8PROC_CATEGORY_ME || cat == UTF8PROC_CATEGORY_NO || // primes (single, double, triple, their reverses, and quadruple) - (wc >= 0x2032 && wc <= 0x2037) || (wc == 0x2057)) + (wc >= 0x2032 && wc <= 0x2037) || (wc == 0x2057) || + // "Rightwards Arrow with Lower Hook" + wc == 0x1f8b2) return 1; return 0; } diff --git a/src/gc-common.c b/src/gc-common.c index 24d6c6cc29178..6f1e85d5db691 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -126,7 +126,7 @@ JL_DLLEXPORT void jl_gc_set_cb_notify_gc_pressure(jl_gc_cb_notify_gc_pressure_t // but with several fixes to improve the correctness of the computation and remove unnecessary parameters #define SAVED_PTR(x) ((void *)((DWORD_PTR)((char *)x - sizeof(void *)) & \ ~(sizeof(void *) - 1))) -static size_t _aligned_msize(void *p) +static size_t _jl_aligned_msize(void *p) { void *alloc_ptr = *(void**)SAVED_PTR(p); return _msize(alloc_ptr) - ((char*)p - (char*)alloc_ptr); @@ -138,7 +138,7 @@ size_t memory_block_usable_size(void *p, int isaligned) JL_NOTSAFEPOINT { #if defined(_OS_WINDOWS_) if (isaligned) - return _aligned_msize(p); + return _jl_aligned_msize(p); else return _msize(p); #elif defined(_OS_DARWIN_) diff --git a/src/gc-debug.c b/src/gc-debug.c index 6e51064035b7b..fa588a2ca886a 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -892,10 +892,7 @@ void gc_heuristics_summary( void jl_gc_debug_init(void) { #ifdef GC_DEBUG_ENV - char *env = getenv("JULIA_GC_NO_GENERATIONAL"); - if (env && strcmp(env, "0") != 0) - jl_gc_debug_env.always_full = 1; - env = getenv("JULIA_GC_WAIT_FOR_DEBUGGER"); + char *env = getenv("JULIA_GC_WAIT_FOR_DEBUGGER"); jl_gc_debug_env.wait_for_debugger = env && strcmp(env, "0") != 0; gc_debug_alloc_init(&jl_gc_debug_env.pool, "POOL"); gc_debug_alloc_init(&jl_gc_debug_env.other, "OTHER"); diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index e92f77e26a9b3..cbe80ca490167 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -3,6 +3,10 @@ #include "mmtkMutator.h" #include "threading.h" +#ifdef _COMPILER_TSAN_ENABLED_ +#include +#endif + // File exists in the binding #include "mmtk.h" @@ -545,9 +549,8 @@ void trace_full_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, TRACE_GLOBALLY_ROOTED(call_cache[i]); } // julia_internal.h - TRACE_GLOBALLY_ROOTED(jl_type_type_mt); - TRACE_GLOBALLY_ROOTED(jl_nonfunction_mt); - TRACE_GLOBALLY_ROOTED(jl_kwcall_mt); + TRACE_GLOBALLY_ROOTED(jl_typeinf_func); + TRACE_GLOBALLY_ROOTED(jl_method_table); TRACE_GLOBALLY_ROOTED(jl_opaque_closure_method); TRACE_GLOBALLY_ROOTED(jl_nulldebuginfo); TRACE_GLOBALLY_ROOTED(_jl_debug_method_invalidation); @@ -707,15 +710,17 @@ void trace_full_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* buf, TRACE_GLOBALLY_ROOTED(jl_newvarnode_type); TRACE_GLOBALLY_ROOTED(jl_intrinsic_type); TRACE_GLOBALLY_ROOTED(jl_methtable_type); + TRACE_GLOBALLY_ROOTED(jl_methcache_type); TRACE_GLOBALLY_ROOTED(jl_typemap_level_type); TRACE_GLOBALLY_ROOTED(jl_typemap_entry_type); + TRACE_GLOBALLY_ROOTED(jl_kwcall_type); + TRACE_GLOBALLY_ROOTED(jl_emptysvec); TRACE_GLOBALLY_ROOTED(jl_emptytuple); TRACE_GLOBALLY_ROOTED(jl_true); TRACE_GLOBALLY_ROOTED(jl_false); TRACE_GLOBALLY_ROOTED(jl_nothing); - TRACE_GLOBALLY_ROOTED(jl_kwcall_func); TRACE_GLOBALLY_ROOTED(jl_libdl_dlopen_func); @@ -743,7 +748,8 @@ void trace_partial_globally_rooted(RootsWorkClosure* closure, RootsWorkBuffer* b // add module TRACE_GLOBALLY_ROOTED(jl_main_module); - // buildin values + // invisible builtin values + TRACE_GLOBALLY_ROOTED(jl_method_table); TRACE_GLOBALLY_ROOTED(jl_an_empty_vec_any); TRACE_GLOBALLY_ROOTED(jl_module_init_order); diff --git a/src/gc-stock.c b/src/gc-stock.c index 340915e38dce3..c8fc8194dce14 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -824,21 +824,29 @@ int jl_gc_classify_pools(size_t sz, int *osize) // sweep phase -gc_fragmentation_stat_t gc_page_fragmentation_stats[JL_GC_N_POOLS]; +JL_DLLEXPORT gc_fragmentation_stat_t jl_gc_page_fragmentation_stats[JL_GC_N_POOLS]; JL_DLLEXPORT double jl_gc_page_utilization_stats[JL_GC_N_MAX_POOLS]; JL_DLLEXPORT gc_fragmentation_stat_t jl_gc_page_fragmentation_stats_export[JL_GC_N_POOLS]; -STATIC_INLINE void gc_update_page_fragmentation_data(jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT +STATIC_INLINE void gc_update_fragmentation_data_for_size_class(jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT { - gc_fragmentation_stat_t *stats = &gc_page_fragmentation_stats[pg->pool_n]; + gc_fragmentation_stat_t *stats = &jl_gc_page_fragmentation_stats[pg->pool_n]; jl_atomic_fetch_add_relaxed(&stats->n_freed_objs, pg->nfree); jl_atomic_fetch_add_relaxed(&stats->n_pages_allocd, 1); } -STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT +STATIC_INLINE void gc_reset_fragmentation_data_for_size_classes(void) JL_NOTSAFEPOINT { for (int i = 0; i < JL_GC_N_POOLS; i++) { - gc_fragmentation_stat_t *stats = &gc_page_fragmentation_stats[i]; + jl_atomic_store_relaxed(&jl_gc_page_fragmentation_stats[i].n_freed_objs, 0); + jl_atomic_store_relaxed(&jl_gc_page_fragmentation_stats[i].n_pages_allocd, 0); + } +} + +STATIC_INLINE void gc_compute_utilization_data_for_size_classes(void) JL_NOTSAFEPOINT +{ + for (int i = 0; i < JL_GC_N_POOLS; i++) { + gc_fragmentation_stat_t *stats = &jl_gc_page_fragmentation_stats[i]; double utilization = 1.0; size_t n_freed_objs = jl_atomic_load_relaxed(&stats->n_freed_objs); size_t n_pages_allocd = jl_atomic_load_relaxed(&stats->n_pages_allocd); @@ -957,7 +965,7 @@ static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_ done: if (re_use_page) { - gc_update_page_fragmentation_data(pg); + gc_update_fragmentation_data_for_size_class(pg); push_lf_back(allocd, pg); } else { @@ -1397,6 +1405,7 @@ static void gc_sweep_pool(void) // the actual sweeping jl_gc_padded_page_stack_t *new_gc_allocd_scratch = (jl_gc_padded_page_stack_t *) calloc_s(n_threads * sizeof(jl_gc_padded_page_stack_t)); jl_ptls_t ptls = jl_current_task->ptls; + gc_reset_fragmentation_data_for_size_classes(); gc_sweep_wake_all_pages(ptls, new_gc_allocd_scratch); gc_sweep_pool_parallel(ptls); gc_sweep_wait_for_all_pages(); @@ -1464,7 +1473,7 @@ static void gc_sweep_pool(void) #else gc_free_pages(); #endif - gc_dump_page_utilization_data(); + gc_compute_utilization_data_for_size_classes(); gc_time_pool_end(current_sweep_full); } @@ -2775,7 +2784,7 @@ void gc_mark_clean_reclaim_sets(void) } } -static void gc_queue_thread_local(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) +static void gc_queue_thread_local(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) JL_NOTSAFEPOINT { jl_task_t *task; task = ptls2->root_task; @@ -2804,7 +2813,7 @@ static void gc_queue_thread_local(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) } } -static void gc_queue_bt_buf(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) +static void gc_queue_bt_buf(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) JL_NOTSAFEPOINT { jl_bt_element_t *bt_data = ptls2->bt_data; size_t bt_size = ptls2->bt_size; @@ -2818,7 +2827,7 @@ static void gc_queue_bt_buf(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) } } -static void gc_queue_remset(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) +static void gc_queue_remset(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) JL_NOTSAFEPOINT { void **items = ptls2->gc_tls.heap.remset.items; size_t len = ptls2->gc_tls.heap.remset.len; @@ -2833,7 +2842,7 @@ static void gc_queue_remset(jl_gc_markqueue_t *mq, jl_ptls_t ptls2) ptls2->gc_tls.heap.remset_nptr = 0; } -static void gc_check_all_remsets_are_empty(void) +static void gc_check_all_remsets_are_empty(void) JL_NOTSAFEPOINT { for (int i = 0; i < gc_n_threads; i++) { jl_ptls_t ptls2 = gc_all_tls_states[i]; @@ -2848,12 +2857,14 @@ extern jl_value_t *cmpswap_names JL_GLOBALLY_ROOTED; extern jl_task_t *wait_empty JL_GLOBALLY_ROOTED; // mark the initial root set -static void gc_mark_roots(jl_gc_markqueue_t *mq) +static void gc_mark_roots(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT { // modules gc_try_claim_and_push(mq, jl_main_module, NULL); gc_heap_snapshot_record_gc_roots((jl_value_t*)jl_main_module, "main_module"); // invisible builtin values + gc_try_claim_and_push(mq, jl_method_table, NULL); + gc_heap_snapshot_record_gc_roots((jl_value_t*)jl_method_table, "global_method_table"); gc_try_claim_and_push(mq, jl_an_empty_vec_any, NULL); gc_heap_snapshot_record_gc_roots((jl_value_t*)jl_an_empty_vec_any, "an_empty_vec_any"); gc_try_claim_and_push(mq, jl_module_init_order, NULL); @@ -3030,7 +3041,7 @@ static uint64_t overallocation(uint64_t old_val, uint64_t val, uint64_t max_val) size_t jl_maxrss(void); // Only one thread should be running in this function -static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) +static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) JL_NOTSAFEPOINT { combine_thread_gc_counts(&gc_num, 1); @@ -3155,7 +3166,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) // If the live data outgrows the suggested max_total_memory // we keep going with minimum intervals and full gcs until // we either free some space or get an OOM error. - if (gc_sweep_always_full) { + if (jl_options.gc_sweep_always_full) { sweep_full = 1; gc_count_full_sweep_reason(FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL); } diff --git a/src/gc-stock.h b/src/gc-stock.h index d478ee1366da0..d9286ec2090bb 100644 --- a/src/gc-stock.h +++ b/src/gc-stock.h @@ -44,7 +44,6 @@ typedef struct { } jl_alloc_num_t; typedef struct { - int always_full; int wait_for_debugger; jl_alloc_num_t pool; jl_alloc_num_t other; @@ -365,15 +364,6 @@ STATIC_INLINE jl_gc_pagemeta_t *pop_page_metadata_back(jl_gc_pagemeta_t **ppg) J return v; } -#ifdef __clang_gcanalyzer__ /* clang may not have __builtin_ffs */ -unsigned ffs_u32(uint32_t bitvec) JL_NOTSAFEPOINT; -#else -STATIC_INLINE unsigned ffs_u32(uint32_t bitvec) -{ - return __builtin_ffs(bitvec) - 1; -} -#endif - extern bigval_t *oldest_generation_of_bigvals; extern int64_t buffered_pages; extern int gc_first_tid; @@ -656,14 +646,12 @@ NOINLINE void gc_mark_loop_unwind(jl_ptls_t ptls, jl_gc_markqueue_t *mq, int off #ifdef GC_DEBUG_ENV JL_DLLEXPORT extern jl_gc_debug_env_t jl_gc_debug_env; -#define gc_sweep_always_full jl_gc_debug_env.always_full int jl_gc_debug_check_other(void); int gc_debug_check_pool(void); void jl_gc_debug_print(void); void gc_scrub_record_task(jl_task_t *ta) JL_NOTSAFEPOINT; void gc_scrub(void); #else -#define gc_sweep_always_full 0 static inline int jl_gc_debug_check_other(void) { return 0; diff --git a/src/gen_sysimg_symtab.jl b/src/gen_sysimg_symtab.jl index a91f2f994194c..110d83ba9083d 100644 --- a/src/gen_sysimg_symtab.jl +++ b/src/gen_sysimg_symtab.jl @@ -11,8 +11,8 @@ import Base.Iterators: take, drop function _eachmethod(f, m::Module, visited, vmt) push!(visited, m) for nm in names(m, all=true) - if isdefined(m, nm) - x = getfield(m, nm) + if isdefinedglobal(m, nm) + x = getglobal(m, nm) if isa(x, Module) && !in(x, visited) _eachmethod(f, x, visited, vmt) elseif isa(x, Type) diff --git a/src/genericmemory.c b/src/genericmemory.c index 81aa724f66b11..9ec6dd8c6a4f5 100644 --- a/src/genericmemory.c +++ b/src/genericmemory.c @@ -56,7 +56,7 @@ jl_genericmemory_t *_new_genericmemory_(jl_value_t *mtype, size_t nel, int8_t is { if (nel == 0) // zero-sized allocation optimization return (jl_genericmemory_t*)((jl_datatype_t*)mtype)->instance; - size_t nbytes; + size_t nbytes = 0; // initialized to workaround clang sa bug on v20: https://github.com/llvm/llvm-project/issues/136292 int overflow = __builtin_mul_overflow(nel, elsz, &nbytes); if (isunion) { // an extra byte for each isbits union memory element, stored at m->ptr + m->length @@ -153,7 +153,7 @@ JL_DLLEXPORT jl_genericmemory_t *jl_ptr_to_genericmemory(jl_value_t *mtype, void if (((uintptr_t)data) & ((align > JL_HEAP_ALIGNMENT ? JL_HEAP_ALIGNMENT : align) - 1)) jl_exceptionf(jl_argumenterror_type, "unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align); - size_t nbytes; + size_t nbytes = 0; // initialized to workaround clang sa bug on v20: https://github.com/llvm/llvm-project/issues/136292 int overflow = __builtin_mul_overflow(nel, elsz, &nbytes); if (isunion) { // an extra byte for each isbits union memory element, stored at m->ptr + m->length @@ -199,7 +199,6 @@ JL_DLLEXPORT jl_value_t *jl_genericmemory_to_string(jl_genericmemory_t *m, size_ } int how = jl_genericmemory_how(m); size_t mlength = m->length; - m->length = 0; if (how != 0) { jl_value_t *o = jl_genericmemory_data_owner_field(m); jl_genericmemory_data_owner_field(m) = NULL; @@ -265,8 +264,8 @@ JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destda JL_DLLEXPORT jl_value_t *jl_genericmemoryref(jl_genericmemory_t *mem, size_t i) { - int isatomic = (jl_tparam0(jl_typetagof(mem)) == (jl_value_t*)jl_atomic_sym); const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(mem))->layout; + int isatomic = layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked; jl_genericmemoryref_t m; m.mem = mem; m.ptr_or_offset = (layout->flags.arrayelem_isunion || layout->size == 0) ? (void*)i : (void*)((char*)mem->ptr + layout->size * i); @@ -285,6 +284,9 @@ JL_DLLEXPORT jl_genericmemory_t *jl_genericmemory_copy_slice(jl_genericmemory_t memcpy(jl_genericmemory_typetagdata(new_mem), jl_genericmemory_typetagdata(mem) + (size_t)data, len); } else if (layout->first_ptr != -1) { + if (data == NULL) { + assert(len * elsz / sizeof(void*) == 0); // make static analyzer happy + } memmove_refs((_Atomic(void*)*)new_mem->ptr, (_Atomic(void*)*)data, len * elsz / sizeof(void*)); } else if (data != NULL) { @@ -342,8 +344,8 @@ static jl_value_t *jl_ptrmemrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT, i JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m, int isatomic) { - assert(isatomic == (jl_tparam0(jl_typetagof(m.mem)) == (jl_value_t*)jl_atomic_sym)); const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + assert(isatomic == (layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked)); if (layout->flags.arrayelem_isboxed) return jl_ptrmemrefget(m, isatomic); jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); @@ -365,7 +367,7 @@ JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m, int isatomic) assert(data - (char*)m.mem->ptr < layout->size * m.mem->length); jl_value_t *r; size_t fsz = jl_datatype_size(eltype); - int needlock = isatomic && fsz > MAX_ATOMIC_SIZE; + int needlock = layout->flags.arrayelem_islocked; if (isatomic && !needlock) { r = jl_atomic_new_bits(eltype, data); } @@ -393,9 +395,6 @@ static int _jl_memoryref_isassigned(jl_genericmemoryref_t m, int isatomic) if (layout->flags.arrayelem_isboxed) { } else if (layout->first_ptr >= 0) { - int needlock = isatomic && layout->size > MAX_ATOMIC_SIZE; - if (needlock) - elem = elem + LLT_ALIGN(sizeof(jl_mutex_t), JL_SMALL_BYTE_ALIGNMENT) / sizeof(jl_value_t*); elem = &elem[layout->first_ptr]; } else { @@ -411,7 +410,8 @@ JL_DLLEXPORT jl_value_t *jl_memoryref_isassigned(jl_genericmemoryref_t m, int is JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, jl_value_t *rhs JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, int isatomic) { - assert(isatomic == (jl_tparam0(jl_typetagof(m.mem)) == (jl_value_t*)jl_atomic_sym)); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + assert(isatomic == (layout->flags.arrayelem_isatomic || layout->flags.arrayelem_islocked)); jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); if (eltype != (jl_value_t*)jl_any_type && !jl_typeis(rhs, eltype)) { JL_GC_PUSH1(&rhs); @@ -419,7 +419,6 @@ JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, j jl_type_error("memoryrefset!", eltype, rhs); JL_GC_POP(); } - const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; if (layout->flags.arrayelem_isboxed) { assert((char*)m.ptr_or_offset - (char*)m.mem->ptr < sizeof(jl_value_t*) * m.mem->length); if (isatomic) @@ -449,7 +448,7 @@ JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, j } if (layout->size != 0) { assert(data - (char*)m.mem->ptr < layout->size * m.mem->length); - int needlock = isatomic && layout->size > MAX_ATOMIC_SIZE; + int needlock = layout->flags.arrayelem_islocked; size_t fsz = jl_datatype_size((jl_datatype_t*)jl_typeof(rhs)); // need to shrink-wrap the final copy if (isatomic && !needlock) { jl_atomic_store_bits(data, rhs, fsz); diff --git a/src/gf.c b/src/gf.c index 17d5695503cf3..bc8ee029bfc1b 100644 --- a/src/gf.c +++ b/src/gf.c @@ -27,6 +27,8 @@ extern "C" { _Atomic(int) allow_new_worlds = 1; JL_DLLEXPORT _Atomic(size_t) jl_world_counter = 1; // uses atomic acquire/release jl_mutex_t world_counter_lock; +jl_methtable_t *jl_method_table; + JL_DLLEXPORT size_t jl_get_world_counter(void) JL_NOTSAFEPOINT { jl_task_t *ct = jl_current_task; @@ -41,31 +43,41 @@ JL_DLLEXPORT size_t jl_get_tls_world_age(void) JL_NOTSAFEPOINT } // Compute the maximum number of times to unroll Varargs{T}, based on -// m->max_varargs (if specified) or a heuristic based on the maximum -// number of non-varargs arguments in the provided method table. +// m->max_varargs (if specified) or a heuristic based on the maximum number of +// non-varargs arguments for the function type of the method signature. // // If provided, `may_increase` is set to 1 if the returned value is // heuristic-based and has a chance of increasing in the future. static size_t get_max_varargs( jl_method_t *m, - jl_methtable_t *kwmt, - jl_methtable_t *mt, uint8_t *may_increase) JL_NOTSAFEPOINT { size_t max_varargs = 1; if (may_increase != NULL) *may_increase = 0; - if (m->max_varargs != UINT8_MAX) + if (m->max_varargs != UINT8_MAX) { max_varargs = m->max_varargs; - else if (kwmt != NULL && kwmt != jl_type_type_mt && kwmt != jl_nonfunction_mt && kwmt != jl_kwcall_mt) { - if (may_increase != NULL) - *may_increase = 1; // `max_args` can increase as new methods are inserted - - max_varargs = jl_atomic_load_relaxed(&kwmt->max_args) + 2; - if (mt == jl_kwcall_mt) - max_varargs += 2; - max_varargs -= m->nargs; + } + else { + jl_datatype_t *dt1 = jl_nth_argument_datatype(m->sig, 1); + jl_datatype_t *dt; + if (jl_kwcall_type && dt1 == jl_kwcall_type) + dt = jl_nth_argument_datatype(m->sig, 3); + else + dt = dt1; + if (dt != NULL && !jl_is_type_type((jl_value_t*)dt) && dt != jl_kwcall_type) { + if (may_increase != NULL) + *may_increase = 1; // `max_args` can increase as new methods are inserted + + max_varargs = jl_atomic_load_relaxed(&dt->name->max_args) + 2; + if (jl_kwcall_type && dt1 == jl_kwcall_type) + max_varargs += 2; + if (max_varargs > m->nargs) + max_varargs -= m->nargs; + else + max_varargs = 0; + } } return max_varargs; } @@ -104,9 +116,9 @@ void jl_call_tracer(tracer_cb callback, jl_value_t *tracee) /// ----- Definitions for various internal TypeMaps ----- /// -static int8_t jl_cachearg_offset(jl_methtable_t *mt) +static int8_t jl_cachearg_offset(void) { - return mt->offs; + return 0; } /// ----- Insertion logic for special entries ----- /// @@ -274,13 +286,13 @@ JL_DLLEXPORT jl_value_t *jl_specializations_lookup(jl_method_t *m, jl_value_t *t return mi; } -JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *type, size_t world) +JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_value_t *type, size_t world) { // TODO: this is sort of an odd lookup strategy (and the only user of // jl_typemap_assoc_by_type with subtype=0), while normally jl_gf_invoke_lookup would be // expected to be used instead struct jl_typemap_assoc search = {type, world, NULL}; - jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mt->defs), &search, jl_cachearg_offset(mt), /*subtype*/0); + jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&jl_method_table->defs), &search, 0, /*subtype*/0); if (!sf) return jl_nothing; return sf->func.value; @@ -288,54 +300,50 @@ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *typ // ----- MethodInstance specialization instantiation ----- // -jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED +jl_method_t *jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args_t fptr) JL_GC_DISABLED { - jl_sym_t *sname = jl_symbol(name); - if (dt == NULL) { - // Builtins are specially considered available from world 0 - jl_value_t *f = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0); - jl_set_const(jl_core_module, sname, f); - dt = (jl_datatype_t*)jl_typeof(f); - } + jl_value_t *params[2]; + params[0] = dt->name->wrapper; + params[1] = jl_tparam0(jl_anytuple_type); + jl_datatype_t *tuptyp = (jl_datatype_t*)jl_apply_tuple_type_v(params, 2); + + jl_typemap_entry_t *newentry = NULL; + jl_method_t *m = NULL; + JL_GC_PUSH3(&m, &newentry, &tuptyp); - jl_method_t *m = jl_new_method_uninit(jl_core_module); + m = jl_new_method_uninit(jl_core_module); m->name = sname; m->module = jl_core_module; m->isva = 1; m->nargs = 2; jl_atomic_store_relaxed(&m->primary_world, 1); - jl_atomic_store_relaxed(&m->deleted_world, ~(size_t)0); - m->sig = (jl_value_t*)jl_anytuple_type; + jl_atomic_store_relaxed(&m->dispatch_status, METHOD_SIG_LATEST_ONLY | METHOD_SIG_LATEST_WHICH); + m->sig = (jl_value_t*)tuptyp; m->slot_syms = jl_an_empty_string; m->nospecialize = 0; m->nospecialize = ~m->nospecialize; - jl_methtable_t *mt = dt->name->mt; - jl_typemap_entry_t *newentry = NULL; - JL_GC_PUSH2(&m, &newentry); - - newentry = jl_typemap_alloc(jl_anytuple_type, NULL, jl_emptysvec, - (jl_value_t*)m, jl_atomic_load_relaxed(&m->primary_world), jl_atomic_load_relaxed(&m->deleted_world)); - jl_typemap_insert(&mt->defs, (jl_value_t*)mt, newentry, jl_cachearg_offset(mt)); - - jl_method_instance_t *mi = jl_get_specialized(m, (jl_value_t*)jl_anytuple_type, jl_emptysvec); + jl_method_instance_t *mi = jl_get_specialized(m, (jl_value_t*)tuptyp, jl_emptysvec); jl_atomic_store_relaxed(&m->unspecialized, mi); jl_gc_wb(m, mi); jl_code_instance_t *codeinst = jl_new_codeinst(mi, jl_nothing, (jl_value_t*)jl_any_type, (jl_value_t*)jl_any_type, jl_nothing, jl_nothing, 0, 1, ~(size_t)0, 0, jl_nothing, NULL, NULL); - jl_mi_cache_insert(mi, codeinst); jl_atomic_store_relaxed(&codeinst->specptr.fptr1, fptr); jl_atomic_store_relaxed(&codeinst->invoke, jl_fptr_args); + jl_mi_cache_insert(mi, codeinst); - newentry = jl_typemap_alloc(jl_anytuple_type, NULL, jl_emptysvec, + newentry = jl_typemap_alloc(tuptyp, NULL, jl_emptysvec, + (jl_value_t*)m, 1, ~(size_t)0); + jl_typemap_insert(&jl_method_table->defs, (jl_value_t*)jl_method_table, newentry, 0); + + newentry = jl_typemap_alloc(tuptyp, NULL, jl_emptysvec, (jl_value_t*)mi, 1, ~(size_t)0); - jl_typemap_insert(&mt->cache, (jl_value_t*)mt, newentry, 0); + jl_typemap_insert(&jl_method_table->cache->cache, (jl_value_t*)jl_method_table->cache, newentry, 0); - mt->frozen = 1; JL_GC_POP(); - return dt; + return m; } // only relevant for bootstrapping. otherwise fairly broken. @@ -349,7 +357,7 @@ static int emit_codeinst_and_edges(jl_code_instance_t *codeinst) JL_GC_PUSH1(&code); jl_method_instance_t *mi = jl_get_ci_mi(codeinst); jl_method_t *def = mi->def.method; - if (jl_is_string(code) && jl_is_method(def)) + if (jl_is_method(def)) code = (jl_value_t*)jl_uncompress_ir(def, codeinst, (jl_value_t*)code); if (jl_is_code_info(code)) { jl_emit_codeinst_to_jit(codeinst, (jl_code_info_t*)code); @@ -395,7 +403,7 @@ static jl_code_instance_t *jl_method_inferred_with_abi(jl_method_instance_t *mi // returns the inferred source, and may cache the result in mi // if successful, also updates the mi argument to describe the validity of this src // if inference doesn't occur (or can't finish), returns NULL instead -jl_code_instance_t *jl_type_infer(jl_method_instance_t *mi, size_t world, uint8_t source_mode) +jl_code_instance_t *jl_type_infer(jl_method_instance_t *mi, size_t world, uint8_t source_mode, uint8_t trim_mode) { if (jl_typeinf_func == NULL) { if (source_mode == SOURCE_MODE_ABI) @@ -419,11 +427,12 @@ jl_code_instance_t *jl_type_infer(jl_method_instance_t *mi, size_t world, uint8_ return NULL; JL_TIMING(INFERENCE, INFERENCE); jl_value_t **fargs; - JL_GC_PUSHARGS(fargs, 4); + JL_GC_PUSHARGS(fargs, 5); fargs[0] = (jl_value_t*)jl_typeinf_func; fargs[1] = (jl_value_t*)mi; fargs[2] = jl_box_ulong(world); fargs[3] = jl_box_uint8(source_mode); + fargs[4] = jl_box_uint8(trim_mode); int last_errno = errno; #ifdef _OS_WINDOWS_ DWORD last_error = GetLastError(); @@ -450,7 +459,7 @@ jl_code_instance_t *jl_type_infer(jl_method_instance_t *mi, size_t world, uint8_ // allocate another bit for the counter. ct->reentrant_timing += 0b10; JL_TRY { - ci = (jl_code_instance_t*)jl_apply(fargs, 4); + ci = (jl_code_instance_t*)jl_apply(fargs, 5); } JL_CATCH { jl_value_t *e = jl_current_exception(ct); @@ -504,12 +513,13 @@ JL_DLLEXPORT jl_code_info_t *jl_gdbcodetyped1(jl_method_instance_t *mi, size_t w ct->world_age = jl_typeinf_world; jl_value_t **fargs; JL_GC_PUSHARGS(fargs, 4); - jl_module_t *CC = (jl_module_t*)jl_get_global(jl_core_module, jl_symbol("Compiler")); + jl_module_t *CC = (jl_module_t*)jl_get_global_value(jl_core_module, jl_symbol("Compiler"), ct->world_age); if (CC != NULL && jl_is_module(CC)) { - fargs[0] = jl_get_global(CC, jl_symbol("NativeInterpreter"));; + JL_GC_PROMISE_ROOTED(CC); + fargs[0] = jl_get_global_value(CC, jl_symbol("NativeInterpreter"), ct->world_age); fargs[1] = jl_box_ulong(world); fargs[1] = jl_apply(fargs, 2); - fargs[0] = jl_get_global(CC, jl_symbol("typeinf_code")); + fargs[0] = jl_get_global_value(CC, jl_symbol("typeinf_code"), ct->world_age); fargs[2] = (jl_value_t*)mi; fargs[3] = jl_true; ci = (jl_code_info_t*)jl_apply(fargs, 4); @@ -546,7 +556,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( { jl_value_t *owner = jl_nothing; // TODO: owner should be arg jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); - while (codeinst) { + for (; codeinst; codeinst = jl_atomic_load_relaxed(&codeinst->next)) { if (jl_atomic_load_relaxed(&codeinst->min_world) == min_world && jl_atomic_load_relaxed(&codeinst->max_world) == max_world && jl_egal(codeinst->owner, owner) && @@ -564,7 +574,6 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( if (e && jl_egal((jl_value_t*)e, (jl_value_t*)edges)) return codeinst; } - codeinst = jl_atomic_load_relaxed(&codeinst->next); } codeinst = jl_new_codeinst( mi, owner, rettype, (jl_value_t*)jl_any_type, NULL, NULL, @@ -585,8 +594,8 @@ JL_DLLEXPORT int jl_mi_cache_has_ci(jl_method_instance_t *mi, return 0; } -// look for something with an egal ABI and properties that is already in the JIT (compiled=true) or simply in the cache (compiled=false) -JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, int compiled) JL_NOTSAFEPOINT +// look for something with an egal ABI and properties that is already in the JIT for a whole edge (target_world=0) or can be added to the JIT with new source just for target_world. +JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, size_t target_world) JL_NOTSAFEPOINT { jl_value_t *def = ci->def; jl_method_instance_t *mi = jl_get_ci_mi(ci); @@ -598,9 +607,9 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPA while (codeinst) { if (codeinst != ci && jl_atomic_load_relaxed(&codeinst->inferred) != NULL && - (!compiled || jl_atomic_load_relaxed(&codeinst->invoke) != NULL) && - jl_atomic_load_relaxed(&codeinst->min_world) <= min_world && - jl_atomic_load_relaxed(&codeinst->max_world) >= max_world && + (target_world ? 1 : jl_atomic_load_relaxed(&codeinst->invoke) != NULL) && + jl_atomic_load_relaxed(&codeinst->min_world) <= (target_world ? target_world : min_world) && + jl_atomic_load_relaxed(&codeinst->max_world) >= (target_world ? target_world : max_world) && jl_egal(codeinst->def, def) && jl_egal(codeinst->owner, owner) && jl_egal(codeinst->rettype, rettype)) { @@ -608,7 +617,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPA } codeinst = jl_atomic_load_relaxed(&codeinst->next); } - return (jl_code_instance_t*)jl_nothing; + return ci; } @@ -769,7 +778,117 @@ JL_DLLEXPORT int jl_mi_try_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, return ret; } -int foreach_mtable_in_module( +enum top_typename_facts { + EXACTLY_ANY = 1 << 0, + HAVE_TYPE = 1 << 1, + EXACTLY_TYPE = 1 << 2, + HAVE_FUNCTION = 1 << 3, + EXACTLY_FUNCTION = 1 << 4, + HAVE_KWCALL = 1 << 5, + EXACTLY_KWCALL = 1 << 6, + SHORT_TUPLE = 1 << 7, +}; + +static void foreach_top_nth_typename(void (*f)(jl_typename_t*, int, void*), jl_value_t *a JL_PROPAGATES_ROOT, int n, unsigned *facts, void *env) +{ + if (jl_is_datatype(a)) { + if (n <= 0) { + jl_datatype_t *dt = ((jl_datatype_t*)a); + if (dt->name == jl_type_typename) { // key Type{T} on T instead of Type + *facts |= HAVE_TYPE; + foreach_top_nth_typename(f, jl_tparam0(a), -1, facts, env); + } + else if (dt == jl_function_type) { + if (n == -1) // key Type{>:Function} as Type instead of Function + *facts |= EXACTLY_TYPE; // HAVE_TYPE is already set + else + *facts |= HAVE_FUNCTION | EXACTLY_FUNCTION; + } + else if (dt == jl_any_type) { + if (n == -1) // key Type{>:Any} and kinds as Type instead of Any + *facts |= EXACTLY_TYPE; // HAVE_TYPE is already set + else + *facts |= EXACTLY_ANY; + } + else if (dt == jl_kwcall_type) { + if (n == -1) // key Type{>:typeof(kwcall)} as exactly kwcall + *facts |= EXACTLY_KWCALL; + else + *facts |= HAVE_KWCALL; + } + else { + while (1) { + jl_datatype_t *super = dt->super; + if (super == jl_function_type) { + *facts |= HAVE_FUNCTION; + break; + } + if (super == jl_any_type || super->super == dt) + break; + dt = super; + } + f(dt->name, 1, env); + } + } + else if (jl_is_tuple_type(a)) { + if (jl_nparams(a) >= n) + foreach_top_nth_typename(f, jl_tparam(a, n - 1), 0, facts, env); + else + *facts |= SHORT_TUPLE; + } + } + else if (jl_is_typevar(a)) { + foreach_top_nth_typename(f, ((jl_tvar_t*)a)->ub, n, facts, env); + } + else if (jl_is_unionall(a)) { + foreach_top_nth_typename(f, ((jl_unionall_t*)a)->body, n, facts, env); + } + else if (jl_is_uniontype(a)) { + jl_uniontype_t *u = (jl_uniontype_t*)a; + foreach_top_nth_typename(f, u->a, n, facts, env); + foreach_top_nth_typename(f, u->b, n, facts, env); + } +} + +// Inspect type `argtypes` for all backedge keys that might be relevant to it, splitting it +// up on some commonly observed patterns to make a better distribution. +// (It could do some of that balancing automatically, but for now just hard-codes kwcall.) +// Along the way, record some facts about what was encountered, so that those additional +// calls can be added later if needed for completeness. +// The `int explct` argument instructs the caller if the callback is due to an exactly +// encountered type or if it rather encountered a subtype. +// This is not capable of walking to all top-typenames for an explicitly encountered +// Function or Any, so the caller a fallback that can scan the entire in that case. +// We do not de-duplicate calls when encountering a Union. +static int jl_foreach_top_typename_for(void (*f)(jl_typename_t*, int, void*), jl_value_t *argtypes JL_PROPAGATES_ROOT, int all_subtypes, void *env) +{ + unsigned facts = 0; + foreach_top_nth_typename(f, argtypes, 1, &facts, env); + if (facts & HAVE_KWCALL) { + // split kwcall on the 3rd argument instead, using the same logic + unsigned kwfacts = 0; + foreach_top_nth_typename(f, argtypes, 3, &kwfacts, env); + // copy kwfacts to original facts + if (kwfacts & SHORT_TUPLE) + kwfacts |= (all_subtypes ? EXACTLY_ANY : EXACTLY_KWCALL); + facts |= kwfacts; + } + if (all_subtypes && (facts & (EXACTLY_FUNCTION | EXACTLY_TYPE | EXACTLY_ANY))) + // flag that we have an explct match than is necessitating a full table scan + return 0; + // or inform caller of only which supertypes are applicable + if (facts & HAVE_FUNCTION) + f(jl_function_type->name, facts & EXACTLY_FUNCTION ? 1 : 0, env); + if (facts & HAVE_TYPE) + f(jl_type_typename, facts & EXACTLY_TYPE ? 1 : 0, env); + if (facts & (HAVE_KWCALL | EXACTLY_KWCALL)) + f(jl_kwcall_type->name, facts & EXACTLY_KWCALL ? 1 : 0, env); + f(jl_any_type->name, facts & EXACTLY_ANY ? 1 : 0, env); + return 1; +} + + +static int foreach_mtable_in_module( jl_module_t *m, int (*visit)(jl_methtable_t *mt, void *env), void *env) @@ -780,22 +899,9 @@ int foreach_mtable_in_module( if ((void*)b == jl_nothing) break; jl_sym_t *name = b->globalref->name; - jl_value_t *v = jl_get_binding_value_if_const(b); + jl_value_t *v = jl_get_latest_binding_value_if_const(b); if (v) { - jl_value_t *uw = jl_unwrap_unionall(v); - if (jl_is_datatype(uw)) { - jl_typename_t *tn = ((jl_datatype_t*)uw)->name; - if (tn->module == m && tn->name == name && tn->wrapper == v) { - // this is the original/primary binding for the type (name/wrapper) - jl_methtable_t *mt = tn->mt; - if (mt != NULL && (jl_value_t*)mt != jl_nothing && mt != jl_type_type_mt && mt != jl_nonfunction_mt) { - assert(mt->module == m); - if (!visit(mt, env)) - return 0; - } - } - } - else if (jl_is_module(v)) { + if (jl_is_module(v)) { jl_module_t *child = (jl_module_t*)v; if (child != m && child->parent == m && child->name == name) { // this is the original/primary binding for the submodule @@ -805,9 +911,7 @@ int foreach_mtable_in_module( } else if (jl_is_mtable(v)) { jl_methtable_t *mt = (jl_methtable_t*)v; - if (mt->module == m && mt->name == name) { - // this is probably an external method table here, so let's - // assume so as there is no way to precisely distinguish them + if (mt && mt != jl_method_table) { if (!visit(mt, env)) return 0; } @@ -818,37 +922,23 @@ int foreach_mtable_in_module( return 1; } -int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), void *env) + +int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), jl_array_t *mod_array, void *env) { - if (!visit(jl_type_type_mt, env)) + if (!visit(jl_method_table, env)) return 0; - if (!visit(jl_nonfunction_mt, env)) - return 0; - jl_array_t *mod_array = jl_get_loaded_modules(); if (mod_array) { - JL_GC_PUSH1(&mod_array); - int i; - for (i = 0; i < jl_array_nrows(mod_array); i++) { + for (size_t i = 0; i < jl_array_nrows(mod_array); i++) { jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); assert(jl_is_module(m)); if (m->parent == m) // some toplevel modules (really just Base) aren't actually - if (!foreach_mtable_in_module(m, visit, env)) { - JL_GC_POP(); + if (!foreach_mtable_in_module(m, visit, env)) return 0; - } } - JL_GC_POP(); - } - else { - if (!foreach_mtable_in_module(jl_main_module, visit, env)) - return 0; - if (!foreach_mtable_in_module(jl_core_module, visit, env)) - return 0; } return 1; } - jl_function_t *jl_typeinf_func JL_GLOBALLY_ROOTED = NULL; JL_DLLEXPORT size_t jl_typeinf_world = 1; @@ -932,7 +1022,7 @@ static jl_value_t *inst_varargp_in_env(jl_value_t *decl, jl_svec_t *sparams) return vm; } -static jl_value_t *ml_matches(jl_methtable_t *mt, +static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, jl_tupletype_t *type, int lim, int include_ambiguous, int intersections, size_t world, int cache_result, size_t *min_valid, size_t *max_valid, int *ambig); @@ -1139,9 +1229,10 @@ static void jl_compilation_sig( // and the types we find should be bigger. if (np >= nspec && jl_va_tuple_kind((jl_datatype_t*)decl) == JL_VARARG_UNBOUND) { if (!*newparams) *newparams = tt->parameters; - if (max_varargs > 0) { + if (max_varargs > 0 && nspec >= 2) { type_i = jl_svecref(*newparams, nspec - 2); - } else { + } + else { // If max varargs is zero, always specialize to (Any...) since // there is no preceding parameter to use for `type_i` type_i = jl_bottom_type; @@ -1216,15 +1307,11 @@ JL_DLLEXPORT int jl_isa_compileable_sig( if (definition->isva) { unsigned nspec_min = nargs + 1; // min number of arg values (including tail vararg) unsigned nspec_max = INT32_MAX; // max number of arg values (including tail vararg) - jl_methtable_t *mt = jl_method_table_for(decl); - jl_methtable_t *kwmt = mt == jl_kwcall_mt ? jl_kwmethod_table_for(decl) : mt; - if ((jl_value_t*)mt != jl_nothing) { - // try to refine estimate of min and max - uint8_t heuristic_used = 0; - nspec_max = nspec_min = nargs + get_max_varargs(definition, kwmt, mt, &heuristic_used); - if (heuristic_used) - nspec_max = INT32_MAX; // new methods may be added, increasing nspec_min later - } + // try to refine estimate of min and max + uint8_t heuristic_used = 0; + nspec_max = nspec_min = nargs + get_max_varargs(definition, &heuristic_used); + if (heuristic_used) + nspec_max = INT32_MAX; // new methods may be added, increasing nspec_min later int isunbound = (jl_va_tuple_kind((jl_datatype_t*)decl) == JL_VARARG_UNBOUND); if (jl_is_vararg(jl_tparam(type, np - 1))) { if (!isunbound || np < nspec_min || np > nspec_max) @@ -1413,34 +1500,52 @@ static inline jl_typemap_entry_t *lookup_leafcache(jl_genericmemory_t *leafcache return NULL; } jl_method_instance_t *cache_method( - jl_methtable_t *mt, _Atomic(jl_typemap_t*) *cache, jl_value_t *parent JL_PROPAGATES_ROOT, + jl_methtable_t *mt, jl_methcache_t *mc, _Atomic(jl_typemap_t*) *cache, jl_value_t *parent JL_PROPAGATES_ROOT, jl_tupletype_t *tt, // the original tupletype of the signature jl_method_t *definition, size_t world, size_t min_valid, size_t max_valid, jl_svec_t *sparams) { - // caller must hold the mt->writelock + // caller must hold the parent->writelock, which this releases // short-circuit (now that we hold the lock) if this entry is already present - int8_t offs = mt ? jl_cachearg_offset(mt) : 1; + int8_t offs = mc ? jl_cachearg_offset() : 1; { // scope block - if (mt) { - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + if (mc) { + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); - if (entry) + if (entry) { + if (mc) JL_UNLOCK(&mc->writelock); return entry->func.linfo; + } } struct jl_typemap_assoc search = {(jl_value_t*)tt, world, NULL}; jl_typemap_t *cacheentry = jl_atomic_load_relaxed(cache); assert(cacheentry != NULL); jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(cacheentry, &search, offs, /*subtype*/1); - if (entry && entry->func.value) + if (entry && entry->func.value) { + if (mc) JL_UNLOCK(&mc->writelock); return entry->func.linfo; + } + } + + jl_method_instance_t *newmeth = NULL; + if (definition->sig == (jl_value_t*)jl_anytuple_type && definition != jl_opaque_closure_method && !definition->is_for_opaque_closure) { + newmeth = jl_atomic_load_relaxed(&definition->unspecialized); + if (newmeth != NULL) { // handle builtin methods de-specialization (for invoke, or if the global cache entry somehow gets lost) + jl_tupletype_t *cachett = (jl_tupletype_t*)newmeth->specTypes; + assert(cachett != jl_anytuple_type); + jl_typemap_entry_t *newentry = jl_typemap_alloc(cachett, NULL, jl_emptysvec, (jl_value_t*)newmeth, min_valid, max_valid); + JL_GC_PUSH1(&newentry); + jl_typemap_insert(cache, parent, newentry, offs); + JL_GC_POP(); + if (mc) JL_UNLOCK(&mc->writelock); + return newmeth; + } } jl_value_t *temp = NULL; jl_value_t *temp2 = NULL; jl_value_t *temp3 = NULL; - jl_method_instance_t *newmeth = NULL; jl_svec_t *newparams = NULL; JL_GC_PUSH5(&temp, &temp2, &temp3, &newmeth, &newparams); @@ -1448,8 +1553,7 @@ jl_method_instance_t *cache_method( // so that we can minimize the number of required cache entries. int cache_with_orig = 1; jl_tupletype_t *compilationsig = tt; - jl_methtable_t *kwmt = mt == jl_kwcall_mt ? jl_kwmethod_table_for(definition->sig) : mt; - intptr_t max_varargs = get_max_varargs(definition, kwmt, mt, NULL); + intptr_t max_varargs = get_max_varargs(definition, NULL); jl_compilation_sig(tt, sparams, definition, max_varargs, &newparams); if (newparams) { temp2 = jl_apply_tuple_type(newparams, 1); @@ -1479,13 +1583,16 @@ jl_method_instance_t *cache_method( if (newmeth->cache_with_orig) cache_with_orig = 1; + // Capture world counter at start to detect races + size_t current_world = mc ? jl_atomic_load_acquire(&jl_world_counter) : ~(size_t)0; + jl_tupletype_t *cachett = tt; - jl_svec_t* guardsigs = jl_emptysvec; + jl_svec_t *guardsigs = jl_emptysvec; if (!cache_with_orig && mt) { // now examine what will happen if we chose to use this sig in the cache size_t min_valid2 = 1; size_t max_valid2 = ~(size_t)0; - temp = ml_matches(mt, compilationsig, MAX_UNSPECIALIZED_CONFLICTS, 1, 1, world, 0, &min_valid2, &max_valid2, NULL); + temp = ml_matches(mt, mc, compilationsig, MAX_UNSPECIALIZED_CONFLICTS, 1, 1, world, 0, &min_valid2, &max_valid2, NULL); int guards = 0; if (temp == jl_nothing) { cache_with_orig = 1; @@ -1533,7 +1640,7 @@ jl_method_instance_t *cache_method( guards++; // alternative approach: insert sentinel entry //jl_typemap_insert(cache, parent, (jl_tupletype_t*)matc->spec_types, - // NULL, jl_emptysvec, /*guard*/NULL, jl_cachearg_offset(mt), other->min_world, other->max_world); + // NULL, jl_emptysvec, /*guard*/NULL, jl_cachearg_offset(), other->min_world, other->max_world); } } assert(guards == jl_svec_len(guardsigs)); @@ -1550,6 +1657,10 @@ jl_method_instance_t *cache_method( } } + int unconstrained_max = max_valid == ~(size_t)0; + if (max_valid > current_world) + max_valid = current_world; + // now scan `cachett` and ensure that `Type{T}` in the cache will be matched exactly by `typeof(T)` // and also reduce the complexity of rejecting this entry in the cache // by replacing non-simple types with jl_any_type to build a new `type` @@ -1593,7 +1704,7 @@ jl_method_instance_t *cache_method( jl_typemap_entry_t *newentry = jl_typemap_alloc(cachett, simplett, guardsigs, (jl_value_t*)newmeth, min_valid, max_valid); temp = (jl_value_t*)newentry; - if (mt && cachett == tt && jl_svec_len(guardsigs) == 0 && tt->hash && !tt->hasfreetypevars) { + if (mc && cachett == tt && jl_svec_len(guardsigs) == 0 && tt->hash && !tt->hasfreetypevars) { // we check `tt->hash` exists, since otherwise the NamedTuple // constructor and `structdiff` method pollutes this lookup with a lot // of garbage in the linear table search @@ -1606,68 +1717,160 @@ jl_method_instance_t *cache_method( jl_cache_type_(tt); JL_UNLOCK(&typecache_lock); // Might GC } - jl_genericmemory_t *oldcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *oldcache = jl_atomic_load_relaxed(&mc->leafcache); jl_typemap_entry_t *old = (jl_typemap_entry_t*)jl_eqtable_get(oldcache, (jl_value_t*)tt, jl_nothing); jl_atomic_store_relaxed(&newentry->next, old); jl_gc_wb(newentry, old); - jl_genericmemory_t *newcache = jl_eqtable_put(jl_atomic_load_relaxed(&mt->leafcache), (jl_value_t*)tt, (jl_value_t*)newentry, NULL); + jl_genericmemory_t *newcache = jl_eqtable_put(jl_atomic_load_relaxed(&mc->leafcache), (jl_value_t*)tt, (jl_value_t*)newentry, NULL); if (newcache != oldcache) { - jl_atomic_store_release(&mt->leafcache, newcache); - jl_gc_wb(mt, newcache); + jl_atomic_store_release(&mc->leafcache, newcache); + jl_gc_wb(mc, newcache); } } else { jl_typemap_insert(cache, parent, newentry, offs); + if (mt) { + jl_datatype_t *dt = jl_nth_argument_datatype((jl_value_t*)tt, 1); + if (dt) { + jl_typename_t *tn = dt->name; + int cache_entry_count = jl_atomic_load_relaxed(&tn->cache_entry_count); + if (cache_entry_count < 31) + jl_atomic_store_relaxed(&tn->cache_entry_count, cache_entry_count + 1); + } + } + } + if (mc) { + JL_UNLOCK(&mc->writelock); + + // Only set METHOD_SIG_LATEST_ONLY on method instance if method does NOT have the bit, no guards required, and min_valid == primary_world + int should_set_dispatch_status = !(jl_atomic_load_relaxed(&definition->dispatch_status) & METHOD_SIG_LATEST_ONLY) && + (!cache_with_orig && jl_svec_len(guardsigs) == 0) && + min_valid == jl_atomic_load_relaxed(&definition->primary_world) && + !(jl_atomic_load_relaxed(&newmeth->dispatch_status) & METHOD_SIG_LATEST_ONLY); + + // Combined trylock for both dispatch_status setting and max_world restoration + if ((should_set_dispatch_status || unconstrained_max) && + jl_atomic_load_relaxed(&jl_world_counter) == current_world) { + JL_LOCK(&world_counter_lock); + if (jl_atomic_load_relaxed(&jl_world_counter) == current_world) { + if (should_set_dispatch_status) { + jl_atomic_store_relaxed(&newmeth->dispatch_status, METHOD_SIG_LATEST_ONLY); + } + if (unconstrained_max) { + jl_atomic_store_relaxed(&newentry->max_world, ~(size_t)0); + } + } + JL_UNLOCK(&world_counter_lock); + } } JL_GC_POP(); return newmeth; } -static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_value_t *mt, size_t world, size_t *min_valid, size_t *max_valid); +static void _jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) JL_NOTSAFEPOINT +{ + if (jl_atomic_load_relaxed(&ci->max_world) != validated_world) + return; + jl_atomic_store_relaxed(&ci->max_world, ~(size_t)0); + jl_svec_t *edges = jl_atomic_load_relaxed(&ci->edges); + for (size_t i = 0; i < jl_svec_len(edges); i++) { + jl_value_t *edge = jl_svecref(edges, i); + if (!jl_is_code_instance(edge)) + continue; + _jl_promote_ci_to_current((jl_code_instance_t *)edge, validated_world); + } +} + +JL_DLLEXPORT void jl_promote_cis_to_current(jl_code_instance_t **cis, size_t n, size_t validated_world) +{ + size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); + // No need to acquire the lock if we've been invalidated anyway + if (current_world > validated_world) + return; + JL_LOCK(&world_counter_lock); + current_world = jl_atomic_load_relaxed(&jl_world_counter); + if (current_world == validated_world) { + for (size_t i = 0; i < n; i++) { + _jl_promote_ci_to_current(cis[i], validated_world); + } + } + JL_UNLOCK(&world_counter_lock); +} + +JL_DLLEXPORT void jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) +{ + jl_promote_cis_to_current(&ci, 1, validated_world); +} + +JL_DLLEXPORT void jl_promote_mi_to_current(jl_method_instance_t *mi, size_t min_world, size_t validated_world) +{ + size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); + // No need to acquire the lock if we've been invalidated anyway + if (current_world > validated_world) + return; + // Only set METHOD_SIG_LATEST_ONLY on method instance if method does NOT have the bit and min_valid == primary_world + jl_method_t *definition = mi->def.method; + if ((jl_atomic_load_relaxed(&definition->dispatch_status) & METHOD_SIG_LATEST_ONLY) || + min_world != jl_atomic_load_relaxed(&definition->primary_world) || + (jl_atomic_load_relaxed(&mi->dispatch_status) & METHOD_SIG_LATEST_ONLY)) + return; + JL_LOCK(&world_counter_lock); + current_world = jl_atomic_load_relaxed(&jl_world_counter); + if (current_world == validated_world) { + jl_atomic_store_relaxed(&mi->dispatch_status, METHOD_SIG_LATEST_ONLY); + } + JL_UNLOCK(&world_counter_lock); +} + +static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_methtable_t *mt, size_t world, int cache, size_t *min_valid, size_t *max_valid); + +JL_DLLEXPORT jl_typemap_entry_t *jl_mt_find_cache_entry(jl_methcache_t *mc JL_PROPAGATES_ROOT, jl_datatype_t *tt JL_MAYBE_UNROOTED JL_ROOTS_TEMPORARILY, size_t world) +{ // exported only for debugging purposes, not for casual use + if (tt->isdispatchtuple) { + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); + jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); + if (entry) + return entry; + } + JL_GC_PUSH1(&tt); + struct jl_typemap_assoc search = {(jl_value_t*)tt, world, NULL}; + jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mc->cache), &search, jl_cachearg_offset(), /*subtype*/1); + JL_GC_POP(); + return entry; +} -static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt JL_PROPAGATES_ROOT, jl_datatype_t *tt JL_MAYBE_UNROOTED, size_t world) +static jl_method_instance_t *jl_mt_assoc_by_type(jl_methcache_t *mc JL_PROPAGATES_ROOT, jl_datatype_t *tt JL_MAYBE_UNROOTED, size_t world) { - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); - jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); + jl_typemap_entry_t *entry = jl_mt_find_cache_entry(mc, tt, world); if (entry) return entry->func.linfo; + assert(tt->isdispatchtuple || tt->hasfreetypevars); JL_TIMING(METHOD_LOOKUP_SLOW, METHOD_LOOKUP_SLOW); jl_method_match_t *matc = NULL; JL_GC_PUSH2(&tt, &matc); - JL_LOCK(&mt->writelock); - assert(tt->isdispatchtuple || tt->hasfreetypevars); + JL_LOCK(&mc->writelock); jl_method_instance_t *mi = NULL; - if (tt->isdispatchtuple) { - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); - jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); - if (entry) - mi = entry->func.linfo; - } - - if (!mi) { - struct jl_typemap_assoc search = {(jl_value_t*)tt, world, NULL}; - jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mt->cache), &search, jl_cachearg_offset(mt), /*subtype*/1); - if (entry) - mi = entry->func.linfo; - } - + entry = jl_mt_find_cache_entry(mc, tt, world); + if (entry) + mi = entry->func.linfo; if (!mi) { size_t min_valid = 0; size_t max_valid = ~(size_t)0; - matc = _gf_invoke_lookup((jl_value_t*)tt, jl_nothing, world, &min_valid, &max_valid); + matc = _gf_invoke_lookup((jl_value_t*)tt, jl_method_table, world, 0, &min_valid, &max_valid); if (matc) { jl_method_t *m = matc->method; jl_svec_t *env = matc->sparams; - mi = cache_method(mt, &mt->cache, (jl_value_t*)mt, tt, m, world, min_valid, max_valid, env); + mi = cache_method(jl_method_table, mc, &mc->cache, (jl_value_t*)mc, tt, m, world, min_valid, max_valid, env); + JL_GC_POP(); + return mi; } } - JL_UNLOCK(&mt->writelock); + JL_UNLOCK(&mc->writelock); JL_GC_POP(); return mi; } - struct matches_env { struct typemap_intersection_env match; jl_typemap_entry_t *newentry; @@ -1678,12 +1881,12 @@ struct matches_env { static int get_intersect_visitor(jl_typemap_entry_t *oldentry, struct typemap_intersection_env *closure0) { struct matches_env *closure = container_of(closure0, struct matches_env, match); + jl_method_t *oldmethod = oldentry->func.method; assert(oldentry != closure->newentry && "entry already added"); assert(jl_atomic_load_relaxed(&oldentry->min_world) <= jl_atomic_load_relaxed(&closure->newentry->min_world) && "old method cannot be newer than new method"); - assert(jl_atomic_load_relaxed(&oldentry->max_world) != jl_atomic_load_relaxed(&closure->newentry->min_world) && "method cannot be added at the same time as method deleted"); + //assert(jl_atomic_load_relaxed(&oldentry->max_world) != jl_atomic_load_relaxed(&closure->newentry->min_world) && "method cannot be added at the same time as method deleted"); + assert((jl_atomic_load_relaxed(&oldentry->max_world) == ~(size_t)0)); // don't need to consider other similar methods if this oldentry will always fully intersect with them and dominates all of them - typemap_slurp_search(oldentry, &closure->match); - jl_method_t *oldmethod = oldentry->func.method; if (closure->match.issubty // e.g. jl_subtype(closure->newentry.sig, oldentry->sig) && jl_subtype(oldmethod->sig, (jl_value_t*)closure->newentry->sig)) { // e.g. jl_type_equal(closure->newentry->sig, oldentry->sig) if (closure->replaced == NULL || jl_atomic_load_relaxed(&closure->replaced->min_world) < jl_atomic_load_relaxed(&oldentry->min_world)) @@ -1691,11 +1894,25 @@ static int get_intersect_visitor(jl_typemap_entry_t *oldentry, struct typemap_in } if (closure->shadowed == NULL) closure->shadowed = (jl_value_t*)jl_alloc_vec_any(0); + // This should be rarely true (in fact, get_intersect_visitor should be + // rarely true), but might as well skip the rest of the scan fast anyways + // since we can. + if (closure->match.issubty) { + int only = jl_atomic_load_relaxed(&oldmethod->dispatch_status) & METHOD_SIG_LATEST_ONLY; + if (only) { + size_t len = jl_array_nrows(closure->shadowed); + if (len > 0) + jl_array_del_end((jl_array_t*)closure->shadowed, len); + jl_array_ptr_1d_push((jl_array_t*)closure->shadowed, (jl_value_t*)oldmethod); + return 0; + } + } jl_array_ptr_1d_push((jl_array_t*)closure->shadowed, (jl_value_t*)oldmethod); + typemap_slurp_search(oldentry, &closure->match); return 1; } -static jl_value_t *get_intersect_matches(jl_typemap_t *defs, jl_typemap_entry_t *newentry, jl_typemap_entry_t **replaced, int8_t offs, size_t world) +static jl_value_t *get_intersect_matches(jl_typemap_t *defs, jl_typemap_entry_t *newentry, jl_typemap_entry_t **replaced, size_t world) { jl_tupletype_t *type = newentry->sig; jl_tupletype_t *ttypes = (jl_tupletype_t*)jl_unwrap_unionall((jl_value_t*)type); @@ -1714,7 +1931,7 @@ static jl_value_t *get_intersect_matches(jl_typemap_t *defs, jl_typemap_entry_t /* .ti = */ NULL, /* .env = */ jl_emptysvec, /* .issubty = */ 0}, /* .newentry = */ newentry, /* .shadowed */ NULL, /* .replaced */ NULL}; JL_GC_PUSH3(&env.match.env, &env.match.ti, &env.shadowed); - jl_typemap_intersection_visitor(defs, offs, &env.match); + jl_typemap_intersection_visitor(defs, 0, &env.match); env.match.env = NULL; env.match.ti = NULL; *replaced = env.replaced; @@ -1738,7 +1955,7 @@ static void method_overwrite(jl_typemap_entry_t *newentry, jl_method_t *oldvalue jl_module_t *newmod = method->module; jl_module_t *oldmod = oldvalue->module; jl_datatype_t *dt = jl_nth_argument_datatype(oldvalue->sig, 1); - if (dt == (jl_datatype_t*)jl_typeof(jl_kwcall_func)) + if (jl_kwcall_type && dt == jl_kwcall_type) dt = jl_nth_argument_datatype(oldvalue->sig, 3); int anon = dt && is_anonfn_typename(jl_symbol_name(dt->name->name)); if ((jl_options.warn_overwrite == JL_OPTIONS_WARN_OVERWRITE_ON) || @@ -1764,18 +1981,20 @@ static void method_overwrite(jl_typemap_entry_t *newentry, jl_method_t *oldvalue } } -static void update_max_args(jl_methtable_t *mt, jl_value_t *type) +static void update_max_args(jl_value_t *type) { - if (mt == jl_type_type_mt || mt == jl_nonfunction_mt || mt == jl_kwcall_mt) - return; type = jl_unwrap_unionall(type); + jl_datatype_t *dt = jl_nth_argument_datatype(type, 1); + if (dt == NULL || dt == jl_kwcall_type || jl_is_type_type((jl_value_t*)dt)) + return; + jl_typename_t *tn = dt->name; assert(jl_is_datatype(type)); size_t na = jl_nparams(type); if (jl_va_tuple_kind((jl_datatype_t*)type) == JL_VARARG_UNBOUND) na--; - // update occurs inside mt->writelock - if (na > jl_atomic_load_relaxed(&mt->max_args)) - jl_atomic_store_relaxed(&mt->max_args, na); + // update occurs inside global writelock + if (na > jl_atomic_load_relaxed(&tn->max_args)) + jl_atomic_store_relaxed(&tn->max_args, na); } jl_array_t *_jl_debug_method_invalidation JL_GLOBALLY_ROOTED = NULL; @@ -1819,7 +2038,9 @@ static void invalidate_code_instance(jl_code_instance_t *replaced, size_t max_wo jl_atomic_store_release(&replaced->max_world, max_world); // recurse to all backedges to update their valid range also _invalidate_backedges(replaced_mi, replaced, max_world, depth + 1); - } else { + // TODO: should we visit all forward edges now and delete ourself from all of those lists too? + } + else { assert(jl_atomic_load_relaxed(&replaced->max_world) <= max_world); } JL_UNLOCK(&replaced_mi->def.method->writelock); @@ -1830,49 +2051,172 @@ JL_DLLEXPORT void jl_invalidate_code_instance(jl_code_instance_t *replaced, size invalidate_code_instance(replaced, max_world, 1); } +JL_DLLEXPORT void jl_maybe_log_binding_invalidation(jl_value_t *replaced) +{ + if (_jl_debug_method_invalidation) { + if (replaced) { + jl_array_ptr_1d_push(_jl_debug_method_invalidation, replaced); + } + jl_value_t *loctag = jl_cstr_to_string("jl_maybe_log_binding_invalidation"); + JL_GC_PUSH1(&loctag); + jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); + JL_GC_POP(); + } +} + static void _invalidate_backedges(jl_method_instance_t *replaced_mi, jl_code_instance_t *replaced_ci, size_t max_world, int depth) { - jl_array_t *backedges = replaced_mi->backedges; - if (backedges) { - // invalidate callers (if any) + uint8_t recursion_flags = 0; + jl_array_t *backedges = jl_mi_get_backedges_mutate(replaced_mi, &recursion_flags); + if (!backedges) + return; + // invalidate callers (if any) + if (!replaced_ci) { + // We know all backedges are deleted - clear them eagerly + // Clears both array and flags replaced_mi->backedges = NULL; - JL_GC_PUSH1(&backedges); - size_t i = 0, l = jl_array_nrows(backedges); - size_t ins = 0; - jl_code_instance_t *replaced; - while (i < l) { - jl_value_t *invokesig = NULL; - i = get_next_edge(backedges, i, &invokesig, &replaced); - JL_GC_PROMISE_ROOTED(replaced); // propagated by get_next_edge from backedges - if (replaced_ci) { - // If we're invalidating a particular codeinstance, only invalidate - // this backedge it actually has an edge for our codeinstance. - jl_svec_t *edges = jl_atomic_load_relaxed(&replaced->edges); - for (size_t j = 0; j < jl_svec_len(edges); ++j) { - jl_value_t *edge = jl_svecref(edges, j); - if (edge == (jl_value_t*)replaced_mi || edge == (jl_value_t*)replaced_ci) - goto found; - } - // Keep this entry in the backedge list, but compact it - ins = set_next_edge(backedges, ins, invokesig, replaced); - continue; - found:; + jl_atomic_fetch_and_relaxed(&replaced_mi->flags, ~MI_FLAG_BACKEDGES_ALL); + } + JL_GC_PUSH1(&backedges); + size_t i = 0, l = jl_array_nrows(backedges); + size_t ins = 0; + jl_code_instance_t *replaced; + while (i < l) { + jl_value_t *invokesig = NULL; + i = get_next_edge(backedges, i, &invokesig, &replaced); + if (!replaced) { + ins = i; + continue; + } + JL_GC_PROMISE_ROOTED(replaced); // propagated by get_next_edge from backedges + if (replaced_ci) { + // If we're invalidating a particular codeinstance, only invalidate + // this backedge it actually has an edge for our codeinstance. + jl_svec_t *edges = jl_atomic_load_relaxed(&replaced->edges); + for (size_t j = 0; j < jl_svec_len(edges); ++j) { + jl_value_t *edge = jl_svecref(edges, j); + if (edge == (jl_value_t*)replaced_mi || edge == (jl_value_t*)replaced_ci) + goto found; } - invalidate_code_instance(replaced, max_world, depth); + ins = set_next_edge(backedges, ins, invokesig, replaced); + continue; + found:; + ins = clear_next_edge(backedges, ins, invokesig, replaced); + jl_atomic_fetch_or(&replaced_mi->flags, MI_FLAG_BACKEDGES_DIRTY); + /* fallthrough */ + } + invalidate_code_instance(replaced, max_world, depth); + if (replaced_ci && !replaced_mi->backedges) { + // Fast-path early out. If `invalidate_code_instance` invalidated + // the entire mi via a recursive edge, there's no point to keep + // iterating - they'll already have been invalidated. + break; + } + } + if (replaced_ci) + jl_mi_done_backedges(replaced_mi, recursion_flags); + JL_GC_POP(); +} + +static int jl_type_intersection2(jl_value_t *t1, jl_value_t *t2, jl_value_t **isect JL_REQUIRE_ROOTED_SLOT, jl_value_t **isect2 JL_REQUIRE_ROOTED_SLOT) +{ + *isect2 = NULL; + int is_subty = 0; + *isect = jl_type_intersection_env_s(t1, t2, NULL, &is_subty); + if (*isect == jl_bottom_type) + return 0; + if (is_subty) + return 1; + // TODO: sometimes type intersection returns types with free variables + if (jl_has_free_typevars(t1) || jl_has_free_typevars(t2)) + return 1; + // determine if type-intersection can be convinced to give a better, non-bad answer + // if the intersection was imprecise, see if we can do better by switching the types + *isect2 = jl_type_intersection(t2, t1); + if (*isect2 == jl_bottom_type) { + *isect = jl_bottom_type; + *isect2 = NULL; + return 0; + } + if (jl_types_egal(*isect2, *isect)) { + *isect2 = NULL; + } + return 1; +} + + +// check if `type` is replacing `m` with an ambiguity here, given other methods in `d` that already match it +static int is_replacing(char ambig, jl_value_t *type, jl_method_t *m, jl_method_t *const *d, size_t n, jl_value_t *isect, jl_value_t *isect2, char *morespec) +{ + size_t k; + for (k = 0; k < n; k++) { + jl_method_t *m2 = d[k]; + // see if m2 also fully covered this intersection + if (m == m2 || !(jl_subtype(isect, m2->sig) || (isect2 && jl_subtype(isect2, m2->sig)))) + continue; + if (morespec[k]) + // not actually shadowing this--m2 will still be better + return 0; + // if type is not more specific than m (thus now dominating it) + // then there is a new ambiguity here, + // since m2 was also a previous match over isect, + // see if m was previously dominant over all m2 + // or if this was already ambiguous before + if (ambig && !jl_type_morespecific(m->sig, m2->sig)) { + // m and m2 were previously ambiguous over the full intersection of mi with type, and will still be ambiguous with addition of type + return 0; + } + } + return 1; +} + +static int _invalidate_dispatch_backedges(jl_method_instance_t *mi, jl_value_t *type, jl_method_t *m, + jl_method_t *const *d, size_t n, int replaced_dispatch, int ambig, + size_t max_world, char *morespec) +{ + uint8_t backedge_recursion_flags = 0; + jl_array_t *backedges = jl_mi_get_backedges_mutate(mi, &backedge_recursion_flags); + if (!backedges) + return 0; + size_t ib = 0, insb = 0, nb = jl_array_nrows(backedges); + jl_value_t *invokeTypes; + jl_code_instance_t *caller; + int invalidated_any = 0; + while (mi->backedges && ib < nb) { + ib = get_next_edge(backedges, ib, &invokeTypes, &caller); + if (!caller) { + insb = ib; + continue; } - if (replaced_ci && ins != 0) { - jl_array_del_end(backedges, l - ins); - // If we're only invalidating one ci, we don't know which ci any particular - // backedge was for, so we can't delete them. Put them back. - replaced_mi->backedges = backedges; - jl_gc_wb(replaced_mi, backedges); + JL_GC_PROMISE_ROOTED(caller); // propagated by get_next_edge from backedges + int replaced_edge; + if (invokeTypes) { + // n.b. normally we must have mi.specTypes <: invokeTypes <: m.sig (though it might not strictly hold), so we only need to check the other subtypes + if (jl_egal(invokeTypes, jl_get_ci_mi(caller)->def.method->sig)) + replaced_edge = 0; // if invokeTypes == m.sig, then the only way to change this invoke is to replace the method itself + else + replaced_edge = jl_subtype(invokeTypes, type) && is_replacing(ambig, type, m, d, n, invokeTypes, NULL, morespec); + } + else { + replaced_edge = replaced_dispatch; + } + if (replaced_edge) { + invalidate_code_instance(caller, max_world, 1); + insb = clear_next_edge(backedges, insb, invokeTypes, caller); + jl_atomic_fetch_or(&mi->flags, MI_FLAG_BACKEDGES_DIRTY); + invalidated_any = 1; + } + else { + insb = set_next_edge(backedges, insb, invokeTypes, caller); } - JL_GC_POP(); } + jl_mi_done_backedges(mi, backedge_recursion_flags); + return invalidated_any; } // invalidate cached methods that overlap this definition static void invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_world, const char *why) { + // Reset dispatch_status when method instance is replaced JL_LOCK(&replaced_mi->def.method->writelock); _invalidate_backedges(replaced_mi, NULL, max_world, 1); JL_UNLOCK(&replaced_mi->def.method->writelock); @@ -1883,6 +2227,7 @@ static void invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_w jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); JL_GC_POP(); } + jl_atomic_store_relaxed(&replaced_mi->dispatch_status, 0); } // add a backedge from callee to caller @@ -1897,85 +2242,166 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, assert(invokesig == NULL || jl_is_type(invokesig)); JL_LOCK(&callee->def.method->writelock); if (jl_atomic_load_relaxed(&allow_new_worlds)) { - int found = 0; + jl_array_t *backedges = jl_mi_get_backedges(callee); // TODO: use jl_cache_type_(invokesig) like cache_method does to save memory - if (!callee->backedges) { + if (!backedges) { // lazy-init the backedges array - callee->backedges = jl_alloc_vec_any(0); - jl_gc_wb(callee, callee->backedges); + backedges = jl_alloc_vec_any(0); + callee->backedges = backedges; + jl_gc_wb(callee, backedges); } - else { - size_t i = 0, l = jl_array_nrows(callee->backedges); - for (i = 0; i < l; i++) { - // optimized version of while (i < l) i = get_next_edge(callee->backedges, i, &invokeTypes, &mi); - jl_value_t *ciedge = jl_array_ptr_ref(callee->backedges, i); - if (ciedge != (jl_value_t*)caller) - continue; - jl_value_t *invokeTypes = i > 0 ? jl_array_ptr_ref(callee->backedges, i - 1) : NULL; - if (invokeTypes && jl_is_method_instance(invokeTypes)) - invokeTypes = NULL; - if ((invokesig == NULL && invokeTypes == NULL) || - (invokesig && invokeTypes && jl_types_equal(invokesig, invokeTypes))) { - found = 1; - break; - } + push_edge(backedges, invokesig, caller); + } + JL_UNLOCK(&callee->def.method->writelock); +} + + +static int jl_foreach_top_typename_for(void (*f)(jl_typename_t*, int, void*), jl_value_t *argtypes JL_PROPAGATES_ROOT, int all_subtypes, void *env); + +struct _typename_add_backedge { + jl_value_t *typ; + jl_value_t *caller; +}; + +static void _typename_add_backedge(jl_typename_t *tn, int explct, void *env0) +{ + struct _typename_add_backedge *env = (struct _typename_add_backedge*)env0; + JL_GC_PROMISE_ROOTED(env->typ); + JL_GC_PROMISE_ROOTED(env->caller); + if (!explct) + return; + jl_genericmemory_t *allbackedges = jl_method_table->backedges; + jl_array_t *backedges = (jl_array_t*)jl_eqtable_get(allbackedges, (jl_value_t*)tn, NULL); + if (backedges == NULL) { + backedges = jl_alloc_vec_any(2); + JL_GC_PUSH1(&backedges); + jl_array_del_end(backedges, 2); + jl_genericmemory_t *newtable = jl_eqtable_put(allbackedges, (jl_value_t*)tn, (jl_value_t*)backedges, NULL); + JL_GC_POP(); + if (newtable != allbackedges) { + jl_method_table->backedges = newtable; + jl_gc_wb(jl_method_table, newtable); + } + } + // check if the edge is already present and avoid adding a duplicate + size_t i, l = jl_array_nrows(backedges); + // reuse an already cached instance of this type, if possible + // TODO: use jl_cache_type_(tt) like cache_method does, instead of this linear scan? + // TODO: use as_global_root and de-dup edges array too + for (i = 1; i < l; i += 2) { + if (jl_array_ptr_ref(backedges, i) == env->caller) { + if (jl_types_equal(jl_array_ptr_ref(backedges, i - 1), env->typ)) { + env->typ = jl_array_ptr_ref(backedges, i - 1); + return; // this edge already recorded } } - if (!found) - push_edge(callee->backedges, invokesig, caller); } - JL_UNLOCK(&callee->def.method->writelock); + for (i = 1; i < l; i += 2) { + if (jl_array_ptr_ref(backedges, i) != env->caller) { + if (jl_types_equal(jl_array_ptr_ref(backedges, i - 1), env->typ)) { + env->typ = jl_array_ptr_ref(backedges, i - 1); + break; + } + } + } + jl_array_ptr_1d_push(backedges, env->typ); + jl_array_ptr_1d_push(backedges, env->caller); } // add a backedge from a non-existent signature to caller -JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_code_instance_t *caller) +JL_DLLEXPORT void jl_method_table_add_backedge(jl_value_t *typ, jl_code_instance_t *caller) { assert(jl_is_code_instance(caller)); if (!jl_atomic_load_relaxed(&allow_new_worlds)) return; - JL_LOCK(&mt->writelock); + // try to pick the best cache(s) for this typ edge + jl_methtable_t *mt = jl_method_table; + jl_methcache_t *mc = mt->cache; + JL_LOCK(&mc->writelock); if (jl_atomic_load_relaxed(&allow_new_worlds)) { - if (!mt->backedges) { - // lazy-init the backedges array - mt->backedges = jl_alloc_vec_any(2); - jl_gc_wb(mt, mt->backedges); - jl_array_ptr_set(mt->backedges, 0, typ); - jl_array_ptr_set(mt->backedges, 1, caller); - } - else { - // check if the edge is already present and avoid adding a duplicate - size_t i, l = jl_array_nrows(mt->backedges); - for (i = 1; i < l; i += 2) { - if (jl_array_ptr_ref(mt->backedges, i) == (jl_value_t*)caller) { - if (jl_types_equal(jl_array_ptr_ref(mt->backedges, i - 1), typ)) { - JL_UNLOCK(&mt->writelock); - return; - } - } - } - // reuse an already cached instance of this type, if possible - // TODO: use jl_cache_type_(tt) like cache_method does, instead of this linear scan? - for (i = 1; i < l; i += 2) { - if (jl_array_ptr_ref(mt->backedges, i) != (jl_value_t*)caller) { - if (jl_types_equal(jl_array_ptr_ref(mt->backedges, i - 1), typ)) { - typ = jl_array_ptr_ref(mt->backedges, i - 1); - break; - } - } - } - jl_array_ptr_1d_push(mt->backedges, typ); - jl_array_ptr_1d_push(mt->backedges, (jl_value_t*)caller); - } + struct _typename_add_backedge env = {typ, (jl_value_t*)caller}; + jl_foreach_top_typename_for(_typename_add_backedge, typ, 0, &env); } - JL_UNLOCK(&mt->writelock); + JL_UNLOCK(&mc->writelock); } -struct invalidate_mt_env { - jl_typemap_entry_t *newentry; - jl_array_t *shadowed; +struct _typename_invalidate_backedge { + jl_value_t *type; + jl_value_t **isect; + jl_value_t **isect2; + jl_method_t *const *d; + size_t n; size_t max_world; int invalidated; }; + +static void _typename_invalidate_backedges(jl_typename_t *tn, int explct, void *env0) +{ + struct _typename_invalidate_backedge *env = (struct _typename_invalidate_backedge*)env0; + JL_GC_PROMISE_ROOTED(env->type); + JL_GC_PROMISE_ROOTED(env->isect); // isJuliaType considers jl_value_t** to be a julia object too + JL_GC_PROMISE_ROOTED(env->isect2); // isJuliaType considers jl_value_t** to be a julia object too + jl_array_t *backedges = (jl_array_t*)jl_eqtable_get(jl_method_table->backedges, (jl_value_t*)tn, NULL); + if (backedges == NULL) + return; + jl_value_t **d = jl_array_ptr_data(backedges); + size_t i, na = jl_array_nrows(backedges); + size_t ins = 0; + for (i = 1; i < na; i += 2) { + jl_value_t *backedgetyp = d[i - 1]; + JL_GC_PROMISE_ROOTED(backedgetyp); + int missing = 0; + if (jl_type_intersection2(backedgetyp, (jl_value_t*)env->type, env->isect, env->isect2)) { + // See if the intersection was actually already fully + // covered, but that the new method is ambiguous. + // -> no previous method: now there is one, need to update the missing edge + // -> one+ previously matching method(s): + // -> more specific then all of them: need to update the missing edge + // -> some may have been ambiguous: now there is a replacement + // -> some may have been called: now there is a replacement (also will be detected in the loop later) + // -> less specific or ambiguous with any one of them: can ignore the missing edge (not missing) + // -> some may have been ambiguous: still are + // -> some may have been called: they may be partly replaced (will be detected in the loop later) + // c.f. `is_replacing`, which is a similar query, but with an existing method match to compare against + missing = 1; + for (size_t j = 0; j < env->n; j++) { + jl_method_t *m = env->d[j]; + JL_GC_PROMISE_ROOTED(m); + if (jl_subtype(*env->isect, m->sig) || (*env->isect2 && jl_subtype(*env->isect2, m->sig))) { + // We now know that there actually was a previous + // method for this part of the type intersection. + if (!jl_type_morespecific(env->type, m->sig)) { + missing = 0; + break; + } + } + } + } + *env->isect = *env->isect2 = NULL; + if (missing) { + jl_code_instance_t *backedge = (jl_code_instance_t*)d[i]; + JL_GC_PROMISE_ROOTED(backedge); + invalidate_code_instance(backedge, env->max_world, 0); + env->invalidated = 1; + if (_jl_debug_method_invalidation) + jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)backedgetyp); + } + else { + d[ins++] = d[i - 1]; + d[ins++] = d[i - 0]; + } + } + if (ins == 0) + jl_eqtable_pop(jl_method_table->backedges, (jl_value_t*)tn, NULL, NULL); + else if (na != ins) + jl_array_del_end(backedges, na - ins); +} + +struct invalidate_mt_env { + jl_typemap_entry_t *newentry; + jl_array_t *shadowed; + size_t max_world; +}; static int invalidate_mt_cache(jl_typemap_entry_t *oldentry, void *closure0) { struct invalidate_mt_env *env = (struct invalidate_mt_env*)closure0; @@ -2016,7 +2442,6 @@ static int invalidate_mt_cache(jl_typemap_entry_t *oldentry, void *closure0) JL_GC_POP(); } jl_atomic_store_relaxed(&oldentry->max_world, env->max_world); - env->invalidated = 1; } } return 1; @@ -2053,35 +2478,36 @@ static jl_typemap_entry_t *do_typemap_search(jl_methtable_t *mt JL_PROPAGATES_RO return (jl_typemap_entry_t *)closure; } -static void jl_method_table_invalidate(jl_methtable_t *mt, jl_method_t *replaced, size_t max_world) +static void _method_table_invalidate(jl_methcache_t *mc, void *env0) { - if (jl_options.incremental && jl_generating_output()) - jl_error("Method deletion is not possible during Module precompile."); - assert(!replaced->is_for_opaque_closure); - assert(jl_atomic_load_relaxed(&jl_world_counter) == max_world); - // drop this method from mt->cache - struct disable_mt_env mt_cache_env; - mt_cache_env.max_world = max_world; - mt_cache_env.replaced = replaced; - jl_typemap_visitor(jl_atomic_load_relaxed(&mt->cache), disable_mt_cache, (void*)&mt_cache_env); - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + // drop this method from mc->cache + jl_typemap_visitor(jl_atomic_load_relaxed(&mc->cache), disable_mt_cache, env0); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); size_t i, l = leafcache->length; for (i = 1; i < l; i += 2) { jl_typemap_entry_t *oldentry = (jl_typemap_entry_t*)jl_genericmemory_ptr_ref(leafcache, i); if (oldentry) { while ((jl_value_t*)oldentry != jl_nothing) { - disable_mt_cache(oldentry, (void*)&mt_cache_env); + disable_mt_cache(oldentry, env0); oldentry = jl_atomic_load_relaxed(&oldentry->next); } } } +} + +static void jl_method_table_invalidate(jl_method_t *replaced, size_t max_world) +{ + if (jl_options.incremental && jl_generating_output()) + jl_error("Method deletion is not possible during Module precompile."); + assert(!replaced->is_for_opaque_closure); + assert(jl_atomic_load_relaxed(&jl_world_counter) == max_world); // Invalidate the backedges int invalidated = 0; jl_value_t *specializations = jl_atomic_load_relaxed(&replaced->specializations); JL_GC_PUSH1(&specializations); if (!jl_is_svec(specializations)) specializations = (jl_value_t*)jl_svec1(specializations); - l = jl_svec_len(specializations); + size_t i, l = jl_svec_len(specializations); for (i = 0; i < l; i++) { jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i); if ((jl_value_t*)mi != jl_nothing) { @@ -2089,6 +2515,12 @@ static void jl_method_table_invalidate(jl_methtable_t *mt, jl_method_t *replaced invalidate_backedges(mi, max_world, "jl_method_table_disable"); } } + + jl_methtable_t *mt = jl_method_get_table(replaced); + struct disable_mt_env mt_cache_env; + mt_cache_env.max_world = max_world; + mt_cache_env.replaced = replaced; + _method_table_invalidate(mt->cache, &mt_cache_env); JL_GC_POP(); // XXX: this might have resolved an ambiguity, for which we have not tracked the edge here, // and thus now introduce a mistake into inference @@ -2111,13 +2543,13 @@ static int erase_method_backedges(jl_typemap_entry_t *def, void *closure) for (i = 0; i < l; i++) { jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i); if ((jl_value_t*)mi != jl_nothing) { - mi->backedges = NULL; + mi->backedges = 0; } } } else { jl_method_instance_t *mi = (jl_method_instance_t*)specializations; - mi->backedges = NULL; + mi->backedges = 0; } JL_UNLOCK(&method->writelock); return 1; @@ -2125,13 +2557,7 @@ static int erase_method_backedges(jl_typemap_entry_t *def, void *closure) static int erase_all_backedges(jl_methtable_t *mt, void *env) { - // removes all method caches - // this might not be entirely safe (GC or MT), thus we only do it very early in bootstrapping - JL_LOCK(&mt->writelock); - mt->backedges = NULL; - JL_UNLOCK(&mt->writelock); - jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), erase_method_backedges, env); - return 1; + return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), erase_method_backedges, env); } JL_DLLEXPORT void jl_disable_new_worlds(void) @@ -2141,122 +2567,203 @@ JL_DLLEXPORT void jl_disable_new_worlds(void) JL_LOCK(&world_counter_lock); jl_atomic_store_relaxed(&allow_new_worlds, 0); JL_UNLOCK(&world_counter_lock); - jl_foreach_reachable_mtable(erase_all_backedges, (void*)NULL); + jl_array_t *mod_array = jl_get_loaded_modules(); + JL_GC_PUSH1(&mod_array); + jl_foreach_reachable_mtable(erase_all_backedges, mod_array, (void*)NULL); + + JL_LOCK(&jl_method_table->cache->writelock); + jl_method_table->backedges = (jl_genericmemory_t*)jl_an_empty_memory_any; + JL_UNLOCK(&jl_method_table->cache->writelock); + JL_GC_POP(); } -JL_DLLEXPORT void jl_method_table_disable(jl_methtable_t *mt, jl_method_t *method) +JL_DLLEXPORT void jl_method_table_disable(jl_method_t *method) { + jl_methtable_t *mt = jl_method_get_table(method); jl_typemap_entry_t *methodentry = do_typemap_search(mt, method); JL_LOCK(&world_counter_lock); if (!jl_atomic_load_relaxed(&allow_new_worlds)) jl_error("Method changes have been disabled via a call to disable_new_worlds."); - JL_LOCK(&mt->writelock); - // Narrow the world age on the method to make it uncallable - size_t world = jl_atomic_load_relaxed(&jl_world_counter); - assert(method == methodentry->func.method); - assert(jl_atomic_load_relaxed(&method->deleted_world) == ~(size_t)0); - jl_atomic_store_relaxed(&method->deleted_world, world); - jl_atomic_store_relaxed(&methodentry->max_world, world); - jl_method_table_invalidate(mt, method, world); - jl_atomic_store_release(&jl_world_counter, world + 1); - JL_UNLOCK(&mt->writelock); + int enabled = jl_atomic_load_relaxed(&methodentry->max_world) == ~(size_t)0; + if (enabled) { + // Narrow the world age on the method to make it uncallable + size_t world = jl_atomic_load_relaxed(&jl_world_counter); + assert(method == methodentry->func.method); + jl_atomic_store_relaxed(&method->dispatch_status, 0); + assert(jl_atomic_load_relaxed(&methodentry->max_world) == ~(size_t)0); + jl_atomic_store_relaxed(&methodentry->max_world, world); + jl_method_table_invalidate(method, world); + jl_atomic_store_release(&jl_world_counter, world + 1); + } JL_UNLOCK(&world_counter_lock); + if (!enabled) + jl_errorf("Method of %s already disabled", jl_symbol_name(method->name)); } -static int jl_type_intersection2(jl_value_t *t1, jl_value_t *t2, jl_value_t **isect JL_REQUIRE_ROOTED_SLOT, jl_value_t **isect2 JL_REQUIRE_ROOTED_SLOT) +jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype) { - *isect2 = NULL; - int is_subty = 0; - *isect = jl_type_intersection_env_s(t1, t2, NULL, &is_subty); - if (*isect == jl_bottom_type) - return 0; - if (is_subty) - return 1; - // TODO: sometimes type intersection returns types with free variables - if (jl_has_free_typevars(t1) || jl_has_free_typevars(t2)) - return 1; - // determine if type-intersection can be convinced to give a better, non-bad answer - // if the intersection was imprecise, see if we can do better by switching the types - *isect2 = jl_type_intersection(t2, t1); - if (*isect2 == jl_bottom_type) { - *isect = jl_bottom_type; - *isect2 = NULL; - return 0; + JL_TIMING(ADD_METHOD, ADD_METHOD); + assert(jl_is_method(method)); + assert(jl_is_mtable(mt)); + jl_timing_show_method(method, JL_TIMING_DEFAULT_BLOCK); + jl_typemap_entry_t *newentry = NULL; + JL_GC_PUSH1(&newentry); + // add our new entry + assert(jl_atomic_load_relaxed(&method->primary_world) == ~(size_t)0); // min-world + assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_WHICH) == 0); + assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_ONLY) == 0); + JL_LOCK(&mt->cache->writelock); + newentry = jl_typemap_alloc((jl_tupletype_t*)method->sig, simpletype, jl_emptysvec, (jl_value_t*)method, ~(size_t)0, 1); + jl_typemap_insert(&mt->defs, (jl_value_t*)mt, newentry, 0); + + if (mt == jl_method_table) + update_max_args(method->sig); + JL_UNLOCK(&mt->cache->writelock); + JL_GC_POP(); + return newentry; +} + +static int has_key(jl_genericmemory_t *keys, jl_value_t *key) +{ + for (size_t l = keys->length, i = 0; i < l; i++) { + jl_value_t *k = jl_genericmemory_ptr_ref(keys, i); + if (k == NULL) + return 0; + if (jl_genericmemory_ptr_ref(keys, i) == key) + return 1; } - if (jl_types_egal(*isect2, *isect)) { - *isect2 = NULL; + return 0; +} + +// Check if m2 is in m1's interferences set, which means !morespecific(m1, m2) +static int method_in_interferences(jl_method_t *m2, jl_method_t *m1) +{ + return has_key(jl_atomic_load_relaxed(&m1->interferences), (jl_value_t*)m2); +} + +// Find the index of a method in the method match array +static int find_method_in_matches(jl_array_t *t, jl_method_t *method) +{ + size_t len = jl_array_nrows(t); + for (size_t i = 0; i < len; i++) { + jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, i); + if (matc->method == method) + return i; } - return 1; + return -1; } -enum morespec_options { - morespec_unknown, - morespec_isnot, - morespec_is -}; +// Recursively check if any method in interferences covers the given type signature +static int check_interferences_covers(jl_method_t *m, jl_value_t *ti, jl_array_t *t, arraylist_t *visited, arraylist_t *recursion_stack) +{ + // Check if we're already visiting this method (cycle detection and memoization) + for (size_t i = 0; i < recursion_stack->len; i++) + if (recursion_stack->items[i] == (void*)m) + return 0; -// check if `type` is replacing `m` with an ambiguity here, given other methods in `d` that already match it -static int is_replacing(char ambig, jl_value_t *type, jl_method_t *m, jl_method_t *const *d, size_t n, jl_value_t *isect, jl_value_t *isect2, char *morespec) + // Add this method to the recursion stack + arraylist_push(recursion_stack, (void*)m); + + jl_genericmemory_t *interferences = jl_atomic_load_relaxed(&m->interferences); + for (size_t i = 0; i < interferences->length; i++) { + jl_method_t *m2 = (jl_method_t*)jl_genericmemory_ptr_ref(interferences, i); + if (m2 == NULL) + continue; + int idx = find_method_in_matches(t, m2); + if (idx < 0) + continue; + if (method_in_interferences(m, m2)) + continue; // ambiguous + assert(visited->items[idx] != (void*)0); + if (visited->items[idx] != (void*)1) + continue; // part of the same SCC cycle (handled by ambiguity later) + if (jl_subtype(ti, m2->sig)) + return 1; + // Recursively check m2's interferences since m2 is more specific + if (check_interferences_covers(m2, ti, t, visited, recursion_stack)) + return 1; + } + return 0; +} + +static int check_fully_ambiguous(jl_method_t *m, jl_value_t *ti, jl_array_t *t, int include_ambiguous, int *has_ambiguity) { - size_t k; - for (k = 0; k < n; k++) { - jl_method_t *m2 = d[k]; - // see if m2 also fully covered this intersection - if (m == m2 || !(jl_subtype(isect, m2->sig) || (isect2 && jl_subtype(isect2, m2->sig)))) + jl_genericmemory_t *interferences = jl_atomic_load_relaxed(&m->interferences); + for (size_t i = 0; i < interferences->length; i++) { + jl_method_t *m2 = (jl_method_t*)jl_genericmemory_ptr_ref(interferences, i); + if (m2 == NULL) continue; - if (morespec[k] == (char)morespec_unknown) - morespec[k] = (char)(jl_type_morespecific(m2->sig, type) ? morespec_is : morespec_isnot); - if (morespec[k] == (char)morespec_is) - // not actually shadowing this--m2 will still be better - return 0; - // if type is not more specific than m (thus now dominating it) - // then there is a new ambiguity here, - // since m2 was also a previous match over isect, - // see if m was previously dominant over all m2 - // or if this was already ambiguous before - if (ambig != morespec_is && !jl_type_morespecific(m->sig, m2->sig)) { - // m and m2 were previously ambiguous over the full intersection of mi with type, and will still be ambiguous with addition of type + int idx = find_method_in_matches(t, m2); + if (idx < 0) + continue; + if (!method_in_interferences(m, m2)) + continue; + *has_ambiguity = 1; + if (!include_ambiguous && jl_subtype(ti, m2->sig)) + return 1; + } + return 0; +} + +// Recursively check if target_method is in the interferences of (morespecific than) start_method, but not the reverse +static int method_in_interferences_recursive(jl_method_t *target_method, jl_method_t *start_method, arraylist_t *seen) +{ + // Check direct interferences first + if (method_in_interferences(start_method, target_method)) + return 0; + if (method_in_interferences(target_method, start_method)) + return 1; + + // Check if we're already visiting this method (cycle prevention and memoization) + for (size_t i = 0; i < seen->len; i++) { + if (seen->items[i] == (void*)start_method) return 0; - } } - return 1; + arraylist_push(seen, (void*)start_method); + + // Recursively check interferences + jl_genericmemory_t *interferences = jl_atomic_load_relaxed(&start_method->interferences); + for (size_t i = 0; i < interferences->length; i++) { + jl_method_t *interference_method = (jl_method_t*)jl_genericmemory_ptr_ref(interferences, i); + if (interference_method == NULL) + continue; + if (method_in_interferences(start_method, interference_method)) + continue; // only follow edges to morespecific methods in search of morespecific target (skip ambiguities) + if (method_in_interferences_recursive(target_method, interference_method, seen)) + return 1; + } + + return 0; } -jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype) +static int method_morespecific_via_interferences(jl_method_t *target_method, jl_method_t *start_method) { - JL_TIMING(ADD_METHOD, ADD_METHOD); - assert(jl_is_method(method)); - assert(jl_is_mtable(mt)); - jl_timing_show_method(method, JL_TIMING_DEFAULT_BLOCK); - jl_typemap_entry_t *newentry = NULL; - JL_GC_PUSH1(&newentry); - JL_LOCK(&mt->writelock); - // add our new entry - assert(jl_atomic_load_relaxed(&method->primary_world) == ~(size_t)0); // min-world - assert(jl_atomic_load_relaxed(&method->deleted_world) == 1); // max-world - newentry = jl_typemap_alloc((jl_tupletype_t*)method->sig, simpletype, jl_emptysvec, (jl_value_t*)method, - jl_atomic_load_relaxed(&method->primary_world), jl_atomic_load_relaxed(&method->deleted_world)); - jl_typemap_insert(&mt->defs, (jl_value_t*)mt, newentry, jl_cachearg_offset(mt)); - update_max_args(mt, method->sig); - JL_UNLOCK(&mt->writelock); - JL_GC_POP(); - return newentry; + if (target_method == start_method) + return 0; + arraylist_t seen; + arraylist_new(&seen, 0); + int result = method_in_interferences_recursive(target_method, start_method, &seen); + arraylist_free(&seen); + //assert(result == jl_method_morespecific(target_method, start_method) || jl_has_empty_intersection(target_method->sig, start_method->sig) || jl_has_empty_intersection(start_method->sig, target_method->sig)); + return result; } -void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) + +void jl_method_table_activate(jl_typemap_entry_t *newentry) { JL_TIMING(ADD_METHOD, ADD_METHOD); jl_method_t *method = newentry->func.method; + jl_methtable_t *mt = jl_method_get_table(method); assert(jl_is_mtable(mt)); assert(jl_is_method(method)); jl_timing_show_method(method, JL_TIMING_DEFAULT_BLOCK); jl_value_t *type = (jl_value_t*)newentry->sig; jl_value_t *oldvalue = NULL; jl_array_t *oldmi = NULL; - JL_LOCK(&mt->writelock); size_t world = jl_atomic_load_relaxed(&method->primary_world); assert(world == jl_atomic_load_relaxed(&jl_world_counter) + 1); // min-world - assert(jl_atomic_load_relaxed(&method->deleted_world) == ~(size_t)0); // max-world + assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_WHICH) == 0); + assert((jl_atomic_load_relaxed(&method->dispatch_status) & METHOD_SIG_LATEST_ONLY) == 0); assert(jl_atomic_load_relaxed(&newentry->min_world) == ~(size_t)0); assert(jl_atomic_load_relaxed(&newentry->max_world) == 1); jl_atomic_store_relaxed(&newentry->min_world, world); @@ -2265,91 +2772,134 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) jl_value_t *loctag = NULL; // debug info for invalidation jl_value_t *isect = NULL; jl_value_t *isect2 = NULL; - jl_value_t *isect3 = NULL; - JL_GC_PUSH6(&oldvalue, &oldmi, &loctag, &isect, &isect2, &isect3); + jl_genericmemory_t *interferences = NULL; + JL_GC_PUSH6(&oldvalue, &oldmi, &loctag, &isect, &isect2, &interferences); jl_typemap_entry_t *replaced = NULL; - // then check what entries we replaced - oldvalue = get_intersect_matches(jl_atomic_load_relaxed(&mt->defs), newentry, &replaced, jl_cachearg_offset(mt), max_world); - int invalidated = 0; - if (replaced) { - oldvalue = (jl_value_t*)replaced; - invalidated = 1; - method_overwrite(newentry, replaced->func.method); - // this is an optimized version of below, given we know the type-intersection is exact - jl_method_table_invalidate(mt, replaced->func.method, max_world); + // Check what entries this intersects with in the prior world. + oldvalue = get_intersect_matches(jl_atomic_load_relaxed(&mt->defs), newentry, &replaced, max_world); + jl_method_t *const *d; + size_t j, n; + if (oldvalue == NULL) { + d = NULL; + n = 0; } else { - jl_method_t *const *d; - size_t j, n; - if (oldvalue == NULL) { - d = NULL; - n = 0; - } - else { - assert(jl_is_array(oldvalue)); - d = (jl_method_t**)jl_array_ptr_data(oldvalue); - n = jl_array_nrows(oldvalue); - } - if (mt->backedges) { - jl_value_t **backedges = jl_array_ptr_data(mt->backedges); - size_t i, na = jl_array_nrows(mt->backedges); - size_t ins = 0; - for (i = 1; i < na; i += 2) { - jl_value_t *backedgetyp = backedges[i - 1]; - JL_GC_PROMISE_ROOTED(backedgetyp); - int missing = 0; - if (jl_type_intersection2(backedgetyp, (jl_value_t*)type, &isect, &isect2)) { - // See if the intersection was actually already fully - // covered, but that the new method is ambiguous. - // -> no previous method: now there is one, need to update the missing edge - // -> one+ previously matching method(s): - // -> more specific then all of them: need to update the missing edge - // -> some may have been ambiguous: now there is a replacement - // -> some may have been called: now there is a replacement (also will be detected in the loop later) - // -> less specific or ambiguous with any one of them: can ignore the missing edge (not missing) - // -> some may have been ambiguous: still are - // -> some may have been called: they may be partly replaced (will be detected in the loop later) - // c.f. `is_replacing`, which is a similar query, but with an existing method match to compare against - missing = 1; - size_t j; - for (j = 0; j < n; j++) { - jl_method_t *m = d[j]; - if (jl_subtype(isect, m->sig) || (isect2 && jl_subtype(isect2, m->sig))) { - // We now know that there actually was a previous - // method for this part of the type intersection. - if (!jl_type_morespecific(type, m->sig)) { - missing = 0; - break; - } - } + assert(jl_is_array(oldvalue)); + d = (jl_method_t**)jl_array_ptr_data(oldvalue); + n = jl_array_nrows(oldvalue); + oldmi = jl_alloc_vec_any(0); + } + + // These get updated from their state stored in the caches files, since content in cache files gets added "all at once". + int invalidated = 0; + int dispatch_bits = METHOD_SIG_LATEST_WHICH; // Always set LATEST_WHICH + // Check precompiled dispatch status bits + int precompiled_status = jl_atomic_load_relaxed(&method->dispatch_status); + if (!(precompiled_status & METHOD_SIG_PRECOMPILE_MANY)) + // This will store if this method will be currently the only result that would returned from `ml_matches` given `sig`. + dispatch_bits |= METHOD_SIG_LATEST_ONLY; // Tentatively set, will be cleared if not applicable + // Holds the set of all intersecting methods not more specific than this one. + // Note: this set may be incomplete (may exclude methods whose intersection + // is covered by another method that is morespecific than both, causing them + // to have no relevant type intersection for sorting). + interferences = (jl_genericmemory_t*)jl_atomic_load_relaxed(&method->interferences); + if (oldvalue) { + assert(n > 0); + if (replaced) { + oldvalue = (jl_value_t*)replaced; + jl_method_t *m = replaced->func.method; + invalidated = 1; + method_overwrite(newentry, m); + // This is an optimized version of below, given we know the type-intersection is exact + jl_method_table_invalidate(m, max_world); + int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); + // Clear METHOD_SIG_LATEST_ONLY and METHOD_SIG_LATEST_WHICH bits + jl_atomic_store_relaxed(&m->dispatch_status, 0); + if (!(m_dispatch & METHOD_SIG_LATEST_ONLY)) + dispatch_bits &= ~METHOD_SIG_LATEST_ONLY; + // Take over the interference list from the replaced method + jl_genericmemory_t *m_interferences = jl_atomic_load_relaxed(&m->interferences); + if (interferences->length == 0) { + interferences = jl_genericmemory_copy(m_interferences); + } + else { + for (size_t i = 0; i < m_interferences->length; i++) { + jl_value_t *k = jl_genericmemory_ptr_ref(m_interferences, i); + if (k && !has_key(interferences, (jl_value_t*)k)) { + ssize_t idx; + interferences = jl_idset_put_key(interferences, (jl_value_t*)k, &idx); } } - if (missing) { - jl_code_instance_t *backedge = (jl_code_instance_t*)backedges[i]; - JL_GC_PROMISE_ROOTED(backedge); - invalidate_code_instance(backedge, max_world, 0); - invalidated = 1; - if (_jl_debug_method_invalidation) - jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)backedgetyp); - } - else { - backedges[ins++] = backedges[i - 1]; - backedges[ins++] = backedges[i - 0]; + } + ssize_t idx; + m_interferences = jl_idset_put_key(m_interferences, (jl_value_t*)method, &idx); + jl_atomic_store_release(&m->interferences, m_interferences); + jl_gc_wb(m, m_interferences); + for (j = 0; j < n; j++) { + jl_method_t *m2 = d[j]; + if (m2 && method_in_interferences(m, m2)) { + jl_genericmemory_t *m2_interferences = jl_atomic_load_relaxed(&m2->interferences); + ssize_t idx; + m2_interferences = jl_idset_put_key(m2_interferences, (jl_value_t*)method, &idx); + jl_atomic_store_release(&m2->interferences, m2_interferences); + jl_gc_wb(m2, m2_interferences); } } - if (ins == 0) - mt->backedges = NULL; - else - jl_array_del_end(mt->backedges, na - ins); + loctag = jl_atomic_load_relaxed(&m->specializations); // use loctag for a gcroot + _Atomic(jl_method_instance_t*) *data; + size_t l; + if (jl_is_svec(loctag)) { + data = (_Atomic(jl_method_instance_t*)*)jl_svec_data(loctag); + l = jl_svec_len(loctag); + } + else { + data = (_Atomic(jl_method_instance_t*)*) &loctag; + l = 1; + } + for (size_t i = 0; i < l; i++) { + jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]); + if ((jl_value_t*)mi == jl_nothing) + continue; + jl_array_ptr_1d_push(oldmi, (jl_value_t*)mi); + } + d = NULL; + n = 0; } - if (oldvalue) { - oldmi = jl_alloc_vec_any(0); + else { char *morespec = (char*)alloca(n); - memset(morespec, morespec_unknown, n); + // Compute all morespec values upfront + for (j = 0; j < n; j++) + morespec[j] = (char)jl_type_morespecific(d[j]->sig, type); for (j = 0; j < n; j++) { jl_method_t *m = d[j]; - if (morespec[j] == (char)morespec_is) + // Compute ambig state: is there an ambiguity between new method and old m? + char ambig = !morespec[j] && !jl_type_morespecific(type, m->sig); + // Compute updates to the dispatch state bits + int m_dispatch = jl_atomic_load_relaxed(&m->dispatch_status); + if (morespec[j] || ambig) { + // !morespecific(new, old) + dispatch_bits &= ~METHOD_SIG_LATEST_ONLY; + // Add the old method to this interference set + ssize_t idx; + if (!has_key(interferences, (jl_value_t*)m)) + interferences = jl_idset_put_key(interferences, (jl_value_t*)m, &idx); + } + if (!morespec[j]) { + // !morespecific(old, new) + m_dispatch &= ~METHOD_SIG_LATEST_ONLY; + // Add the new method to its interference set + jl_genericmemory_t *m_interferences = jl_atomic_load_relaxed(&m->interferences); + ssize_t idx; + m_interferences = jl_idset_put_key(m_interferences, (jl_value_t*)method, &idx); + jl_atomic_store_release(&m->interferences, m_interferences); + jl_gc_wb(m, m_interferences); + } + // Add methods that intersect but are not more specific to interference list + jl_atomic_store_relaxed(&m->dispatch_status, m_dispatch); + if (morespec[j]) continue; + + // Now examine if this caused any invalidations. loctag = jl_atomic_load_relaxed(&m->specializations); // use loctag for a gcroot _Atomic(jl_method_instance_t*) *data; size_t l; @@ -2361,100 +2911,101 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) data = (_Atomic(jl_method_instance_t*)*) &loctag; l = 1; } - enum morespec_options ambig = morespec_unknown; for (size_t i = 0; i < l; i++) { jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]); if ((jl_value_t*)mi == jl_nothing) continue; - isect3 = jl_type_intersection(m->sig, (jl_value_t*)mi->specTypes); - if (jl_type_intersection2(type, isect3, &isect, &isect2)) { + if (jl_type_intersection2(type, mi->specTypes, &isect, &isect2)) { + // Replacing a method--see if this really was the selected method previously + // over the intersection (not ambiguous) and the new method will be selected now (morespec). // TODO: this only checks pair-wise for ambiguities, but the ambiguities could arise from the interaction of multiple methods - // and thus might miss a case where we introduce an ambiguity between two existing methods + // and thus might miss a case where we introduce an ambiguity between`.u two existing methods // We could instead work to sort this into 3 groups `morespecific .. ambiguous .. lesspecific`, with `type` in ambiguous, // such that everything in `morespecific` dominates everything in `ambiguous`, and everything in `ambiguous` dominates everything in `lessspecific` // And then compute where each isect falls, and whether it changed group--necessitating invalidation--or not. - if (morespec[j] == (char)morespec_unknown) - morespec[j] = (char)(jl_type_morespecific(m->sig, type) ? morespec_is : morespec_isnot); - if (morespec[j] == (char)morespec_is) - // not actually shadowing--the existing method is still better - break; - if (ambig == morespec_unknown) - ambig = jl_type_morespecific(type, m->sig) ? morespec_is : morespec_isnot; - // replacing a method--see if this really was the selected method previously - // over the intersection (not ambiguous) and the new method will be selected now (morespec_is) int replaced_dispatch = is_replacing(ambig, type, m, d, n, isect, isect2, morespec); // found that this specialization dispatch got replaced by m // call invalidate_backedges(mi, max_world, "jl_method_table_insert"); // but ignore invoke-type edges - jl_array_t *backedges = mi->backedges; - if (backedges) { - size_t ib = 0, insb = 0, nb = jl_array_nrows(backedges); - jl_value_t *invokeTypes; - jl_code_instance_t *caller; - while (ib < nb) { - ib = get_next_edge(backedges, ib, &invokeTypes, &caller); - JL_GC_PROMISE_ROOTED(caller); // propagated by get_next_edge from backedges - int replaced_edge; - if (invokeTypes) { - // n.b. normally we must have mi.specTypes <: invokeTypes <: m.sig (though it might not strictly hold), so we only need to check the other subtypes - if (jl_egal(invokeTypes, jl_get_ci_mi(caller)->def.method->sig)) - replaced_edge = 0; // if invokeTypes == m.sig, then the only way to change this invoke is to replace the method itself - else - replaced_edge = jl_subtype(invokeTypes, type) && is_replacing(ambig, type, m, d, n, invokeTypes, NULL, morespec); - } - else { - replaced_edge = replaced_dispatch; - } - if (replaced_edge) { - invalidate_code_instance(caller, max_world, 1); - invalidated = 1; - } - else { - insb = set_next_edge(backedges, insb, invokeTypes, caller); - } - } - jl_array_del_end(backedges, nb - insb); + int invalidatedmi = _invalidate_dispatch_backedges(mi, type, m, d, n, replaced_dispatch, ambig, max_world, morespec); + if (replaced_dispatch) { + jl_atomic_store_relaxed(&mi->dispatch_status, 0); + jl_array_ptr_1d_push(oldmi, (jl_value_t*)mi); } - jl_array_ptr_1d_push(oldmi, (jl_value_t*)mi); - if (_jl_debug_method_invalidation && invalidated) { + if (_jl_debug_method_invalidation && invalidatedmi) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)mi); loctag = jl_cstr_to_string("jl_method_table_insert"); jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); } + invalidated |= invalidatedmi; } + // TODO: do we have any interesting cases left where isect3 is useful + //jl_value_t *isect3 = NULL; + //jl_value_t *isect4 = NULL; + //jl_value_t *isect5 = NULL; + //JL_GC_PUSH3(&isec3, &isect4, &isect5); + //isect3 = jl_type_intersection(m->sig, (jl_value_t*)mi->specTypes); + //jl_type_intersection2(type, isect3, &isect4, &isect5); + //if (!jl_types_equal(isect, isect4) && (!isect2 || !jl_types_equal(isect2, isect4)) && + // (!isect5 || (!jl_types_equal(isect, isect5) && (!isect2 || !jl_types_equal(isect2, isect5))))) { + // jl_(type); + // jl_(mi->specTypes); + // jl_(m->sig); + //} + //JL_GC_POP(); + isect = NULL; + isect2 = NULL; } } - if (jl_array_nrows(oldmi)) { - // search mt->cache and leafcache and drop anything that might overlap with the new method - // this is very cheap, so we don't mind being fairly conservative at over-approximating this - struct invalidate_mt_env mt_cache_env; - mt_cache_env.max_world = max_world; - mt_cache_env.shadowed = oldmi; - mt_cache_env.newentry = newentry; - mt_cache_env.invalidated = 0; - - jl_typemap_visitor(jl_atomic_load_relaxed(&mt->cache), invalidate_mt_cache, (void*)&mt_cache_env); - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); - size_t i, l = leafcache->length; - for (i = 1; i < l; i += 2) { - jl_value_t *entry = jl_genericmemory_ptr_ref(leafcache, i); - if (entry) { - while (entry != jl_nothing) { - invalidate_mt_cache((jl_typemap_entry_t*)entry, (void*)&mt_cache_env); - entry = (jl_value_t*)jl_atomic_load_relaxed(&((jl_typemap_entry_t*)entry)->next); - } - } + } + } + + jl_methcache_t *mc = jl_method_table->cache; + JL_LOCK(&mc->writelock); + struct _typename_invalidate_backedge typename_env = {type, &isect, &isect2, d, n, max_world, invalidated}; + if (!jl_foreach_top_typename_for(_typename_invalidate_backedges, type, 1, &typename_env)) { + // if the new method cannot be split into exact backedges, scan the whole table for anything that might be affected + jl_genericmemory_t *allbackedges = jl_method_table->backedges; + for (size_t i = 0, n = allbackedges->length; i < n; i += 2) { + jl_value_t *tn = jl_genericmemory_ptr_ref(allbackedges, i); + jl_value_t *backedges = jl_genericmemory_ptr_ref(allbackedges, i+1); + if (tn && tn != jl_nothing && backedges) + _typename_invalidate_backedges((jl_typename_t*)tn, 0, &typename_env); + } + } + invalidated |= typename_env.invalidated; + if (oldmi && jl_array_nrows(oldmi)) { + // drop leafcache and search mc->cache and drop anything that might overlap with the new method + // this is very cheap, so we don't mind being very conservative at over-approximating this + struct invalidate_mt_env mt_cache_env; + mt_cache_env.max_world = max_world; + mt_cache_env.shadowed = oldmi; + mt_cache_env.newentry = newentry; + + jl_typemap_visitor(jl_atomic_load_relaxed(&mc->cache), invalidate_mt_cache, (void*)&mt_cache_env); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); + size_t i, l = leafcache->length; + for (i = 1; i < l; i += 2) { + jl_value_t *entry = jl_genericmemory_ptr_ref(leafcache, i); + if (entry) { + while (entry != jl_nothing) { + jl_atomic_store_relaxed(&((jl_typemap_entry_t*)entry)->max_world, max_world); + entry = (jl_value_t*)jl_atomic_load_relaxed(&((jl_typemap_entry_t*)entry)->next); } } } + jl_atomic_store_relaxed(&mc->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); } + JL_UNLOCK(&mc->writelock); if (invalidated && _jl_debug_method_invalidation) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)method); loctag = jl_cstr_to_string("jl_method_table_insert"); jl_array_ptr_1d_push(_jl_debug_method_invalidation, loctag); } - jl_atomic_store_relaxed(&newentry->max_world, jl_atomic_load_relaxed(&method->deleted_world)); - JL_UNLOCK(&mt->writelock); + jl_atomic_store_relaxed(&newentry->max_world, ~(size_t)0); + jl_atomic_store_relaxed(&method->dispatch_status, dispatch_bits); // TODO: this should be sequenced fully after the world counter store + jl_atomic_store_release(&method->interferences, interferences); + jl_gc_wb(method, interferences); JL_GC_POP(); } @@ -2467,8 +3018,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method jl_error("Method changes have been disabled via a call to disable_new_worlds."); size_t world = jl_atomic_load_relaxed(&jl_world_counter) + 1; jl_atomic_store_relaxed(&method->primary_world, world); - jl_atomic_store_relaxed(&method->deleted_world, ~(size_t)0); - jl_method_table_activate(mt, newentry); + jl_method_table_activate(newentry); jl_atomic_store_release(&jl_world_counter, world); JL_UNLOCK(&world_counter_lock); JL_GC_POP(); @@ -2520,13 +3070,15 @@ static jl_tupletype_t *lookup_arg_type_tuple(jl_value_t *arg1 JL_PROPAGATES_ROOT JL_DLLEXPORT jl_value_t *jl_method_lookup_by_tt(jl_tupletype_t *tt, size_t world, jl_value_t *_mt) { jl_methtable_t *mt = NULL; - if (_mt == jl_nothing) - mt = jl_gf_ft_mtable(jl_tparam0(tt)); + if (_mt == jl_nothing) { + mt = jl_method_table; + } else { - assert(jl_isa(_mt, (jl_value_t*)jl_methtable_type)); + assert(jl_is_mtable(_mt)); mt = (jl_methtable_t*) _mt; } - jl_method_instance_t* mi = jl_mt_assoc_by_type(mt, tt, world); + jl_methcache_t *mc = mt->cache; + jl_method_instance_t *mi = jl_mt_assoc_by_type(mc, tt, world); if (!mi) return jl_nothing; return (jl_value_t*) mi; @@ -2535,13 +3087,13 @@ JL_DLLEXPORT jl_value_t *jl_method_lookup_by_tt(jl_tupletype_t *tt, size_t world JL_DLLEXPORT jl_method_instance_t *jl_method_lookup(jl_value_t **args, size_t nargs, size_t world) { assert(nargs > 0 && "expected caller to handle this case"); - jl_methtable_t *mt = jl_gf_mtable(args[0]); - jl_typemap_t *cache = jl_atomic_load_relaxed(&mt->cache); // XXX: gc root for this? - jl_typemap_entry_t *entry = jl_typemap_assoc_exact(cache, args[0], &args[1], nargs, jl_cachearg_offset(mt), world); + jl_methcache_t *mc = jl_method_table->cache; + jl_typemap_t *cache = jl_atomic_load_relaxed(&mc->cache); // XXX: gc root for this? + jl_typemap_entry_t *entry = jl_typemap_assoc_exact(cache, args[0], &args[1], nargs, jl_cachearg_offset(), world); if (entry) return entry->func.linfo; jl_tupletype_t *tt = arg_type_tuple(args[0], &args[1], nargs); - return jl_mt_assoc_by_type(mt, tt, world); + return jl_mt_assoc_by_type(mc, tt, world); } // return a Vector{Any} of svecs, each describing a method match: @@ -2564,10 +3116,9 @@ JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, jl_value_t * if (unw == (jl_value_t*)jl_emptytuple_type || jl_tparam0(unw) == jl_bottom_type) return (jl_value_t*)jl_an_empty_vec_any; if (mt == jl_nothing) - mt = (jl_value_t*)jl_method_table_for(unw); - if (mt == jl_nothing) - mt = NULL; - return ml_matches((jl_methtable_t*)mt, types, lim, include_ambiguous, 1, world, 1, min_valid, max_valid, ambig); + mt = (jl_value_t*)jl_method_table; + jl_methcache_t *mc = ((jl_methtable_t*)mt)->cache; + return ml_matches((jl_methtable_t*)mt, mc, types, lim, include_ambiguous, 1, world, 1, min_valid, max_valid, ambig); } JL_DLLEXPORT jl_method_instance_t *jl_get_unspecialized(jl_method_t *def JL_PROPAGATES_ROOT) @@ -2620,20 +3171,30 @@ JL_DLLEXPORT jl_value_t *jl_rettype_inferred_native(jl_method_instance_t *mi, si JL_DLLEXPORT jl_value_t *(*const jl_rettype_inferred_addr)(jl_method_instance_t *mi, size_t min_world, size_t max_world) JL_NOTSAFEPOINT = jl_rettype_inferred_native; -jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi, size_t world) +STATIC_INLINE jl_callptr_t jl_method_compiled_callptr(jl_method_instance_t *mi, size_t world, jl_code_instance_t **codeinst_out) JL_NOTSAFEPOINT { jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); for (; codeinst; codeinst = jl_atomic_load_relaxed(&codeinst->next)) { if (codeinst->owner != jl_nothing) continue; if (jl_atomic_load_relaxed(&codeinst->min_world) <= world && world <= jl_atomic_load_relaxed(&codeinst->max_world)) { - if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL) - return codeinst; + jl_callptr_t invoke = jl_atomic_load_acquire(&codeinst->invoke); + if (!invoke) + continue; + *codeinst_out = codeinst; + return invoke; } } return NULL; } +jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi, size_t world) JL_NOTSAFEPOINT +{ + jl_code_instance_t *codeinst = NULL; + jl_method_compiled_callptr(mi, world, &codeinst); + return codeinst; +} + jl_mutex_t precomp_statement_out_lock; _Atomic(uint8_t) jl_force_trace_compile_timing_enabled = 0; @@ -2743,7 +3304,8 @@ static void record_dispatch_statement(jl_method_instance_t *mi) s_dispatch = (JL_STREAM*) &f_dispatch; } } - if (!jl_has_free_typevars(mi->specTypes)) { + // NOTE: For builtin functions, the specType is just `Tuple`, which is not useful to print. + if (!jl_has_free_typevars(mi->specTypes) && (jl_datatype_t*)mi->specTypes != jl_tuple_type) { jl_printf(s_dispatch, "precompile("); jl_static_show(s_dispatch, mi->specTypes); jl_printf(s_dispatch, ")\n"); @@ -2753,6 +3315,19 @@ static void record_dispatch_statement(jl_method_instance_t *mi) JL_UNLOCK(&dispatch_statement_out_lock); } +static void record_dispatch_statement_on_first_dispatch(jl_method_instance_t *mfunc) { + uint8_t force_trace_dispatch = jl_atomic_load_relaxed(&jl_force_trace_dispatch_enabled); + if (force_trace_dispatch || jl_options.trace_dispatch != NULL) { + uint8_t miflags = jl_atomic_load_relaxed(&mfunc->flags); + uint8_t was_dispatched = miflags & JL_MI_FLAGS_MASK_DISPATCHED; + if (!was_dispatched) { + miflags |= JL_MI_FLAGS_MASK_DISPATCHED; + jl_atomic_store_relaxed(&mfunc->flags, miflags); + record_dispatch_statement(mfunc); + } + } +} + // If waitcompile is 0, this will return NULL if compiling is on-going in the JIT. This is // useful for the JIT itself, since it just doesn't cause redundant work or missed updates, // but merely causes it to look into the current JIT worklist. @@ -2799,19 +3374,6 @@ JL_DLLEXPORT void jl_add_codeinst_to_jit(jl_code_instance_t *codeinst, jl_code_i { assert(jl_is_code_info(src)); jl_emit_codeinst_to_jit(codeinst, src); - jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - if (jl_generating_output() && jl_is_method(mi->def.method) && jl_atomic_load_relaxed(&codeinst->inferred) == jl_nothing) { - jl_value_t *compressed = jl_compress_ir(mi->def.method, src); - // These should already be compatible (and should be an assert), but make sure of it anyways - if (jl_is_svec(src->edges)) { - jl_atomic_store_release(&codeinst->edges, (jl_svec_t*)src->edges); - jl_gc_wb(codeinst, src->edges); - } - jl_atomic_store_release(&codeinst->debuginfo, src->debuginfo); - jl_gc_wb(codeinst, src->debuginfo); - jl_atomic_store_release(&codeinst->inferred, compressed); - jl_gc_wb(codeinst, compressed); - } } jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t world) @@ -2949,7 +3511,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t int should_skip_inference = !jl_is_method(mi->def.method) || jl_symbol_name(mi->def.method->name)[0] == '@'; if (!should_skip_inference) { - codeinst = jl_type_infer(mi, world, SOURCE_MODE_ABI); + codeinst = jl_type_infer(mi, world, SOURCE_MODE_ABI, jl_options.trim); } } @@ -3015,19 +3577,19 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t return codeinst; } -JL_DLLEXPORT jl_value_t *jl_fptr_const_return(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) +jl_value_t *jl_fptr_const_return(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) { return m->rettype_const; } -JL_DLLEXPORT jl_value_t *jl_fptr_args(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) +jl_value_t *jl_fptr_args(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) { jl_fptr_args_t invoke = jl_atomic_load_relaxed(&m->specptr.fptr1); assert(invoke && "Forgot to set specptr for jl_fptr_args!"); return invoke(f, args, nargs); } -JL_DLLEXPORT jl_value_t *jl_fptr_sparam(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) +jl_value_t *jl_fptr_sparam(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) { jl_svec_t *sparams = jl_get_ci_mi(m)->sparam_vals; assert(sparams != jl_emptysvec); @@ -3072,6 +3634,7 @@ JL_DLLEXPORT const jl_callptr_t jl_fptr_const_return_addr = &jl_fptr_const_retur JL_DLLEXPORT const jl_callptr_t jl_fptr_sparam_addr = &jl_fptr_sparam; +JL_CALLABLE(jl_f_opaque_closure_call); JL_DLLEXPORT const jl_callptr_t jl_f_opaque_closure_call_addr = (jl_callptr_t)&jl_f_opaque_closure_call; JL_DLLEXPORT const jl_callptr_t jl_fptr_wait_for_compiled_addr = &jl_fptr_wait_for_compiled; @@ -3093,14 +3656,13 @@ JL_DLLEXPORT int32_t jl_invoke_api(jl_code_instance_t *codeinst) return -1; } -JL_DLLEXPORT jl_value_t *jl_normalize_to_compilable_sig(jl_methtable_t *mt, jl_tupletype_t *ti, jl_svec_t *env, jl_method_t *m, +JL_DLLEXPORT jl_value_t *jl_normalize_to_compilable_sig(jl_tupletype_t *ti, jl_svec_t *env, jl_method_t *m, int return_if_compileable) { jl_tupletype_t *tt = NULL; jl_svec_t *newparams = NULL; JL_GC_PUSH2(&tt, &newparams); - jl_methtable_t *kwmt = mt == jl_kwcall_mt ? jl_kwmethod_table_for(m->sig) : mt; - intptr_t max_varargs = get_max_varargs(m, kwmt, mt, NULL); + intptr_t max_varargs = get_max_varargs(m, NULL); jl_compilation_sig(ti, env, m, max_varargs, &newparams); int is_compileable = ((jl_datatype_t*)ti)->isdispatchtuple; if (newparams) { @@ -3126,10 +3688,7 @@ jl_method_instance_t *jl_normalize_to_compilable_mi(jl_method_instance_t *mi JL_ jl_method_t *def = mi->def.method; if (!jl_is_method(def) || !jl_is_datatype(mi->specTypes)) return mi; - jl_methtable_t *mt = jl_method_get_table(def); - if ((jl_value_t*)mt == jl_nothing) - return mi; - jl_value_t *compilationsig = jl_normalize_to_compilable_sig(mt, (jl_datatype_t*)mi->specTypes, mi->sparam_vals, def, 1); + jl_value_t *compilationsig = jl_normalize_to_compilable_sig((jl_datatype_t*)mi->specTypes, mi->sparam_vals, def, 1); if (compilationsig == jl_nothing || jl_egal(compilationsig, mi->specTypes)) return mi; jl_svec_t *env = NULL; @@ -3150,30 +3709,27 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *matc jl_tupletype_t *ti = match->spec_types; jl_method_instance_t *mi = NULL; if (jl_is_datatype(ti)) { - jl_methtable_t *mt = jl_method_get_table(m); - assert(mt != NULL); - if ((jl_value_t*)mt != jl_nothing) { - // get the specialization, possibly also caching it - if (mt_cache && ((jl_datatype_t*)ti)->isdispatchtuple) { - // Since we also use this presence in the cache - // to trigger compilation when producing `.ji` files, - // inject it there now if we think it will be - // used via dispatch later (e.g. because it was hinted via a call to `precompile`) - JL_LOCK(&mt->writelock); - mi = cache_method(mt, &mt->cache, (jl_value_t*)mt, ti, m, world, min_valid, max_valid, env); - JL_UNLOCK(&mt->writelock); - } - else { - jl_value_t *tt = jl_normalize_to_compilable_sig(mt, ti, env, m, 1); - if (tt != jl_nothing) { - JL_GC_PUSH2(&tt, &env); - if (!jl_egal(tt, (jl_value_t*)ti)) { - jl_value_t *ti = jl_type_intersection_env((jl_value_t*)tt, (jl_value_t*)m->sig, &env); - assert(ti != jl_bottom_type); (void)ti; - } - mi = jl_specializations_get_linfo(m, (jl_value_t*)tt, env); - JL_GC_POP(); + // get the specialization, possibly also caching it + if (mt_cache && ((jl_datatype_t*)ti)->isdispatchtuple) { + // Since we also use this presence in the cache + // to trigger compilation when producing `.ji` files, + // inject it there now if we think it will be + // used via dispatch later (e.g. because it was hinted via a call to `precompile`) + jl_methcache_t *mc = jl_method_table->cache; + assert(mc); + JL_LOCK(&mc->writelock); + mi = cache_method(jl_method_get_table(m), mc, &mc->cache, (jl_value_t*)mc, ti, m, world, min_valid, max_valid, env); + } + else { + jl_value_t *tt = jl_normalize_to_compilable_sig(ti, env, m, 1); + if (tt != jl_nothing) { + JL_GC_PUSH2(&tt, &env); + if (!jl_egal(tt, (jl_value_t*)ti)) { + jl_value_t *ti = jl_type_intersection_env((jl_value_t*)tt, (jl_value_t*)m->sig, &env); + assert(ti != jl_bottom_type); (void)ti; } + mi = jl_specializations_get_linfo(m, (jl_value_t*)tt, env); + JL_GC_POP(); } } } @@ -3181,6 +3737,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_match_to_mi(jl_method_match_t *matc } // compile-time method lookup +// intersect types with the MT, and return a single compileable specialization that covers the intersection. jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types, size_t world, int mt_cache) { if (jl_has_free_typevars((jl_value_t*)types)) @@ -3273,7 +3830,7 @@ static void _generate_from_hint(jl_method_instance_t *mi, size_t world) { jl_value_t *codeinst = jl_rettype_inferred_native(mi, world, world); if (codeinst == jl_nothing) { - (void)jl_type_infer(mi, world, SOURCE_MODE_NOT_REQUIRED); + (void)jl_type_infer(mi, world, SOURCE_MODE_NOT_REQUIRED, jl_options.trim); codeinst = jl_rettype_inferred_native(mi, world, world); } if (codeinst != jl_nothing) { @@ -3316,10 +3873,10 @@ JL_DLLEXPORT void jl_compile_method_instance(jl_method_instance_t *mi, jl_tuplet miflags = jl_atomic_load_relaxed(&mi2->flags) | JL_MI_FLAGS_MASK_PRECOMPILED; jl_atomic_store_relaxed(&mi2->flags, miflags); if (jl_rettype_inferred_native(mi2, world, world) == jl_nothing) - (void)jl_type_infer(mi2, world, SOURCE_MODE_NOT_REQUIRED); + (void)jl_type_infer(mi2, world, SOURCE_MODE_NOT_REQUIRED, jl_options.trim); if (jl_typeinf_func && jl_atomic_load_relaxed(&mi->def.method->primary_world) <= tworld) { if (jl_rettype_inferred_native(mi2, tworld, tworld) == jl_nothing) - (void)jl_type_infer(mi2, tworld, SOURCE_MODE_NOT_REQUIRED); + (void)jl_type_infer(mi2, tworld, SOURCE_MODE_NOT_REQUIRED, jl_options.trim); } } } @@ -3336,6 +3893,15 @@ JL_DLLEXPORT void jl_compile_method_sig(jl_method_t *m, jl_value_t *types, jl_sv jl_compile_method_instance(mi, NULL, world); } +JL_DLLEXPORT int jl_is_compilable(jl_tupletype_t *types) +{ + size_t world = jl_atomic_load_acquire(&jl_world_counter); + size_t min_valid = 0; + size_t max_valid = ~(size_t)0; + jl_method_instance_t *mi = jl_get_compile_hint_specialization(types, world, &min_valid, &max_valid, 1); + return mi == NULL ? 0 : 1; +} + JL_DLLEXPORT int jl_compile_hint(jl_tupletype_t *types) { size_t world = jl_atomic_load_acquire(&jl_world_counter); @@ -3427,17 +3993,11 @@ STATIC_INLINE jl_value_t *verify_type(jl_value_t *v) JL_NOTSAFEPOINT STATIC_INLINE jl_value_t *_jl_invoke(jl_value_t *F, jl_value_t **args, uint32_t nargs, jl_method_instance_t *mfunc, size_t world) { - // manually inlined copy of jl_method_compiled - jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mfunc->cache); - while (codeinst) { - if (jl_atomic_load_relaxed(&codeinst->min_world) <= world && world <= jl_atomic_load_relaxed(&codeinst->max_world)) { - jl_callptr_t invoke = jl_atomic_load_acquire(&codeinst->invoke); - if (invoke != NULL) { - jl_value_t *res = invoke(F, args, nargs, codeinst); - return verify_type(res); - } - } - codeinst = jl_atomic_load_relaxed(&codeinst->next); + jl_code_instance_t *codeinst = NULL; + jl_callptr_t invoke = jl_method_compiled_callptr(mfunc, world, &codeinst); + if (invoke) { + jl_value_t *res = invoke(F, args, nargs, codeinst); + return verify_type(res); } int64_t last_alloc = jl_options.malloc_log ? jl_gc_diff_total_bytes() : 0; int last_errno = errno; @@ -3451,7 +4011,7 @@ STATIC_INLINE jl_value_t *_jl_invoke(jl_value_t *F, jl_value_t **args, uint32_t errno = last_errno; if (jl_options.malloc_log) jl_gc_sync_total_bytes(last_alloc); // discard allocation count from compilation - jl_callptr_t invoke = jl_atomic_load_acquire(&codeinst->invoke); + invoke = jl_atomic_load_acquire(&codeinst->invoke); jl_value_t *res = invoke(F, args, nargs, codeinst); return verify_type(res); } @@ -3547,7 +4107,6 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t (callsite >> 16) & (N_CALL_CACHE - 1), (callsite >> 24 | callsite << 8) & (N_CALL_CACHE - 1)}; jl_typemap_entry_t *entry = NULL; - jl_methtable_t *mt = NULL; int i; // check each cache entry to see if it matches //#pragma unroll @@ -3574,19 +4133,19 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t if (i == 4) { // if no method was found in the associative cache, check the full cache JL_TIMING(METHOD_LOOKUP_FAST, METHOD_LOOKUP_FAST); - mt = jl_gf_mtable(F); - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_methcache_t *mc = jl_method_table->cache; + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); entry = NULL; - if (leafcache != (jl_genericmemory_t*)jl_an_empty_memory_any && - jl_typetagis(jl_atomic_load_relaxed(&mt->cache), jl_typemap_level_type)) { - // hashing args is expensive, but looking at mt->cache is probably even more expensive + int cache_entry_count = jl_atomic_load_relaxed(&((jl_datatype_t*)FT)->name->cache_entry_count); + if (leafcache != (jl_genericmemory_t*)jl_an_empty_memory_any && (cache_entry_count == 0 || cache_entry_count >= 8)) { + // hashing args is expensive, but so do that only if looking at mc->cache is probably even more expensive tt = lookup_arg_type_tuple(F, args, nargs); if (tt != NULL) entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); } if (entry == NULL) { - jl_typemap_t *cache = jl_atomic_load_relaxed(&mt->cache); // XXX: gc root required? - entry = jl_typemap_assoc_exact(cache, F, args, nargs, jl_cachearg_offset(mt), world); + jl_typemap_t *cache = jl_atomic_load_relaxed(&mc->cache); // XXX: gc root required? + entry = jl_typemap_assoc_exact(cache, F, args, nargs, jl_cachearg_offset(), world); if (entry == NULL) { last_alloc = jl_options.malloc_log ? jl_gc_diff_total_bytes() : 0; if (tt == NULL) { @@ -3604,6 +4163,11 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t jl_atomic_store_relaxed(&pick_which[cache_idx[0]], which); jl_atomic_store_release(&call_cache[cache_idx[which & 3]], entry); } + if (entry) { + // mfunc was found in slow path, so log --trace-dispatch + jl_method_instance_t *mfunc = entry->func.linfo; + record_dispatch_statement_on_first_dispatch(mfunc); + } } jl_method_instance_t *mfunc; @@ -3614,7 +4178,8 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t else { assert(tt); // cache miss case - mfunc = jl_mt_assoc_by_type(mt, tt, world); + jl_methcache_t *mc = jl_method_table->cache; + mfunc = jl_mt_assoc_by_type(mc, tt, world); if (jl_options.malloc_log) jl_gc_sync_total_bytes(last_alloc); // discard allocation count from compilation if (mfunc == NULL) { @@ -3625,23 +4190,15 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t jl_method_error(F, args, nargs, world); // unreachable } - // mfunc is about to be dispatched - uint8_t force_trace_dispatch = jl_atomic_load_relaxed(&jl_force_trace_dispatch_enabled); - if (force_trace_dispatch || jl_options.trace_dispatch != NULL) { - uint8_t miflags = jl_atomic_load_relaxed(&mfunc->flags); - uint8_t was_dispatched = miflags & JL_MI_FLAGS_MASK_DISPATCHED; - if (!was_dispatched) { - miflags |= JL_MI_FLAGS_MASK_DISPATCHED; - jl_atomic_store_relaxed(&mfunc->flags, miflags); - record_dispatch_statement(mfunc); - } - } + // mfunc was found in slow path, so log --trace-dispatch + record_dispatch_statement_on_first_dispatch(mfunc); } #ifdef JL_TRACE if (traceen) jl_printf(JL_STDOUT, " at %s:%d\n", jl_symbol_name(mfunc->def.method->file), mfunc->def.method->line); #endif + return mfunc; } @@ -3655,18 +4212,15 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t *F, jl_value_t **args, uint return _jl_invoke(F, args, nargs, mfunc, world); } -static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_value_t *mt, size_t world, size_t *min_valid, size_t *max_valid) +static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT, jl_methtable_t *mt, size_t world, int cache_result, size_t *min_valid, size_t *max_valid) { jl_value_t *unw = jl_unwrap_unionall((jl_value_t*)types); if (!jl_is_tuple_type(unw)) return NULL; if (jl_tparam0(unw) == jl_bottom_type) return NULL; - if (mt == jl_nothing) - mt = (jl_value_t*)jl_method_table_for(unw); - if (mt == jl_nothing) - mt = NULL; - jl_value_t *matches = ml_matches((jl_methtable_t*)mt, (jl_tupletype_t*)types, 1, 0, 0, world, 1, min_valid, max_valid, NULL); + jl_methcache_t *mc = ((jl_methtable_t*)mt)->cache; + jl_value_t *matches = ml_matches((jl_methtable_t*)mt, mc, (jl_tupletype_t*)types, 1, 0, 0, world, cache_result, min_valid, max_valid, NULL); if (matches == jl_nothing || jl_array_nrows(matches) != 1) return NULL; jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(matches, 0); @@ -3678,7 +4232,9 @@ JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup(jl_value_t *types, jl_value_t *mt, // Deprecated: Use jl_gf_invoke_lookup_worlds for future development size_t min_valid = 0; size_t max_valid = ~(size_t)0; - jl_method_match_t *matc = _gf_invoke_lookup(types, mt, world, &min_valid, &max_valid); + if (mt == jl_nothing) + mt = (jl_value_t*)jl_method_table; + jl_method_match_t *matc = _gf_invoke_lookup(types, (jl_methtable_t*)mt, world, 1, &min_valid, &max_valid); if (matc == NULL) return jl_nothing; return (jl_value_t*)matc->method; @@ -3687,7 +4243,9 @@ JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup(jl_value_t *types, jl_value_t *mt, JL_DLLEXPORT jl_value_t *jl_gf_invoke_lookup_worlds(jl_value_t *types, jl_value_t *mt, size_t world, size_t *min_world, size_t *max_world) { - jl_method_match_t *matc = _gf_invoke_lookup(types, mt, world, min_world, max_world); + if (mt == jl_nothing) + mt = (jl_value_t*)jl_method_table; + jl_method_match_t *matc = _gf_invoke_lookup(types, (jl_methtable_t*)mt, world, 1, min_world, max_world); if (matc == NULL) return jl_nothing; return (jl_value_t*)matc; @@ -3749,8 +4307,7 @@ jl_value_t *jl_gf_invoke_by_method(jl_method_t *method, jl_value_t *gf, jl_value int sub = jl_subtype_matching((jl_value_t*)tt, (jl_value_t*)method->sig, &tpenv); assert(sub); (void)sub; } - - mfunc = cache_method(NULL, &method->invokes, (jl_value_t*)method, tt, method, 1, 1, ~(size_t)0, tpenv); + mfunc = cache_method(NULL, NULL, &method->invokes, (jl_value_t*)method, tt, method, 1, 1, ~(size_t)0, tpenv); } JL_UNLOCK(&method->writelock); JL_GC_POP(); @@ -3794,8 +4351,8 @@ jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_ 0, 0, 0); assert(jl_is_datatype(ftype)); JL_GC_PUSH1(&ftype); - ftype->name->mt->name = name; - jl_gc_wb(ftype->name->mt, name); + ftype->name->singletonname = name; + jl_gc_wb(ftype->name, name); jl_declare_constant_val3(NULL, module, tname, (jl_value_t*)ftype, PARTITION_KIND_CONST, new_world); jl_value_t *f = jl_new_struct(ftype); ftype->instance = f; @@ -3857,31 +4414,29 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio closure->match.min_valid = max_world + 1; return 1; } + if (closure->match.max_valid > max_world) + closure->match.max_valid = max_world; jl_method_t *meth = ml->func.method; - if (closure->lim >= 0 && jl_is_dispatch_tupletype(meth->sig)) { - int replaced = 0; - // check if this is replaced, in which case we need to avoid double-counting it against the limit - // (although it will figure out later which one to keep and return) - size_t len = jl_array_nrows(closure->t); - for (int i = 0; i < len; i++) { - if (jl_types_equal(((jl_method_match_t*)jl_array_ptr_ref(closure->t, i))->method->sig, meth->sig)) { - replaced = 1; - break; - } - } - if (!replaced) { - if (closure->lim == 0) - return 0; - closure->lim--; + int only = jl_atomic_load_relaxed(&meth->dispatch_status) & METHOD_SIG_LATEST_ONLY; + if (closure->lim >= 0 && only) { + if (closure->lim == 0) { + closure->t = jl_an_empty_vec_any; + return 0; } + closure->lim--; } - // don't need to consider other similar methods if this ml will always fully intersect with them and dominates all of them - if (!closure->include_ambiguous || closure->lim != -1) - typemap_slurp_search(ml, &closure->match); closure->matc = make_method_match((jl_tupletype_t*)closure->match.ti, closure->match.env, meth, closure->match.issubty ? FULLY_COVERS : NOT_FULLY_COVERS); size_t len = jl_array_nrows(closure->t); + if (closure->match.issubty && only) { + if (len == 0) + closure->t = (jl_value_t*)jl_alloc_vec_any(1); + else if (len > 1) + jl_array_del_end((jl_array_t*)closure->t, len - 1); + jl_array_ptr_set(closure->t, 0, (jl_value_t*)closure->matc); + return 0; + } if (len == 0) { closure->t = (jl_value_t*)jl_alloc_vec_any(1); jl_array_ptr_set(closure->t, 0, (jl_value_t*)closure->matc); @@ -3889,19 +4444,15 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio else { jl_array_ptr_1d_push((jl_array_t*)closure->t, (jl_value_t*)closure->matc); } + // don't need to consider other similar methods if this ml will always fully intersect with them and dominates all of them + if (!closure->include_ambiguous || closure->lim != -1) + typemap_slurp_search(ml, &closure->match); return 1; } -static int ml_mtable_visitor(jl_methtable_t *mt, void *closure0) -{ - struct typemap_intersection_env* env = (struct typemap_intersection_env*)closure0; - return jl_typemap_intersection_visitor(jl_atomic_load_relaxed(&mt->defs), jl_cachearg_offset(mt), env); -} - // Visit the candidate methods, starting from t[idx], to determine a possible valid sort ordering, // where every morespecific method appears before any method which it has a common -// intersection with but is not partly ambiguous with (ambiguity is transitive, particularly -// if lim==-1, although morespecific is not transitive). +// intersection with but is not partly ambiguous with (ambiguity is not transitive, since morespecific is not transitive). // Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable // Inputs: // * `t`: the array of vertexes (method matches) @@ -3909,262 +4460,260 @@ static int ml_mtable_visitor(jl_methtable_t *mt, void *closure0) // * `visited`: the state of the algorithm for each vertex in `t`: either 1 if we visited it already or 1+depth if we are visiting it now // * `stack`: the state of the algorithm for the current vertex (up to length equal to `t`): the list of all vertexes currently in the depth-first path or in the current SCC // * `result`: the output of the algorithm, a sorted list of vertexes (up to length `lim`) -// * `allambig`: a list of all vertexes with an ambiguity (up to length equal to `t`), discovered while running the rest of the algorithm +// * `recursion_stack`: an array for temporary use // * `lim`: either -1 for unlimited matches, or the maximum length for `result` before returning failure (return -1). -// If specified as -1, this will return extra matches that would have been elided from the list because they were already covered by an earlier match. -// This gives a sort of maximal set of matching methods (up to the first minmax method). -// If specified as -1, the sorting will also include all "weak" edges (every ambiguous pair) which will create much larger ambiguity cycles, -// resulting in a less accurate sort order and much less accurate `*has_ambiguity` result. // * `include_ambiguous`: whether to filter out fully ambiguous matches from `result` // * `*has_ambiguity`: whether the algorithm does not need to compute if there is an unresolved ambiguity // * `*found_minmax`: whether there is a minmax method already found, so future fully_covers matches should be ignored // Outputs: -// * `*has_ambiguity`: whether the caller should check if there remains an unresolved ambiguity (in `allambig`) +// * `*has_ambiguity`: whether there are any ambiguities that mean the sort order is not exact +// Stack frame for iterative sort_mlmatches implementation +enum sort_state { + STATE_VISITING, // Initial visit and setup + STATE_PROCESSING_INTERFERENCES, // Processing interference loop + STATE_CHECK_COVERS, // Check coverage conditions + STATE_FINALIZE_SCC // SCC processing and cleanup +}; + +typedef struct { + size_t idx; // Current method match index + size_t interference_index; // Current position in interferences loop + size_t interference_count; // Total interferences count + size_t depth; // Stack depth when frame created + size_t cycle; // Cycle depth tracking + jl_method_match_t *matc; // Current method match + jl_method_t *m; // Current method + jl_value_t *ti; // Type intersection + int subt; // Subtype flag + jl_genericmemory_t *interferences; // Method interferences + int child_result; // Result from child recursive call + enum sort_state state; +} sort_stack_frame_t; + // Returns: // * -1: too many matches for lim, other outputs are undefined // * 0: the child(ren) have been added to the output // * 1+: the children are part of this SCC (up to this depth) -// TODO: convert this function into an iterative call, rather than recursive -static int sort_mlmatches(jl_array_t *t, size_t idx, arraylist_t *visited, arraylist_t *stack, arraylist_t *result, arraylist_t *allambig, int lim, int include_ambiguous, int *has_ambiguity, int *found_minmax) -{ - size_t cycle = (size_t)visited->items[idx]; - if (cycle != 0) - return cycle - 1; // depth remaining - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, idx); - jl_method_t *m = matc->method; - jl_value_t *ti = (jl_value_t*)matc->spec_types; - int subt = matc->fully_covers != NOT_FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m->sig) - // first check if this new method is actually already fully covered by an - // existing match and we can just ignore this entry quickly - size_t result_len = 0; - if (subt) { - if (*found_minmax == 2) - visited->items[idx] = (void*)1; - } - else if (lim != -1) { - for (; result_len < result->len; result_len++) { - size_t idx2 = (size_t)result->items[result_len]; - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, idx2); - jl_method_t *m2 = matc2->method; - if (jl_subtype(ti, m2->sig)) { - if (include_ambiguous) { - if (!jl_method_morespecific(m2, m)) - continue; +static int sort_mlmatches(jl_array_t *t, size_t idx, arraylist_t *visited, arraylist_t *stack, arraylist_t *result, arraylist_t *recursion_stack, int lim, int include_ambiguous, int *has_ambiguity, int *found_minmax) +{ + // Use arraylist_t for explicit stack of processing frames + arraylist_t frame_stack; + arraylist_new(&frame_stack, 0); + + // Push initial frame + sort_stack_frame_t initial_frame = { + .idx = idx, + .interference_index = 0, + .interference_count = 0, + .depth = 0, + .cycle = 0, + .matc = NULL, + .m = NULL, + .ti = NULL, + .subt = 0, + .interferences = NULL, + .child_result = 0, + .state = STATE_VISITING + }; + arraylist_push(&frame_stack, memcpy(malloc(sizeof(sort_stack_frame_t)), &initial_frame, sizeof(sort_stack_frame_t))); + + int final_result = 0; + + while (1) { + sort_stack_frame_t *current = (sort_stack_frame_t*)frame_stack.items[frame_stack.len - 1]; + JL_GC_PROMISE_ROOTED(current->m); + JL_GC_PROMISE_ROOTED(current->interferences); + JL_GC_PROMISE_ROOTED(current->ti); + + switch (current->state) { + case STATE_VISITING: { + size_t cycle = (size_t)visited->items[current->idx]; + if (cycle != 0) { + final_result = cycle - 1; + goto propagate_to_parent; } - visited->items[idx] = (void*)1; + + arraylist_push(stack, (void*)current->idx); + current->depth = stack->len; + visited->items[current->idx] = (void*)(1 + current->depth); + current->matc = (jl_method_match_t*)jl_array_ptr_ref(t, current->idx); + current->m = current->matc->method; + current->ti = (jl_value_t*)current->matc->spec_types; + current->subt = current->matc->fully_covers != NOT_FULLY_COVERS; + current->interferences = jl_atomic_load_relaxed(¤t->m->interferences); + current->cycle = current->depth; + current->interference_count = current->interferences->length; + current->interference_index = 0; + current->state = STATE_PROCESSING_INTERFERENCES; break; } - } - } - if ((size_t)visited->items[idx] == 1) - return 0; - arraylist_push(stack, (void*)idx); - size_t depth = stack->len; - visited->items[idx] = (void*)(1 + depth); - cycle = depth; - int addambig = 0; - int mayexclude = 0; - // First visit all "strong" edges where the child is definitely better. - // This likely won't hit any cycles, but might (because morespecific is not transitive). - // Along the way, record if we hit any ambiguities-we may need to track those later. - for (size_t childidx = 0; childidx < jl_array_nrows(t); childidx++) { - if (childidx == idx) - continue; - int child_cycle = (size_t)visited->items[childidx]; - if (child_cycle == 1) - continue; // already handled - if (child_cycle != 0 && child_cycle - 1 >= cycle) - continue; // already part of this cycle - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - jl_method_t *m2 = matc2->method; - int subt2 = matc2->fully_covers == FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m2->sig) - // TODO: we could change this to jl_has_empty_intersection(ti, (jl_value_t*)matc2->spec_types); - // since we only care about sorting of the intersections the user asked us about - if (!subt2 && jl_has_empty_intersection(m2->sig, m->sig)) - continue; - int msp = jl_method_morespecific(m, m2); - int msp2 = !msp && jl_method_morespecific(m2, m); - if (!msp) { - if (subt || !include_ambiguous || (lim != -1 && msp2)) { - if (subt2 || ((lim != -1 || (!include_ambiguous && !msp2)) && jl_subtype((jl_value_t*)ti, m2->sig))) { - // this may be filtered out as fully intersected, if applicable later - mayexclude = 1; + + case STATE_PROCESSING_INTERFERENCES: { + // If we have a child result to process, handle it first + if (current->child_result != 0) { + if (current->child_result == -1) { + final_result = -1; + goto propagate_to_parent; + } + // record the cycle will resolve at depth "cycle" + if (current->child_result && current->child_result < current->cycle) + current->cycle = current->child_result; + current->child_result = 0; // Clear after processing } - } - if (!msp2) { - addambig = 1; // record there is a least one previously-undetected ambiguity that may need to be investigated later (between m and m2) - } - } - if (lim == -1 ? msp : !msp2) // include only strong or also weak edges, depending on whether the result size is limited - continue; - // m2 is (lim!=-1 ? better : not-worse), so attempt to visit it first - // if limited, then we want to visit only better edges, because that results in finding k best matches quickest - // if not limited, then we want to visit all edges, since that results in finding the largest SCC cycles, which requires doing the fewest intersections - child_cycle = sort_mlmatches(t, childidx, visited, stack, result, allambig, lim, include_ambiguous, has_ambiguity, found_minmax); - if (child_cycle == -1) - return -1; - if (child_cycle && child_cycle < cycle) { - // record the cycle will resolve at depth "cycle" - cycle = child_cycle; - } - if (stack->len == depth) { - // if this child resolved without hitting a cycle, then there is - // some probability that this method is already fully covered now - // (same check as before), and we can delete this vertex now without - // anyone noticing (too much) - if (subt) { - if (*found_minmax == 2) - visited->items[idx] = (void*)1; - } - else if (lim != -1) { - for (; result_len < result->len; result_len++) { - size_t idx2 = (size_t)result->items[result_len]; - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, idx2); - jl_method_t *m2 = matc2->method; - if (jl_subtype(ti, m2->sig)) { - if (include_ambiguous) { - if (!jl_method_morespecific(m2, m)) - continue; + + // Process interferences iteratively + while (current->interference_index < current->interference_count) { + jl_method_t *m2 = (jl_method_t*)jl_genericmemory_ptr_ref(current->interferences, current->interference_index); + current->interference_index++; + + if (m2 == NULL) + continue; + + int childidx = find_method_in_matches(t, m2); + if (childidx < 0 || (size_t)childidx == current->idx) + continue; + + int child_cycle = (size_t)visited->items[childidx]; + if (child_cycle == 1) + continue; // already handled + if (child_cycle != 0 && child_cycle - 1 >= current->cycle) + continue; // already part of this cycle + if (method_in_interferences(current->m, m2)) + continue; + + // m2 is morespecific, so attempt to visit it first + if (child_cycle != 0) { + // Child already being processed, use cached result + int child_result = child_cycle - 1; + if (child_result == -1) { + final_result = -1; + goto propagate_to_parent; } - visited->items[idx] = (void*)1; - break; + if (child_result && child_result < current->cycle) + current->cycle = child_result; } - } - } - if ((size_t)visited->items[idx] == 1) { - // n.b. cycle might be < depth, if we had a cycle with a child - // idx, but since we are on the top of the stack, nobody - // observed that and so we are content to ignore this - size_t childidx = (size_t)arraylist_pop(stack); - assert(childidx == idx); (void)childidx; - assert(!subt || *found_minmax == 2); - return 0; - } - } - } - if (matc->fully_covers == NOT_FULLY_COVERS && addambig) - arraylist_push(allambig, (void*)idx); - if (cycle != depth) - return cycle; - result_len = result->len; - if (stack->len == depth) { - // Found one "best" method to add right now. But we might exclude it if - // we determined earlier that we had that option. - if (mayexclude) { - if (!subt || *found_minmax == 2) - visited->items[idx] = (void*)1; - } - } - else { - // We have a set of ambiguous methods. Record that. - // This is greatly over-approximated for lim==-1 - *has_ambiguity = 1; - // If we followed weak edges above, then this also fully closed the ambiguity cycle - if (lim == -1) - addambig = 0; - // If we're only returning possible matches, now filter out this method - // if its intersection is fully ambiguous in this SCC group. - // This is a repeat of the "first check", now that we have completed the cycle analysis - for (size_t i = depth - 1; i < stack->len; i++) { - size_t childidx = (size_t)stack->items[i]; - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - jl_value_t *ti = (jl_value_t*)matc->spec_types; - int subt = matc->fully_covers != NOT_FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m->sig) - if ((size_t)visited->items[childidx] == 1) { - assert(subt); - continue; - } - assert(visited->items[childidx] == (void*)(2 + i)); - // if we only followed strong edges before above - // check also if this set has an unresolved ambiguity missing from it - if (lim != -1 && !addambig) { - for (size_t j = 0; j < allambig->len; j++) { - if ((size_t)allambig->items[j] == childidx) { - addambig = 1; - break; + else { + // Need to process child - push new frame and pause current processing + sort_stack_frame_t child_frame = { + .idx = childidx, + .interference_index = 0, + .interference_count = 0, + .depth = 0, + .cycle = 0, + .matc = NULL, + .m = NULL, + .ti = NULL, + .subt = 0, + .interferences = NULL, + .child_result = 0, + .state = STATE_VISITING + }; + arraylist_push(&frame_stack, memcpy(malloc(sizeof(sort_stack_frame_t)), &child_frame, sizeof(sort_stack_frame_t))); + goto continue_main_loop; // Resume processing after child completes } } + + current->state = STATE_CHECK_COVERS; + break; } - // always remove fully_covers matches after the first minmax ambiguity group is handled - if (subt) { - if (*found_minmax) - visited->items[childidx] = (void*)1; - continue; + + case STATE_CHECK_COVERS: { + // There is some probability that this method is already fully covered + // now, and we can delete this vertex now without anyone noticing. + if (current->subt && *found_minmax) { + if (*found_minmax == 2) + visited->items[current->idx] = (void*)1; + } + else if (check_interferences_covers(current->m, current->ti, t, visited, recursion_stack)) { + visited->items[current->idx] = (void*)1; + } + else if (check_fully_ambiguous(current->m, current->ti, t, include_ambiguous, has_ambiguity)) { + visited->items[current->idx] = (void*)1; + } + + // If there were no cycles hit either, then we can potentially delete all of its edges too. + if ((size_t)visited->items[current->idx] == 1 && stack->len == current->depth) { + // n.b. cycle might be < depth, if we had a cycle with a child + // idx, but since we are on the top of the stack, nobody + // observed that and so we are content to ignore this + size_t childidx = (size_t)arraylist_pop(stack); + assert(childidx == current->idx); (void)childidx; + final_result = 0; + goto propagate_to_parent; + } + + if (current->cycle != current->depth) { + final_result = current->cycle; + goto propagate_to_parent; + } + + current->state = STATE_FINALIZE_SCC; + break; } - else if (lim != -1) { - // when limited, don't include this match if it was covered by an earlier one - for (size_t result_len = 0; result_len < result->len; result_len++) { - size_t idx2 = (size_t)result->items[result_len]; - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, idx2); - jl_method_t *m2 = matc2->method; - if (jl_subtype(ti, m2->sig)) { - if (include_ambiguous) { - if (!jl_method_morespecific(m2, m)) - continue; - } - visited->items[childidx] = (void*)1; - break; + + case STATE_FINALIZE_SCC: { + // If this is in an SCC group, do some additional checks before returning or setting has_ambiguity + if (current->depth != stack->len) { + int scc_count = 0; + for (size_t i = current->depth - 1; i < stack->len; i++) { + size_t childidx = (size_t)stack->items[i]; + if (visited->items[childidx] == (void*)1) + continue; + scc_count++; } + if (scc_count > 1) + *has_ambiguity = 1; } - } - } - if (!include_ambiguous && lim == -1) { - for (size_t i = depth - 1; i < stack->len; i++) { - size_t childidx = (size_t)stack->items[i]; - if ((size_t)visited->items[childidx] == 1) - continue; - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - jl_method_t *m = matc->method; - jl_value_t *ti = (jl_value_t*)matc->spec_types; - for (size_t j = depth - 1; j < stack->len; j++) { - if (i == j) + + // copy this cycle into the results + for (size_t i = current->depth - 1; i < stack->len; i++) { + size_t childidx = (size_t)stack->items[i]; + jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); + int subt = matc->fully_covers != NOT_FULLY_COVERS; + if (subt && *found_minmax) + visited->items[childidx] = (void*)1; + if ((size_t)visited->items[childidx] == 1) continue; - size_t idx2 = (size_t)stack->items[j]; - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(t, idx2); - jl_method_t *m2 = matc2->method; - int subt2 = matc2->fully_covers == FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m2->sig) - // if their intersection contributes to the ambiguity cycle - // and the contribution of m is fully ambiguous with the portion of the cycle from m2 - if (subt2 || jl_subtype((jl_value_t*)ti, m2->sig)) { - // but they aren't themselves simply ordered (here - // we don't consider that a third method might be - // disrupting that ordering and just consider them - // pairwise to keep this simple). - if (!jl_method_morespecific(m, m2) && !jl_method_morespecific(m2, m)) { - visited->items[childidx] = (void*)-1; - break; - } + assert(visited->items[childidx] == (void*)(2 + i)); + visited->items[childidx] = (void*)1; + if (lim == -1 || result->len < lim) + arraylist_push(result, (void*)childidx); + else { + final_result = -1; + goto propagate_to_parent; } } + + // now finally cleanup the stack + while (stack->len >= current->depth) { + size_t childidx = (size_t)arraylist_pop(stack); + // always remove fully_covers matches after the first minmax ambiguity group is handled + jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); + int subt = matc->fully_covers == FULLY_COVERS; + if (subt && *found_minmax == 1) + *found_minmax = 2; + assert(visited->items[childidx] == (void*)1); + } + + final_result = 0; + goto propagate_to_parent; } } - } - // copy this cycle into the results - for (size_t i = depth - 1; i < stack->len; i++) { - size_t childidx = (size_t)stack->items[i]; - if ((size_t)visited->items[childidx] == 1) - continue; - if ((size_t)visited->items[childidx] != -1) { - assert(visited->items[childidx] == (void*)(2 + i)); - visited->items[childidx] = (void*)-1; - if (lim == -1 || result->len < lim) - arraylist_push(result, (void*)childidx); - else - return -1; - } - } - // now finally cleanup the stack - while (stack->len >= depth) { - size_t childidx = (size_t)arraylist_pop(stack); - // always remove fully_covers matches after the first minmax ambiguity group is handled - //jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(t, childidx); - if (matc->fully_covers != NOT_FULLY_COVERS && !addambig) - *found_minmax = 2; - if (visited->items[childidx] != (void*)-1) + + continue_main_loop: continue; - visited->items[childidx] = (void*)1; + + propagate_to_parent: + // Propagate result to parent if exists + free(arraylist_pop(&frame_stack)); + if (frame_stack.len == 0) + break; + sort_stack_frame_t *parent = (sort_stack_frame_t*)frame_stack.items[frame_stack.len - 1]; + parent->child_result = final_result; } - return 0; + assert(frame_stack.len == 0); + arraylist_free(&frame_stack); + return final_result; } @@ -4178,7 +4727,7 @@ static int sort_mlmatches(jl_array_t *t, size_t idx, arraylist_t *visited, array // fully-covers is a Bool indicating subtyping, though temporarily it may be // tri-values, with `nothing` indicating a match that is not a subtype, but // which is dominated by one which is (and thus should be excluded unless ambiguous) -static jl_value_t *ml_matches(jl_methtable_t *mt, +static jl_value_t *ml_matches(jl_methtable_t *mt, jl_methcache_t *mc, jl_tupletype_t *type, int lim, int include_ambiguous, int intersections, size_t world, int cache_result, size_t *min_valid, size_t *max_valid, int *ambig) @@ -4207,10 +4756,10 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_value_t *isect2 = NULL; JL_GC_PUSH6(&env.t, &env.matc, &env.match.env, &search.env, &env.match.ti, &isect2); - if (mt) { + if (mc) { // check the leaf cache if this type can be in there if (((jl_datatype_t*)unw)->isdispatchtuple) { - jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mc->leafcache); jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)type, world); if (entry) { jl_method_instance_t *mi = entry->func.linfo; @@ -4243,49 +4792,43 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, } // then check the full cache if it seems profitable if (((jl_datatype_t*)unw)->isdispatchtuple) { - jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mt->cache), &search, jl_cachearg_offset(mt), /*subtype*/1); + jl_typemap_entry_t *entry = jl_typemap_assoc_by_type(jl_atomic_load_relaxed(&mc->cache), &search, jl_cachearg_offset(), /*subtype*/1); if (entry && (((jl_datatype_t*)unw)->isdispatchtuple || entry->guardsigs == jl_emptysvec)) { jl_method_instance_t *mi = entry->func.linfo; jl_method_t *meth = mi->def.method; - if (!jl_is_unionall(meth->sig) && ((jl_datatype_t*)unw)->isdispatchtuple) { - env.match.env = jl_emptysvec; - env.match.ti = unw; - } - else { - // this just calls jl_subtype_env (since we know that `type <: meth->sig` by transitivity) - env.match.ti = jl_type_intersection_env((jl_value_t*)type, (jl_value_t*)meth->sig, &env.match.env); - } - env.matc = make_method_match((jl_tupletype_t*)env.match.ti, - env.match.env, meth, FULLY_COVERS); - env.t = (jl_value_t*)jl_alloc_vec_any(1); - jl_array_ptr_set(env.t, 0, env.matc); size_t min_world = jl_atomic_load_relaxed(&entry->min_world); - size_t max_world = jl_atomic_load_relaxed(&entry->max_world); - if (*min_valid < min_world) - *min_valid = min_world; - if (*max_valid > max_world) - *max_valid = max_world; - JL_GC_POP(); - return env.t; + // only return this if it appears min_would is fully computed, otherwise do the full lookup to compute min_world exactly + if (min_world == jl_atomic_load_relaxed(&meth->primary_world)) { + size_t max_world = jl_atomic_load_relaxed(&entry->max_world); + if (!jl_is_unionall(meth->sig) && ((jl_datatype_t*)unw)->isdispatchtuple) { + env.match.env = jl_emptysvec; + env.match.ti = unw; + } + else { + // this just calls jl_subtype_env (since we know that `type <: meth->sig` by transitivity) + env.match.ti = jl_type_intersection_env((jl_value_t*)type, (jl_value_t*)meth->sig, &env.match.env); + } + env.matc = make_method_match((jl_tupletype_t*)env.match.ti, + env.match.env, meth, FULLY_COVERS); + env.t = (jl_value_t*)jl_alloc_vec_any(1); + jl_array_ptr_set(env.t, 0, env.matc); + if (*min_valid < min_world) + *min_valid = min_world; + if (*max_valid > max_world) + *max_valid = max_world; + JL_GC_POP(); + return env.t; + } } } - if (!ml_mtable_visitor(mt, &env.match)) { - JL_GC_POP(); - // if we return early, set only the min/max valid collected from matching - *min_valid = env.match.min_valid; - *max_valid = env.match.max_valid; - return jl_nothing; - } } - else { - // else: scan everything - if (!jl_foreach_reachable_mtable(ml_mtable_visitor, &env.match)) { - JL_GC_POP(); - // if we return early, set only the min/max valid collected from matching - *min_valid = env.match.min_valid; - *max_valid = env.match.max_valid; - return jl_nothing; - } + // then scan everything + if (!jl_typemap_intersection_visitor(jl_atomic_load_relaxed(&mt->defs), 0, &env.match) && env.t == jl_an_empty_vec_any) { + JL_GC_POP(); + // if we return early without returning methods, set only the min/max valid collected from matching + *min_valid = env.match.min_valid; + *max_valid = env.match.max_valid; + return jl_nothing; } // if we return early, set only the min/max valid collected from matching *min_valid = env.match.min_valid; @@ -4294,43 +4837,31 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, env.match.ti = NULL; env.matc = NULL; env.match.env = NULL; search.env = NULL; size_t i, j, len = jl_array_nrows(env.t); jl_method_match_t *minmax = NULL; - int minmax_ambig = 0; - int all_subtypes = 1; + int any_subtypes = 0; if (len > 1) { // first try to pre-process the results to find the most specific - // result that fully covers the input, since we can do this in linear - // time, and the rest is O(n^2) + // result that fully covers the input, since we can do this in O(n^2) + // time, and the rest is O(n^3) // - first find a candidate for the best of these method results for (i = 0; i < len; i++) { jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, i); if (matc->fully_covers == FULLY_COVERS) { + any_subtypes = 1; jl_method_t *m = matc->method; - if (minmax != NULL) { - jl_method_t *minmaxm = minmax->method; - if (jl_method_morespecific(minmaxm, m)) + for (j = 0; j < len; j++) { + if (i == j) continue; + jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(env.t, j); + if (matc2->fully_covers == FULLY_COVERS) { + jl_method_t *m2 = matc2->method; + if (!method_morespecific_via_interferences(m, m2)) + break; + } } - minmax = matc; - } - else { - all_subtypes = 0; - } - } - // - then see if it dominated all of the other choices - if (minmax != NULL) { - for (i = 0; i < len; i++) { - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, i); - if (matc == minmax) + if (j == len) { + // Found the minmax method + minmax = matc; break; - if (matc->fully_covers == FULLY_COVERS) { - jl_method_t *m = matc->method; - jl_method_t *minmaxm = minmax->method; - if (!jl_method_morespecific(minmaxm, m)) { - minmax_ambig = 1; - minmax = NULL; - has_ambiguity = 1; - break; - } } } } @@ -4343,24 +4874,31 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, // cost much extra and is less likely to help us hit a fast path // (we will look for this later, when we compute ambig_groupid, for // correctness) - if (!all_subtypes && minmax != NULL) { - jl_method_t *minmaxm = minmax->method; - all_subtypes = 1; + int all_subtypes = any_subtypes; + if (any_subtypes) { + jl_method_t *minmaxm = NULL; + if (minmax != NULL) + minmaxm = minmax->method; for (i = 0; i < len; i++) { jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, i); if (matc->fully_covers != FULLY_COVERS) { jl_method_t *m = matc->method; - if (jl_method_morespecific(minmaxm, m)) - matc->fully_covers = SENTINEL; // put a sentinel value here for sorting - else - all_subtypes = 0; + if (minmaxm) { + if (method_morespecific_via_interferences(minmaxm, m)) { + matc->fully_covers = SENTINEL; // put a sentinel value here for sorting + continue; + } + if (method_in_interferences(minmaxm, m)) // !morespecific(m, minmaxm) + has_ambiguity = 1; + } + all_subtypes = 0; } } } // - now we might have a fast-return here, if we see that // we've already processed all of the possible outputs if (all_subtypes) { - if (minmax_ambig) { + if (minmax == NULL) { if (!include_ambiguous) { len = 0; env.t = jl_an_empty_vec_any; @@ -4371,7 +4909,6 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, } } else { - assert(minmax != NULL); jl_array_ptr_set(env.t, 0, minmax); jl_array_del_end((jl_array_t*)env.t, len - 1); len = 1; @@ -4384,20 +4921,23 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, } } if (len > 1) { - arraylist_t stack, visited, result, allambig; + arraylist_t stack, visited, result, recursion_stack; arraylist_new(&result, lim != -1 && lim < len ? lim : len); arraylist_new(&stack, 0); arraylist_new(&visited, len); - arraylist_new(&allambig, len); + arraylist_new(&recursion_stack, len); arraylist_grow(&visited, len); memset(visited.items, 0, len * sizeof(size_t)); // if we had a minmax method (any subtypes), now may now be able to // quickly cleanup some of methods int found_minmax = 0; - if (minmax != NULL) + if (has_ambiguity) + found_minmax = 1; + else if (minmax != NULL) found_minmax = 2; - else if (minmax_ambig && !include_ambiguous) + else if (any_subtypes && !include_ambiguous) found_minmax = 1; + has_ambiguity = 0; if (ambig == NULL) // if we don't care about the result, set it now so we won't bother attempting to compute it accurately later has_ambiguity = 1; for (i = 0; i < len; i++) { @@ -4408,9 +4948,9 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, // by visiting it and it might be a bit costly continue; } - int child_cycle = sort_mlmatches((jl_array_t*)env.t, i, &visited, &stack, &result, &allambig, lim == -1 || minmax == NULL ? lim : lim - 1, include_ambiguous, &has_ambiguity, &found_minmax); + int child_cycle = sort_mlmatches((jl_array_t*)env.t, i, &visited, &stack, &result, &recursion_stack, lim == -1 || minmax == NULL ? lim : lim - 1, include_ambiguous, &has_ambiguity, &found_minmax); if (child_cycle == -1) { - arraylist_free(&allambig); + arraylist_free(&recursion_stack); arraylist_free(&visited); arraylist_free(&stack); arraylist_free(&result); @@ -4421,89 +4961,7 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, assert(stack.len == 0); assert(visited.items[i] == (void*)1); } - // now compute whether there were ambiguities left in this cycle - if (has_ambiguity == 0 && allambig.len > 0) { - if (lim == -1) { - // lim is over-approximated, so has_ambiguities is too - has_ambiguity = 1; - } - else { - // go back and find the additional ambiguous methods and temporary add them to the stack - // (potentially duplicating them from lower on the stack to here) - jl_value_t *ti = NULL; - jl_value_t *isect2 = NULL; - JL_GC_PUSH2(&ti, &isect2); - for (size_t i = 0; i < allambig.len; i++) { - size_t idx = (size_t)allambig.items[i]; - jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, idx); - jl_method_t *m = matc->method; - int subt = matc->fully_covers == FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m->sig) - for (size_t idx2 = 0; idx2 < jl_array_nrows(env.t); idx2++) { - if (idx2 == idx) - continue; - // laborious test, checking for existence and coverage of another method (m3) - // outside of the ambiguity group that dominates any ambiguous methods, - // and means we can ignore this for has_ambiguity - // (has_ambiguity is overestimated for lim==-1, since we don't compute skipped matches either) - // n.b. even if we skipped them earlier, they still might - // contribute to the ambiguities (due to lock of transitivity of - // morespecific over subtyping) - // TODO: we could improve this result by checking if the removal of some - // edge earlier means that this subgraph is now well-ordered and then be - // allowed to ignore these vertexes entirely here - jl_method_match_t *matc2 = (jl_method_match_t*)jl_array_ptr_ref(env.t, idx2); - jl_method_t *m2 = matc2->method; - int subt2 = matc2->fully_covers == FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m2->sig) - if (subt) { - ti = (jl_value_t*)matc2->spec_types; - isect2 = NULL; - } - else if (subt2) { - ti = (jl_value_t*)matc->spec_types; - isect2 = NULL; - } - else { - jl_type_intersection2((jl_value_t*)matc->spec_types, (jl_value_t*)matc2->spec_types, &ti, &isect2); - } - // if their intersection contributes to the ambiguity cycle - if (ti == jl_bottom_type) - continue; - // and they aren't themselves simply ordered - if (jl_method_morespecific(m, m2) || jl_method_morespecific(m2, m)) - continue; - // now look for a third method m3 that dominated these and that fully covered this intersection already - size_t k; - for (k = 0; k < result.len; k++) { - size_t idx3 = (size_t)result.items[k]; - if (idx3 == idx || idx3 == idx2) { - has_ambiguity = 1; - break; - } - jl_method_match_t *matc3 = (jl_method_match_t*)jl_array_ptr_ref(env.t, idx3); - jl_method_t *m3 = matc3->method; - if ((jl_subtype(ti, m3->sig) || (isect2 && jl_subtype(isect2, m3->sig))) - && jl_method_morespecific(m3, m) && jl_method_morespecific(m3, m2)) { - //if (jl_subtype(matc->spec_types, ti) || jl_subtype(matc->spec_types, matc3->m3->sig)) - // // check if it covered not only this intersection, but all intersections with matc - // // if so, we do not need to check all of them separately - // j = len; - break; - } - } - if (k == result.len) - has_ambiguity = 1; - isect2 = NULL; - ti = NULL; - if (has_ambiguity) - break; - } - if (has_ambiguity) - break; - } - JL_GC_POP(); - } - } - arraylist_free(&allambig); + arraylist_free(&recursion_stack); arraylist_free(&visited); arraylist_free(&stack); for (j = 0; j < result.len; j++) { @@ -4529,21 +4987,17 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_method_t *m = matc->method; // method applicability is the same as typemapentry applicability size_t min_world = jl_atomic_load_relaxed(&m->primary_world); - size_t max_world = jl_atomic_load_relaxed(&m->deleted_world); // intersect the env valid range with method lookup's inclusive valid range if (env.match.min_valid < min_world) env.match.min_valid = min_world; - if (env.match.max_valid > max_world) - env.match.max_valid = max_world; } - if (mt && cache_result && ((jl_datatype_t*)unw)->isdispatchtuple) { // cache_result parameter keeps this from being recursive + if (mc && cache_result && ((jl_datatype_t*)unw)->isdispatchtuple) { // cache_result parameter keeps this from being recursive if (len == 1 && !has_ambiguity) { env.matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, 0); jl_method_t *meth = env.matc->method; jl_svec_t *tpenv = env.matc->sparams; - JL_LOCK(&mt->writelock); - cache_method(mt, &mt->cache, (jl_value_t*)mt, (jl_tupletype_t*)unw, meth, world, env.match.min_valid, env.match.max_valid, tpenv); - JL_UNLOCK(&mt->writelock); + JL_LOCK(&mc->writelock); + cache_method(mt, mc, &mc->cache, (jl_value_t*)mc, (jl_tupletype_t*)unw, meth, world, env.match.min_valid, env.match.max_valid, tpenv); } } *min_valid = env.match.min_valid; @@ -4609,8 +5063,6 @@ JL_DLLEXPORT void jl_extern_c(jl_value_t *name, jl_value_t *declrt, jl_tupletype jl_error("@ccallable: function object must be a singleton"); // compute / validate return type - if (!jl_is_concrete_type(declrt) || jl_is_kind(declrt)) - jl_error("@ccallable: return type must be concrete and correspond to a C type"); if (!jl_type_mappable_to_c(declrt)) jl_error("@ccallable: return type doesn't correspond to a C type"); @@ -4623,7 +5075,7 @@ JL_DLLEXPORT void jl_extern_c(jl_value_t *name, jl_value_t *declrt, jl_tupletype } // save a record of this so that the alias is generated when we write an object file - jl_method_t *meth = (jl_method_t*)jl_methtable_lookup(ft->name->mt, (jl_value_t*)sigt, jl_atomic_load_acquire(&jl_world_counter)); + jl_method_t *meth = (jl_method_t*)jl_methtable_lookup((jl_value_t*)sigt, jl_atomic_load_acquire(&jl_world_counter)); if (!jl_is_method(meth)) jl_error("@ccallable: could not find requested method"); JL_GC_PUSH1(&meth); diff --git a/src/init.c b/src/init.c index 6d5212cf8d370..a64f43c62f776 100644 --- a/src/init.c +++ b/src/init.c @@ -23,9 +23,7 @@ #include "julia.h" #include "julia_internal.h" -#define DEFINE_BUILTIN_GLOBALS #include "builtin_proto.h" -#undef DEFINE_BUILTIN_GLOBALS #include "threading.h" #include "julia_assert.h" #include "processor.h" @@ -251,7 +249,7 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) JL_NOTSAFEPOINT_ENTER if (jl_base_module) { size_t last_age = ct->world_age; ct->world_age = jl_get_world_counter(); - jl_value_t *f = jl_get_global(jl_base_module, jl_symbol("_atexit")); + jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("_atexit"), ct->world_age); if (f != NULL) { jl_value_t **fargs; JL_GC_PUSHARGS(fargs, 2); @@ -357,13 +355,14 @@ JL_DLLEXPORT void jl_postoutput_hook(void) if (jl_base_module) { jl_task_t *ct = jl_get_current_task(); - jl_value_t *f = jl_get_global(jl_base_module, jl_symbol("_postoutput")); + size_t last_age = ct->world_age; + ct->world_age = jl_get_world_counter(); + jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("_postoutput"), ct->world_age); if (f != NULL) { JL_TRY { - size_t last_age = ct->world_age; - ct->world_age = jl_get_world_counter(); + JL_GC_PUSH1(&f); jl_apply(&f, 1); - ct->world_age = last_age; + JL_GC_POP(); } JL_CATCH { jl_printf((JL_STREAM*)STDERR_FILENO, "\npostoutput hook threw an error: "); @@ -372,6 +371,7 @@ JL_DLLEXPORT void jl_postoutput_hook(void) jlbacktrace(); // written to STDERR_FILENO } } + ct->world_age = last_age; } return; } @@ -535,169 +535,6 @@ int jl_isabspath(const char *in) JL_NOTSAFEPOINT return 0; // relative path } -static char *absrealpath(const char *in, int nprefix) -{ // compute an absolute realpath location, so that chdir doesn't change the file reference - // ignores (copies directly over) nprefix characters at the start of abspath -#ifndef _OS_WINDOWS_ - char *out = realpath(in + nprefix, NULL); - if (out) { - if (nprefix > 0) { - size_t sz = strlen(out) + 1; - char *cpy = (char*)malloc_s(sz + nprefix); - memcpy(cpy, in, nprefix); - memcpy(cpy + nprefix, out, sz); - free(out); - out = cpy; - } - } - else { - size_t sz = strlen(in + nprefix) + 1; - if (in[nprefix] == PATHSEPSTRING[0]) { - out = (char*)malloc_s(sz + nprefix); - memcpy(out, in, sz + nprefix); - } - else { - size_t path_size = JL_PATH_MAX; - char *path = (char*)malloc_s(JL_PATH_MAX); - if (uv_cwd(path, &path_size)) { - jl_error("fatal error: unexpected error while retrieving current working directory"); - } - out = (char*)malloc_s(path_size + 1 + sz + nprefix); - memcpy(out, in, nprefix); - memcpy(out + nprefix, path, path_size); - out[nprefix + path_size] = PATHSEPSTRING[0]; - memcpy(out + nprefix + path_size + 1, in + nprefix, sz); - free(path); - } - } -#else - // GetFullPathName intentionally errors if given an empty string so manually insert `.` to invoke cwd - char *in2 = (char*)malloc_s(JL_PATH_MAX); - if (strlen(in) - nprefix == 0) { - memcpy(in2, in, nprefix); - in2[nprefix] = '.'; - in2[nprefix+1] = '\0'; - in = in2; - } - DWORD n = GetFullPathName(in + nprefix, 0, NULL, NULL); - if (n <= 0) { - jl_error("fatal error: jl_options.image_file path too long or GetFullPathName failed"); - } - char *out = (char*)malloc_s(n + nprefix); - DWORD m = GetFullPathName(in + nprefix, n, out + nprefix, NULL); - if (n != m + 1) { - jl_error("fatal error: jl_options.image_file path too long or GetFullPathName failed"); - } - memcpy(out, in, nprefix); - free(in2); -#endif - return out; -} - -// create an absolute-path copy of the input path format string -// formed as `joinpath(replace(pwd(), "%" => "%%"), in)` -// unless `in` starts with `%` -static const char *absformat(const char *in) -{ - if (in[0] == '%' || jl_isabspath(in)) - return in; - // get an escaped copy of cwd - size_t path_size = JL_PATH_MAX; - char path[JL_PATH_MAX]; - if (uv_cwd(path, &path_size)) { - jl_error("fatal error: unexpected error while retrieving current working directory"); - } - size_t sz = strlen(in) + 1; - size_t i, fmt_size = 0; - for (i = 0; i < path_size; i++) - fmt_size += (path[i] == '%' ? 2 : 1); - char *out = (char*)malloc_s(fmt_size + 1 + sz); - fmt_size = 0; - for (i = 0; i < path_size; i++) { // copy-replace pwd portion - char c = path[i]; - out[fmt_size++] = c; - if (c == '%') - out[fmt_size++] = '%'; - } - out[fmt_size++] = PATHSEPSTRING[0]; // path sep - memcpy(out + fmt_size, in, sz); // copy over format, including nul - return out; -} - -static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) -{ - // this function resolves the paths in jl_options to absolute file locations as needed - // and it replaces the pointers to `julia_bindir`, `julia_bin`, `image_file`, and output file paths - // it may fail, print an error, and exit(1) if any of these paths are longer than JL_PATH_MAX - // - // note: if you care about lost memory, you should call the appropriate `free()` function - // on the original pointer for each `char*` you've inserted into `jl_options`, after - // calling `julia_init()` - char *free_path = (char*)malloc_s(JL_PATH_MAX); - size_t path_size = JL_PATH_MAX; - if (uv_exepath(free_path, &path_size)) { - jl_error("fatal error: unexpected error while retrieving exepath"); - } - if (path_size >= JL_PATH_MAX) { - jl_error("fatal error: jl_options.julia_bin path too long"); - } - jl_options.julia_bin = (char*)malloc_s(path_size + 1); - memcpy((char*)jl_options.julia_bin, free_path, path_size); - ((char*)jl_options.julia_bin)[path_size] = '\0'; - if (!jl_options.julia_bindir) { - jl_options.julia_bindir = getenv("JULIA_BINDIR"); - if (!jl_options.julia_bindir) { - jl_options.julia_bindir = dirname(free_path); - } - } - if (jl_options.julia_bindir) - jl_options.julia_bindir = absrealpath(jl_options.julia_bindir, 0); - free(free_path); - free_path = NULL; - if (jl_options.image_file) { - if (rel == JL_IMAGE_JULIA_HOME && !jl_isabspath(jl_options.image_file)) { - // build time path, relative to JULIA_BINDIR - free_path = (char*)malloc_s(JL_PATH_MAX); - int n = snprintf(free_path, JL_PATH_MAX, "%s" PATHSEPSTRING "%s", - jl_options.julia_bindir, jl_options.image_file); - if (n >= JL_PATH_MAX || n < 0) { - jl_error("fatal error: jl_options.image_file path too long"); - } - jl_options.image_file = free_path; - } - if (jl_options.image_file) - jl_options.image_file = absrealpath(jl_options.image_file, 0); - if (free_path) { - free(free_path); - free_path = NULL; - } - } - if (jl_options.outputo) - jl_options.outputo = absrealpath(jl_options.outputo, 0); - if (jl_options.outputji) - jl_options.outputji = absrealpath(jl_options.outputji, 0); - if (jl_options.outputbc) - jl_options.outputbc = absrealpath(jl_options.outputbc, 0); - if (jl_options.outputasm) - jl_options.outputasm = absrealpath(jl_options.outputasm, 0); - if (jl_options.machine_file) - jl_options.machine_file = absrealpath(jl_options.machine_file, 0); - if (jl_options.output_code_coverage) - jl_options.output_code_coverage = absformat(jl_options.output_code_coverage); - if (jl_options.tracked_path) - jl_options.tracked_path = absrealpath(jl_options.tracked_path, 0); - - const char **cmdp = jl_options.cmds; - if (cmdp) { - for (; *cmdp; cmdp++) { - const char *cmd = *cmdp; - if (cmd[0] == 'L') { - *cmdp = absrealpath(cmd, 1); - } - } - } -} - JL_DLLEXPORT int jl_is_file_tracked(jl_sym_t *path) { const char* path_ = jl_symbol_name(path); @@ -722,9 +559,90 @@ static void restore_fp_env(void) if (jl_set_zero_subnormals(0) || jl_set_default_nans(0)) { jl_error("Failed to configure floating point environment"); } + if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_OFF && jl_atomic_load_relaxed(&jl_n_threads) > 1) { + jl_error("Cannot use `--handle-signals=no` with multiple threads (JULIA_NUM_THREADS > 1).\n" + "This will cause segmentation faults due to GC safepoint failures.\n" + "Remove `--handle-signals=no` or set JULIA_NUM_THREADS=1.\n" + "See: https://github.com/JuliaLang/julia/issues/50278"); + } +} +static NOINLINE void _finish_jl_init_(jl_image_buf_t sysimage, jl_ptls_t ptls, jl_task_t *ct) +{ + JL_TIMING(JULIA_INIT, JULIA_INIT); + + if (sysimage.kind == JL_IMAGE_KIND_SO) + jl_gc_notify_image_load(sysimage.data, sysimage.size); + + if (jl_options.cpu_target == NULL) + jl_options.cpu_target = "native"; + + // Parse image, perform relocations, and init JIT targets, etc. + jl_image_t parsed_image = jl_init_processor_sysimg(sysimage, jl_options.cpu_target); + + jl_init_codegen(); + jl_init_common_symbols(); + + if (sysimage.kind != JL_IMAGE_KIND_NONE) { + // Load the .ji or .so sysimage + jl_restore_system_image(&parsed_image, sysimage); + } else { + // No sysimage provided, init a minimal environment + jl_init_types(); + jl_global_roots_list = (jl_genericmemory_t*)jl_an_empty_memory_any; + jl_global_roots_keyset = (jl_genericmemory_t*)jl_an_empty_memory_any; + } + + jl_init_flisp(); + jl_init_serializer(); + + if (sysimage.kind == JL_IMAGE_KIND_NONE) { + jl_top_module = jl_core_module; + jl_init_intrinsic_functions(); + jl_init_primitives(); + jl_init_main_module(); + jl_load(jl_core_module, "boot.jl"); + post_boot_hooks(); + } + + if (jl_base_module == NULL) { + // nthreads > 1 requires code in Base + jl_atomic_store_relaxed(&jl_n_threads, 1); + jl_n_markthreads = 0; + jl_n_sweepthreads = 0; + jl_n_gcthreads = 0; + jl_n_threads_per_pool[JL_THREADPOOL_ID_INTERACTIVE] = 0; + jl_n_threads_per_pool[JL_THREADPOOL_ID_DEFAULT] = 1; + } else { + post_image_load_hooks(); + } + jl_start_threads(); + jl_start_gc_threads(); + uv_barrier_wait(&thread_init_done); + + jl_gc_enable(1); + + if ((sysimage.kind != JL_IMAGE_KIND_NONE) && + (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) { + jl_array_t *init_order = jl_module_init_order; + JL_GC_PUSH1(&init_order); + jl_module_init_order = NULL; + int i, l = jl_array_nrows(init_order); + for (i = 0; i < l; i++) { + jl_value_t *mod = jl_array_ptr_ref(init_order, i); + jl_module_run_initializer((jl_module_t*)mod); + } + JL_GC_POP(); + } + + if (jl_options.trim) { + jl_entrypoint_mis = (arraylist_t *)malloc_s(sizeof(arraylist_t)); + arraylist_new(jl_entrypoint_mis, 0); + } + + if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON) + jl_install_sigint_handler(); } -static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_task_t *ct); JL_DLLEXPORT int jl_default_debug_info_kind; JL_DLLEXPORT jl_cgparams_t jl_default_cgparams = { @@ -752,12 +670,12 @@ static void init_global_mutexes(void) { JL_MUTEX_INIT(&profile_show_peek_cond_lock, "profile_show_peek_cond_lock"); } -JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) +JL_DLLEXPORT void jl_init_(jl_image_buf_t sysimage) { // initialize many things, in no particular order // but generally running from simple platform things to optional // configuration features - jl_init_timing(); + // Make sure we finalize the tls callback before starting any threads. (void)jl_get_pgcstack(); @@ -862,96 +780,7 @@ JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel) jl_task_t *ct = jl_init_root_task(ptls, stack_lo, stack_hi); #pragma GCC diagnostic pop JL_GC_PROMISE_ROOTED(ct); - _finish_julia_init(rel, ptls, ct); -} - -static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_task_t *ct) -{ - JL_TIMING(JULIA_INIT, JULIA_INIT); - jl_resolve_sysimg_location(rel); - - // loads sysimg if available, and conditionally sets jl_options.cpu_target - jl_image_buf_t sysimage = { JL_IMAGE_KIND_NONE }; - if (rel == JL_IMAGE_IN_MEMORY) { - sysimage = jl_set_sysimg_so(jl_exe_handle); - jl_options.image_file = jl_options.julia_bin; - } - else if (jl_options.image_file) - sysimage = jl_preload_sysimg(jl_options.image_file); - - if (sysimage.kind == JL_IMAGE_KIND_SO) - jl_gc_notify_image_load(sysimage.data, sysimage.size); - - if (jl_options.cpu_target == NULL) - jl_options.cpu_target = "native"; - - // Parse image, perform relocations, and init JIT targets, etc. - jl_image_t parsed_image = jl_init_processor_sysimg(sysimage, jl_options.cpu_target); - - jl_init_codegen(); - jl_init_common_symbols(); - - if (sysimage.kind != JL_IMAGE_KIND_NONE) { - // Load the .ji or .so sysimage - jl_restore_system_image(&parsed_image, sysimage); - } else { - // No sysimage provided, init a minimal environment - jl_init_types(); - jl_global_roots_list = (jl_genericmemory_t*)jl_an_empty_memory_any; - jl_global_roots_keyset = (jl_genericmemory_t*)jl_an_empty_memory_any; - } - - jl_init_flisp(); - jl_init_serializer(); - - if (sysimage.kind == JL_IMAGE_KIND_NONE) { - jl_top_module = jl_core_module; - jl_init_intrinsic_functions(); - jl_init_primitives(); - jl_init_main_module(); - jl_load(jl_core_module, "boot.jl"); - jl_current_task->world_age = jl_atomic_load_acquire(&jl_world_counter); - post_boot_hooks(); - } - - if (jl_base_module == NULL) { - // nthreads > 1 requires code in Base - jl_atomic_store_relaxed(&jl_n_threads, 1); - jl_n_markthreads = 0; - jl_n_sweepthreads = 0; - jl_n_gcthreads = 0; - jl_n_threads_per_pool[JL_THREADPOOL_ID_INTERACTIVE] = 0; - jl_n_threads_per_pool[JL_THREADPOOL_ID_DEFAULT] = 1; - } else { - jl_current_task->world_age = jl_atomic_load_acquire(&jl_world_counter); - post_image_load_hooks(); - } - jl_start_threads(); - jl_start_gc_threads(); - uv_barrier_wait(&thread_init_done); - - jl_gc_enable(1); - - if ((sysimage.kind != JL_IMAGE_KIND_NONE) && - (!jl_generating_output() || jl_options.incremental) && jl_module_init_order) { - jl_array_t *init_order = jl_module_init_order; - JL_GC_PUSH1(&init_order); - jl_module_init_order = NULL; - int i, l = jl_array_nrows(init_order); - for (i = 0; i < l; i++) { - jl_value_t *mod = jl_array_ptr_ref(init_order, i); - jl_module_run_initializer((jl_module_t*)mod); - } - JL_GC_POP(); - } - - if (jl_options.trim) { - jl_entrypoint_mis = (arraylist_t *)malloc_s(sizeof(arraylist_t)); - arraylist_new(jl_entrypoint_mis, 0); - } - - if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON) - jl_install_sigint_handler(); + _finish_jl_init_(sysimage, ptls, ct); } #ifdef __cplusplus diff --git a/src/interpreter.c b/src/interpreter.c index a85f908ed8530..c7b2da6d388cb 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -112,14 +112,13 @@ static jl_value_t *eval_methoddef(jl_expr_t *ex, interpreter_state *s) fname = eval_value(args[0], s); jl_methtable_t *mt = NULL; - if (jl_typetagis(fname, jl_methtable_type)) { + if (jl_is_mtable(fname)) mt = (jl_methtable_t*)fname; - } atypes = eval_value(args[1], s); meth = eval_value(args[2], s); - jl_method_def((jl_svec_t*)atypes, mt, (jl_code_info_t*)meth, s->module); + jl_method_t *ret = jl_method_def((jl_svec_t*)atypes, mt, (jl_code_info_t*)meth, s->module); JL_GC_POP(); - return jl_nothing; + return (jl_value_t *)ret; } // expression evaluator @@ -173,17 +172,19 @@ static jl_value_t *do_invoke(jl_value_t **args, size_t nargs, interpreter_state return result; } -jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e) +// get the global (throwing if null) in the current world +jl_value_t *jl_eval_global_var(jl_module_t *m, jl_sym_t *e, size_t world) { - jl_value_t *v = jl_get_global(m, e); + jl_value_t *v = jl_get_global_value(m, e, world); if (v == NULL) jl_undefined_var_error(e, (jl_value_t*)m); return v; } -jl_value_t *jl_eval_globalref(jl_globalref_t *g) +// get the global (throwing if null) in the current world, optimized +jl_value_t *jl_eval_globalref(jl_globalref_t *g, size_t world) { - jl_value_t *v = jl_get_globalref_value(g); + jl_value_t *v = jl_get_globalref_value(g, world); if (v == NULL) jl_undefined_var_error(g->name, (jl_value_t*)g->mod); return v; @@ -228,10 +229,10 @@ static jl_value_t *eval_value(jl_value_t *e, interpreter_state *s) return jl_quotenode_value(e); } if (jl_is_globalref(e)) { - return jl_eval_globalref((jl_globalref_t*)e); + return jl_eval_globalref((jl_globalref_t*)e, jl_current_task->world_age); } if (jl_is_symbol(e)) { // bare symbols appear in toplevel exprs not wrapped in `thunk` - return jl_eval_global_var(s->module, (jl_sym_t*)e); + return jl_eval_global_var(s->module, (jl_sym_t*)e, jl_current_task->world_age); } if (jl_is_pinode(e)) { jl_value_t *val = eval_value(jl_fieldref_noalloc(e, 0), s); @@ -637,7 +638,8 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, } else if (toplevel) { if (head == jl_method_sym && jl_expr_nargs(stmt) > 1) { - eval_methoddef((jl_expr_t*)stmt, s); + jl_value_t *res = eval_methoddef((jl_expr_t*)stmt, s); + s->locals[jl_source_nslots(s->src) + s->ip] = res; } else if (head == jl_toplevel_sym) { jl_value_t *res = jl_toplevel_eval(s->module, stmt); @@ -711,7 +713,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, s->locals[n - 1] = NULL; } else if (toplevel && jl_is_linenode(stmt)) { - jl_lineno = jl_linenode_line(stmt); + jl_atomic_store_relaxed(&jl_lineno, jl_linenode_line(stmt)); } else { eval_stmt_value(stmt, s); diff --git a/src/ircode.c b/src/ircode.c index 58c0601f3bbee..f19f2f5a990bd 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -989,7 +989,7 @@ static int codelocs_nstmts(jl_string_t *cl) JL_NOTSAFEPOINT #define IR_DATASIZE_FLAGS sizeof(uint16_t) #define IR_DATASIZE_PURITY sizeof(uint16_t) -#define IR_DATASIZE_INLINING_COST sizeof(uint16_t) +#define IR_DATASIZE_INLINING_COST sizeof(uint8_t) #define IR_DATASIZE_NSLOTS sizeof(int32_t) typedef enum { ir_offset_flags = 0, @@ -1044,7 +1044,7 @@ JL_DLLEXPORT jl_string_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) code->ssaflags); write_uint16(s.s, checked_size(flags.packed, IR_DATASIZE_FLAGS)); write_uint16(s.s, checked_size(code->purity.bits, IR_DATASIZE_PURITY)); - write_uint16(s.s, checked_size(code->inlining_cost, IR_DATASIZE_INLINING_COST)); + write_uint8(s.s, checked_size(jl_encode_inlining_cost(code->inlining_cost), IR_DATASIZE_INLINING_COST)); size_t nslots = jl_array_nrows(code->slotflags); assert(nslots >= m->nargs && nslots < INT32_MAX); // required by generated functions @@ -1109,6 +1109,8 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t { if (jl_is_code_info(data)) return (jl_code_info_t*)data; + if (!jl_is_string(data)) + return (jl_code_info_t*)jl_nothing; JL_TIMING(AST_UNCOMPRESS, AST_UNCOMPRESS); JL_LOCK(&m->writelock); // protect the roots array (Might GC) assert(jl_is_method(m)); @@ -1139,7 +1141,7 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t code->nospecializeinfer = flags.bits.nospecializeinfer; code->isva = flags.bits.isva; code->purity.bits = read_uint16(s.s); - code->inlining_cost = read_uint16(s.s); + code->inlining_cost = jl_decode_inlining_cost(read_uint8(s.s)); size_t nslots = read_int32(s.s); code->slotflags = jl_alloc_array_1d(jl_array_uint8_type, nslots); @@ -1240,12 +1242,46 @@ JL_DLLEXPORT uint8_t jl_ir_flag_has_image_globalref(jl_string_t *data) return flags.bits.has_image_globalref; } -JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_string_t *data) +// create a compressed u16 value with range 0..3968, 3 bits exponent, 5 bits mantissa, implicit first digit, rounding up, full accuracy over 0..63 +JL_DLLEXPORT uint8_t jl_encode_inlining_cost(uint16_t inlining_cost) { + unsigned shift = 0; + unsigned mantissa; + if (inlining_cost <= 0x1f) { + mantissa = inlining_cost; + } + else { + while (inlining_cost >> 5 >> shift != 0) + shift++; + assert(1 <= shift && shift <= 11); + mantissa = (inlining_cost >> (shift - 1)) & 0x1f; + mantissa += (inlining_cost & ((1 << (shift - 1)) - 1)) != 0; // round up if trailing bits non-zero, overflowing into exp + } + unsigned r = (shift << 5) + mantissa; + if (r > 0xff) + r = 0xff; + return r; +} + +JL_DLLEXPORT uint16_t jl_decode_inlining_cost(uint8_t inlining_cost) +{ + unsigned shift = inlining_cost >> 5; + if (inlining_cost == 0xff) + return 0xffff; + else if (shift == 0) + return inlining_cost; + else + return (0x20 | (inlining_cost & 0x1f)) << (shift - 1); +} + +JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_value_t *data) +{ + if (jl_is_uint8(data)) + return jl_decode_inlining_cost(*(uint8_t*)data); if (jl_is_code_info(data)) return ((jl_code_info_t*)data)->inlining_cost; assert(jl_is_string(data)); - uint16_t res = jl_load_unaligned_i16(jl_string_data(data) + ir_offset_inlining_cost); + uint16_t res = jl_decode_inlining_cost(*(uint8_t*)(jl_string_data(data) + ir_offset_inlining_cost)); return res; } @@ -1616,14 +1652,12 @@ void jl_init_serializer(void) jl_densearray_type, jl_function_type, jl_typename_type, jl_builtin_type, jl_task_type, jl_uniontype_type, jl_array_any_type, jl_intrinsic_type, - jl_methtable_type, jl_typemap_level_type, jl_voidpointer_type, jl_newvarnode_type, jl_abstractstring_type, jl_array_symbol_type, jl_anytuple_type, jl_tparam0(jl_anytuple_type), jl_emptytuple_type, jl_array_uint8_type, jl_array_uint32_type, jl_code_info_type, jl_typeofbottom_type, jl_typeofbottom_type->super, jl_namedtuple_type, jl_array_int32_type, jl_uint32_type, jl_uint64_type, - jl_type_type_mt, jl_nonfunction_mt, jl_opaque_closure_type, jl_memory_any_type, jl_memory_uint8_type, diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index a66e4c8b121fd..da26d210ea6f2 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -6,6 +6,7 @@ #include #include "llvm/IR/Mangler.h" +#include #include #include #include @@ -15,7 +16,8 @@ #include #include #if JL_LLVM_VERSION >= 200000 -#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h" +#include +#include #endif #if JL_LLVM_VERSION >= 180000 #include @@ -49,6 +51,7 @@ using namespace llvm; #include "jitlayers.h" #include "julia_assert.h" #include "processor.h" +#include "llvm-julia-task-dispatcher.h" #if JL_LLVM_VERSION >= 180000 # include @@ -71,7 +74,6 @@ STATISTIC(OptO0, "Number of modules optimized at level -O0"); STATISTIC(OptO1, "Number of modules optimized at level -O1"); STATISTIC(OptO2, "Number of modules optimized at level -O2"); STATISTIC(OptO3, "Number of modules optimized at level -O3"); -STATISTIC(ModulesMerged, "Number of modules merged"); STATISTIC(InternedGlobals, "Number of global constants interned in the string pool"); #ifdef _COMPILER_MSAN_ENABLED_ @@ -255,12 +257,46 @@ static void finish_params(Module *M, jl_codegen_params_t ¶ms, SmallVectorrettype, jl_pinned_ref_get(from_abi.rt))) + return specptr; // no adapter required + + target = specptr; + target_specsig = false; + } + else if (specsigflags & 0b1) { + assert(specptr != nullptr); + if (from_abi.specsig && jl_egal(mi->specTypes, jl_pinned_ref_get(from_abi.sigt)) && jl_egal(codeinst->rettype, jl_pinned_ref_get(from_abi.rt))) + return specptr; // no adapter required + + target = specptr; + target_specsig = true; + } + } + } + orc::ThreadSafeModule result_m; std::string gf_thunk_name; { @@ -272,14 +308,14 @@ void *jl_jit_abi_converter_impl(jl_task_t *ct, void *unspecialized, jl_value_t * Module *M = result_m.getModuleUnlocked(); if (target) { Value *llvmtarget = literal_static_pointer_val((void*)target, PointerType::get(M->getContext(), 0)); - gf_thunk_name = emit_abi_converter(M, params, declrt, sigt, nargs, specsig, codeinst, llvmtarget, target_specsig); + gf_thunk_name = emit_abi_converter(M, params, from_abi, codeinst, llvmtarget, target_specsig); } else if (invoke == jl_fptr_const_return_addr) { - gf_thunk_name = emit_abi_constreturn(M, params, declrt, sigt, nargs, specsig, codeinst->rettype_const); + gf_thunk_name = emit_abi_constreturn(M, params, from_abi, codeinst->rettype_const); } else { Value *llvminvoke = invoke ? literal_static_pointer_val((void*)invoke, PointerType::get(M->getContext(), 0)) : nullptr; - gf_thunk_name = emit_abi_dispatcher(M, params, declrt, sigt, nargs, specsig, codeinst, llvminvoke); + gf_thunk_name = emit_abi_dispatcher(M, params, from_abi, codeinst, llvminvoke); } SmallVector sharedmodules; finish_params(M, params, sharedmodules); @@ -296,8 +332,6 @@ void *jl_jit_abi_converter_impl(jl_task_t *ct, void *unspecialized, jl_value_t * // lock for places where only single threaded behavior is implemented, so we need GC support static jl_mutex_t jitlock; - // locks for adding external code to the JIT atomically -static std::mutex extern_c_lock; // locks and barriers for this state static std::mutex engine_lock; static std::condition_variable engine_wait; @@ -325,7 +359,6 @@ static DenseMap> incompl // jitlock is outermost, can contain others and allows GC // engine_lock is next // ThreadSafeContext locks are next, they should not be nested (unless engine_lock is also held, but this may make TSAN sad anyways) -// extern_c_lock is next // jl_ExecutionEngine internal locks are exclusive to this list, since OrcJIT promises to never hold a lock over a materialization unit: // construct a query object from a query set and query handler // lock the session @@ -339,177 +372,196 @@ static DenseMap> incompl static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t ¶ms, bool forceall=false) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER { jl_task_t *ct = jl_current_task; - decltype(params.workqueue) edges; + jl_workqueue_t edges; std::swap(params.workqueue, edges); for (auto &it : edges) { jl_code_instance_t *codeinst = it.first; JL_GC_PROMISE_ROOTED(codeinst); auto &proto = it.second; - // try to emit code for this item from the workqueue - StringRef invokeName = ""; - StringRef preal_decl = ""; - bool preal_specsig = false; - jl_callptr_t invoke = nullptr; - bool isedge = false; - assert(params.cache); - // Checking the cache here is merely an optimization and not strictly required - // But it must be consistent with the following invokenames lookup, which is protected by the engine_lock - uint8_t specsigflags; - void *fptr; - void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile) JL_NOTSAFEPOINT; // declare it is not a safepoint (or deadlock) in this file due to 0 parameter - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); - //if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) - if (invoke == jl_fptr_args_addr) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); - } - else if (specsigflags & 0b1) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); - preal_specsig = true; - } - bool force = forceall || invoke != nullptr; - if (preal_decl.empty()) { - auto it = invokenames.find(codeinst); - if (it != invokenames.end()) { - auto &decls = it->second; - invokeName = decls.functionObject; - if (decls.functionObject == "jl_fptr_args") { - preal_decl = decls.specFunctionObject; - isedge = true; - } - else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") { - preal_decl = decls.specFunctionObject; - preal_specsig = true; - isedge = true; - } - force = true; + if (proto.external_linkage || proto.decl->isDeclaration()) { // if it is not expected externally and has a definition locally, there is no need to patch this edge up + // try to emit code for this item from the workqueue + StringRef invokeName = ""; + StringRef preal_decl = ""; + bool preal_specsig = false; + jl_callptr_t invoke = nullptr; + bool isedge = false; + assert(params.cache); + // Checking the cache here is merely an optimization and not strictly required + // But it must be consistent with the following invokenames lookup, which is protected by the engine_lock + uint8_t specsigflags; + void *fptr; + void jl_read_codeinst_invoke(jl_code_instance_t *ci, uint8_t *specsigflags, jl_callptr_t *invoke, void **specptr, int waitcompile) JL_NOTSAFEPOINT; // declare it is not a safepoint (or deadlock) in this file due to 0 parameter + jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); + //if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) + if (invoke == jl_fptr_args_addr) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); } - } - if (preal_decl.empty()) { - // there may be an equivalent method already compiled (or at least registered with the JIT to compile), in which case we should be using that instead - jl_code_instance_t *compiled_ci = jl_get_ci_equiv(codeinst, 1); - if ((jl_value_t*)compiled_ci != jl_nothing) { - codeinst = compiled_ci; - uint8_t specsigflags; - void *fptr; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); - //if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) - if (invoke == jl_fptr_args_addr) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); - } - else if (specsigflags & 0b1) { - preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); - preal_specsig = true; - } - if (preal_decl.empty()) { - auto it = invokenames.find(codeinst); - if (it != invokenames.end()) { - auto &decls = it->second; - invokeName = decls.functionObject; - if (decls.functionObject == "jl_fptr_args") { - preal_decl = decls.specFunctionObject; - isedge = true; - } - else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") { - preal_decl = decls.specFunctionObject; - preal_specsig = true; - isedge = true; - } - } - } + else if (specsigflags & 0b1) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); + preal_specsig = true; } - } - if (!preal_decl.empty() || force) { - // if we have a prototype emitted, compare it to what we emitted earlier - Module *mod = proto.decl->getParent(); - assert(proto.decl->isDeclaration()); - Function *pinvoke = nullptr; + bool force = forceall || invoke != nullptr; if (preal_decl.empty()) { - if (invoke != nullptr && invokeName.empty()) { - assert(invoke != jl_fptr_args_addr); - if (invoke == jl_fptr_sparam_addr) - invokeName = "jl_fptr_sparam"; - else if (invoke == jl_f_opaque_closure_call_addr) - invokeName = "jl_f_opaque_closure_call"; - else - invokeName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst); + auto it = invokenames.find(codeinst); + if (it != invokenames.end()) { + auto &decls = it->second; + invokeName = decls.functionObject; + if (decls.functionObject == "jl_fptr_args") { + preal_decl = decls.specFunctionObject; + isedge = true; + } + else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") { + preal_decl = decls.specFunctionObject; + preal_specsig = true; + isedge = true; + } + force = true; } - pinvoke = emit_tojlinvoke(codeinst, invokeName, mod, params); - if (!proto.specsig) - proto.decl->replaceAllUsesWith(pinvoke); - isedge = false; } - if (proto.specsig && !preal_specsig) { - // get or build an fptr1 that can invoke codeinst - if (pinvoke == nullptr) - pinvoke = get_or_emit_fptr1(preal_decl, mod); - // emit specsig-to-(jl)invoke conversion - proto.decl->setLinkage(GlobalVariable::InternalLinkage); - //protodecl->setAlwaysInline(); - jl_init_function(proto.decl, params.TargetTriple); - // TODO: maybe this can be cached in codeinst->specfptr? - int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) - jl_method_instance_t *mi = jl_get_ci_mi(codeinst); - size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed - bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; - emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke); - jl_gc_unsafe_leave(ct->ptls, gc_state); - preal_decl = ""; // no need to fixup the name - } - if (!preal_decl.empty()) { - // merge and/or rename this prototype to the real function - if (Value *specfun = mod->getNamedValue(preal_decl)) { - if (proto.decl != specfun) - proto.decl->replaceAllUsesWith(specfun); - } - else { - proto.decl->setName(preal_decl); + if (preal_decl.empty()) { + // there may be an equivalent method already compiled (or at least registered with the JIT to compile), in which case we should be using that instead + jl_code_instance_t *compiled_ci = jl_get_ci_equiv(codeinst, 0); + if (compiled_ci != codeinst) { + codeinst = compiled_ci; + uint8_t specsigflags; + void *fptr; + jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &fptr, 0); + //if (specsig ? specsigflags & 0b1 : invoke == jl_fptr_args_addr) + if (invoke == jl_fptr_args_addr) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); + } + else if (specsigflags & 0b1) { + preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, invoke, codeinst); + preal_specsig = true; + } + if (preal_decl.empty()) { + auto it = invokenames.find(codeinst); + if (it != invokenames.end()) { + auto &decls = it->second; + invokeName = decls.functionObject; + if (decls.functionObject == "jl_fptr_args") { + preal_decl = decls.specFunctionObject; + isedge = true; + } + else if (decls.functionObject != "jl_fptr_sparam" && decls.functionObject != "jl_f_opaque_closure_call") { + preal_decl = decls.specFunctionObject; + preal_specsig = true; + isedge = true; + } + } + } } } - if (proto.oc) { // additionally, if we are dealing with an OC constructor, then we might also need to fix up the fptr1 reference too - assert(proto.specsig); - StringRef ocinvokeDecl = invokeName; - if (invoke != nullptr && ocinvokeDecl.empty()) { - // check for some special tokens used by opaque_closure.c and convert those to their real functions - assert(invoke != jl_fptr_args_addr); - assert(invoke != jl_fptr_sparam_addr); - if (invoke == jl_fptr_interpret_call_addr) - ocinvokeDecl = "jl_fptr_interpret_call"; - else if (invoke == jl_fptr_const_return_addr) - ocinvokeDecl = "jl_fptr_const_return"; - else if (invoke == jl_f_opaque_closure_call_addr) - ocinvokeDecl = "jl_f_opaque_closure_call"; - //else if (invoke == jl_interpret_opaque_closure_addr) - else - ocinvokeDecl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst); + if (!preal_decl.empty() || force) { + // if we have a prototype emitted, compare it to what we emitted earlier + Module *mod = proto.decl->getParent(); + Function *pinvoke = nullptr; + if (proto.decl->isDeclaration()) { + if (preal_decl.empty()) { + if (invoke != nullptr && invokeName.empty()) { + assert(invoke != jl_fptr_args_addr); + if (invoke == jl_fptr_sparam_addr) + invokeName = "jl_fptr_sparam"; + else if (invoke == jl_f_opaque_closure_call_addr) + invokeName = "jl_f_opaque_closure_call"; + else + invokeName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst); + } + pinvoke = emit_tojlinvoke(codeinst, invokeName, mod, params); + if (!proto.specsig) { + proto.decl->replaceAllUsesWith(pinvoke); + proto.decl->eraseFromParent(); + proto.decl = pinvoke; + } + isedge = false; + } + if (proto.specsig && !preal_specsig) { + // get or build an fptr1 that can invoke codeinst + if (pinvoke == nullptr) + pinvoke = get_or_emit_fptr1(preal_decl, mod); + // emit specsig-to-(jl)invoke conversion + proto.decl->setLinkage(GlobalVariable::InternalLinkage); + //protodecl->setAlwaysInline(); + jl_init_function(proto.decl, params.TargetTriple); + // TODO: maybe this can be cached in codeinst->specfptr? + int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed + bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; + emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke); + jl_gc_unsafe_leave(ct->ptls, gc_state); + preal_decl = ""; // no need to fixup the name + } } - // if OC expected a specialized specsig dispatch, but we don't have it, use the inner trampoline here too - // XXX: this invoke translation logic is supposed to exactly match new_opaque_closure - if (!preal_specsig || ocinvokeDecl == "jl_f_opaque_closure_call" || ocinvokeDecl == "jl_fptr_interpret_call" || ocinvokeDecl == "jl_fptr_const_return") { - if (pinvoke == nullptr) - ocinvokeDecl = get_or_emit_fptr1(preal_decl, mod)->getName(); - else - ocinvokeDecl = pinvoke->getName(); + else if (proto.specsig && !preal_specsig) { + // privatize our definition, since for some reason we couldn't use the external one but have an internal one + proto.decl->setLinkage(GlobalValue::PrivateLinkage); + preal_decl = ""; // no need to fixup the name } - assert(!ocinvokeDecl.empty()); - assert(ocinvokeDecl != "jl_fptr_args"); - assert(ocinvokeDecl != "jl_fptr_sparam"); - // merge and/or rename this prototype to the real function - if (Value *specfun = mod->getNamedValue(ocinvokeDecl)) { - if (proto.oc != specfun) - proto.oc->replaceAllUsesWith(specfun); + if (!preal_decl.empty()) { + // merge and/or rename this prototype to the real function + if (Function *specfun = cast_or_null(mod->getNamedValue(preal_decl))) { + if (proto.decl != specfun) { + proto.decl->replaceAllUsesWith(specfun); + if (!proto.decl->isDeclaration() && specfun->isDeclaration()) + linkFunctionBody(*specfun, *proto.decl); + proto.decl->eraseFromParent(); + proto.decl = specfun; + } + } + else { + proto.decl->setName(preal_decl); + } } - else { - proto.oc->setName(ocinvokeDecl); + if (proto.oc) { // additionally, if we are dealing with an OC constructor, then we might also need to fix up the fptr1 reference too + assert(proto.specsig); + StringRef ocinvokeDecl = invokeName; + if (invoke != nullptr && ocinvokeDecl.empty()) { + // check for some special tokens used by opaque_closure.c and convert those to their real functions + assert(invoke != jl_fptr_args_addr); + assert(invoke != jl_fptr_sparam_addr); + if (invoke == jl_fptr_interpret_call_addr) + ocinvokeDecl = "jl_fptr_interpret_call"; + else if (invoke == jl_fptr_const_return_addr) + ocinvokeDecl = "jl_fptr_const_return"; + else if (invoke == jl_f_opaque_closure_call_addr) + ocinvokeDecl = "jl_f_opaque_closure_call"; + //else if (invoke == jl_interpret_opaque_closure_addr) + else + ocinvokeDecl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, invoke, codeinst); + } + // if OC expected a specialized specsig dispatch, but we don't have it, use the inner trampoline here too + // XXX: this invoke translation logic is supposed to exactly match new_opaque_closure + if (!preal_specsig || ocinvokeDecl == "jl_f_opaque_closure_call" || ocinvokeDecl == "jl_fptr_interpret_call" || ocinvokeDecl == "jl_fptr_const_return") { + if (pinvoke == nullptr) + ocinvokeDecl = get_or_emit_fptr1(preal_decl, mod)->getName(); + else + ocinvokeDecl = pinvoke->getName(); + } + assert(!ocinvokeDecl.empty()); + assert(ocinvokeDecl != "jl_fptr_args"); + assert(ocinvokeDecl != "jl_fptr_sparam"); + // merge and/or rename this prototype to the real function + if (Function *specfun = cast_or_null(mod->getNamedValue(ocinvokeDecl))) { + if (proto.oc != specfun) { + proto.oc->replaceAllUsesWith(specfun); + proto.oc->eraseFromParent(); + proto.oc = specfun; + } + } + else { + proto.oc->setName(ocinvokeDecl); + } } } + else { + isedge = true; + params.workqueue.push_back(it); + incomplete_rgraph[codeinst].push_back(callee); + } + if (isedge) + complete_graph[callee].push_back(codeinst); } - else { - isedge = true; - params.workqueue.push_back(it); - incomplete_rgraph[codeinst].push_back(callee); - } - if (isedge) - complete_graph[callee].push_back(codeinst); } return params.workqueue.size(); } @@ -580,10 +632,11 @@ static void complete_emit(jl_code_instance_t *edge) JL_NOTSAFEPOINT_LEAVE JL_NOT auto ¶ms = std::get<0>(it->second); params.tsctx_lock = params.tsctx.getLock(); assert(callee == it->first); + orc::ThreadSafeModule &M = emittedmodules[callee]; + emit_always_inline(M, params); // may safepoint int waiting = jl_analyze_workqueue(callee, params); // may safepoint assert(!waiting); (void)waiting; - Module *M = emittedmodules[callee].getModuleUnlocked(); - finish_params(M, params, sharedmodules); + finish_params(M.getModuleUnlocked(), params, sharedmodules); incompletemodules.erase(it); } } @@ -768,6 +821,7 @@ void jl_emit_codeinst_to_jit_impl( } jl_optimize_roots(params, jl_get_ci_mi(codeinst), *result_m.getModuleUnlocked()); // contains safepoints params.temporary_roots = nullptr; + params.temporary_roots_set.clear(); JL_GC_POP(); { // drop lock before acquiring engine_lock auto release = std::move(params.tsctx_lock); @@ -796,6 +850,7 @@ void jl_emit_codeinst_to_jit_impl( invokenames[codeinst] = std::move(decls); complete_emit(codeinst); params.tsctx_lock = params.tsctx.getLock(); // re-acquire lock + emit_always_inline(result_m, params); int waiting = jl_analyze_workqueue(codeinst, params); if (waiting) { auto release = std::move(params.tsctx_lock); // unlock again before moving from it @@ -1327,11 +1382,13 @@ namespace { #endif if (TheTriple.isAArch64()) codemodel = CodeModel::Small; +#if JL_LLVM_VERSION < 200000 else if (TheTriple.isRISCV()) { - // RISC-V will support large code model in LLVM 21 + // RISC-V only supports large code model from LLVM 20 // https://github.com/llvm/llvm-project/pull/70308 codemodel = CodeModel::Medium; } +#endif // Generate simpler code for JIT Reloc::Model relocmodel = Reloc::Static; if (TheTriple.isRISCV()) { @@ -1725,7 +1782,8 @@ struct JuliaOJIT::DLSymOptimizer { Thunk = cast(GV.getInitializer()->stripPointerCasts()); assert(++Thunk->uses().begin() == Thunk->uses().end() && "Thunk should only have one use in PLT initializer!"); assert(Thunk->hasLocalLinkage() && "Thunk should not have non-local linkage!"); - } else { + } + else { GV.setLinkage(GlobalValue::PrivateLinkage); } auto init = ConstantExpr::getIntToPtr(ConstantInt::get(M.getDataLayout().getIntPtrType(M.getContext()), (uintptr_t)addr), GV.getValueType()); @@ -1835,7 +1893,7 @@ llvm::DataLayout jl_create_datalayout(TargetMachine &TM) { JuliaOJIT::JuliaOJIT() : TM(createTargetMachine()), DL(jl_create_datalayout(*TM)), - ES(cantFail(orc::SelfExecutorProcessControl::Create())), + ES(cantFail(orc::SelfExecutorProcessControl::Create(nullptr, std::make_unique<::JuliaTaskDispatcher>()))), GlobalJD(ES.createBareJITDylib("JuliaGlobals")), JD(ES.createBareJITDylib("JuliaOJIT")), ExternalJD(ES.createBareJITDylib("JuliaExternal")), @@ -1985,17 +2043,14 @@ JuliaOJIT::JuliaOJIT() reinterpret_cast(static_cast(msan_workaround::MSanTLS::origin))), JITSymbolFlags::Exported}; cantFail(GlobalJD.define(orc::absoluteSymbols(msan_crt))); #endif -#if JL_LLVM_VERSION < 200000 #ifdef _COMPILER_ASAN_ENABLED_ // this is a hack to work around a bad assertion: // /workspace/srcdir/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp:3028: llvm::Error llvm::orc::ExecutionSession::OL_notifyResolved(llvm::orc::MaterializationResponsibility&, const SymbolMap&): Assertion `(KV.second.getFlags() & ~JITSymbolFlags::Common) == (I->second & ~JITSymbolFlags::Common) && "Resolving symbol with incorrect flags"' failed. - // hopefully fixed upstream by e7698a13e319a9919af04d3d693a6f6ea7168a44 static int64_t jl___asan_globals_registered; orc::SymbolMap asan_crt; asan_crt[mangle("___asan_globals_registered")] = {ExecutorAddr::fromPtr(&jl___asan_globals_registered), JITSymbolFlags::Common | JITSymbolFlags::Exported}; cantFail(JD.define(orc::absoluteSymbols(asan_crt))); #endif -#endif } JuliaOJIT::~JuliaOJIT() = default; @@ -2026,6 +2081,13 @@ void JuliaOJIT::addModule(orc::ThreadSafeModule TSM) TSM = (*JITPointers)(std::move(TSM)); auto Lock = TSM.getContext().getLock(); Module &M = *TSM.getModuleUnlocked(); + + for (auto &f : M) { + if (!f.isDeclaration()){ + jl_timing_puts(JL_TIMING_DEFAULT_BLOCK, f.getName().str().c_str()); + } + } + // Treat this as if one of the passes might contain a safepoint // even though that shouldn't be the case and might be unwise Expected> Obj = CompileLayer.getCompiler()(M); @@ -2089,7 +2151,7 @@ SmallVector JuliaOJIT::findSymbols(ArrayRef Names) Unmangled[NonOwningSymbolStringPtr(Mangled)] = Unmangled.size(); Exports.add(std::move(Mangled)); } - SymbolMap Syms = cantFail(ES.lookup(orc::makeJITDylibSearchOrder(ArrayRef(&JD)), std::move(Exports))); + SymbolMap Syms = cantFail(::safelookup(ES, orc::makeJITDylibSearchOrder(ArrayRef(&JD)), std::move(Exports))); SmallVector Addrs(Names.size()); for (auto it : Syms) { Addrs[Unmangled.at(orc::NonOwningSymbolStringPtr(it.first))] = it.second.getAddress().getValue(); @@ -2101,7 +2163,7 @@ Expected JuliaOJIT::findSymbol(StringRef Name, bool ExportedS { orc::JITDylib* SearchOrders[3] = {&JD, &GlobalJD, &ExternalJD}; ArrayRef SearchOrder = ArrayRef(&SearchOrders[0], ExportedSymbolsOnly ? 3 : 1); - auto Sym = ES.lookup(SearchOrder, Name); + auto Sym = ::safelookup(ES, SearchOrder, Name); return Sym; } @@ -2114,7 +2176,7 @@ Expected JuliaOJIT::findExternalJDSymbol(StringRef Name, bool { orc::JITDylib* SearchOrders[3] = {&ExternalJD, &GlobalJD, &JD}; ArrayRef SearchOrder = ArrayRef(&SearchOrders[0], ExternalJDOnly ? 1 : 3); - auto Sym = ES.lookup(SearchOrder, getMangledName(Name)); + auto Sym = ::safelookup(ES, SearchOrder, getMangledName(Name)); return Sym; } @@ -2298,125 +2360,6 @@ void JuliaOJIT::optimizeDLSyms(Module &M) { JuliaOJIT *jl_ExecutionEngine; -// destructively move the contents of src into dest -// this assumes that the targets of the two modules are the same -// including the DataLayout and ModuleFlags (for example) -// and that there is no module-level assembly -// Comdat is also removed, since the JIT doesn't need it -void jl_merge_module(orc::ThreadSafeModule &destTSM, orc::ThreadSafeModule srcTSM) -{ - ++ModulesMerged; - destTSM.withModuleDo([&](Module &dest) JL_NOTSAFEPOINT { - srcTSM.withModuleDo([&](Module &src) JL_NOTSAFEPOINT { - assert(&dest != &src && "Cannot merge module with itself!"); - assert(&dest.getContext() == &src.getContext() && "Cannot merge modules with different contexts!"); - assert(dest.getDataLayout() == src.getDataLayout() && "Cannot merge modules with different data layouts!"); - assert(dest.getTargetTriple() == src.getTargetTriple() && "Cannot merge modules with different target triples!"); - - for (auto &SG : make_early_inc_range(src.globals())) { - GlobalVariable *dG = cast_or_null(dest.getNamedValue(SG.getName())); - if (SG.hasLocalLinkage()) { - dG = nullptr; - } - // Replace a declaration with the definition: - if (dG && !dG->hasLocalLinkage()) { - if (SG.isDeclaration()) { - SG.replaceAllUsesWith(dG); - SG.eraseFromParent(); - continue; - } - //// If we start using llvm.used, we need to enable and test this - //else if (!dG->isDeclaration() && dG->hasAppendingLinkage() && SG.hasAppendingLinkage()) { - // auto *dCA = cast(dG->getInitializer()); - // auto *sCA = cast(SG.getInitializer()); - // SmallVector Init; - // for (auto &Op : dCA->operands()) - // Init.push_back(cast_or_null(Op)); - // for (auto &Op : sCA->operands()) - // Init.push_back(cast_or_null(Op)); - // ArrayType *ATy = ArrayType::get(PointerType::get(dest.getContext()), Init.size()); - // GlobalVariable *GV = new GlobalVariable(dest, ATy, dG->isConstant(), - // GlobalValue::AppendingLinkage, ConstantArray::get(ATy, Init), "", - // dG->getThreadLocalMode(), dG->getType()->getAddressSpace()); - // GV->copyAttributesFrom(dG); - // SG.replaceAllUsesWith(GV); - // dG->replaceAllUsesWith(GV); - // GV->takeName(SG); - // SG.eraseFromParent(); - // dG->eraseFromParent(); - // continue; - //} - else { - assert(dG->isDeclaration() || dG->getInitializer() == SG.getInitializer()); - dG->replaceAllUsesWith(&SG); - dG->eraseFromParent(); - } - } - // Reparent the global variable: - SG.removeFromParent(); - dest.insertGlobalVariable(&SG); - // Comdat is owned by the Module - SG.setComdat(nullptr); - } - - for (auto &SG : make_early_inc_range(src)) { - Function *dG = cast_or_null(dest.getNamedValue(SG.getName())); - if (SG.hasLocalLinkage()) { - dG = nullptr; - } - // Replace a declaration with the definition: - if (dG && !dG->hasLocalLinkage()) { - if (SG.isDeclaration()) { - SG.replaceAllUsesWith(dG); - SG.eraseFromParent(); - continue; - } - else { - assert(dG->isDeclaration()); - dG->replaceAllUsesWith(&SG); - dG->eraseFromParent(); - } - } - // Reparent the global variable: - SG.removeFromParent(); - dest.getFunctionList().push_back(&SG); - // Comdat is owned by the Module - SG.setComdat(nullptr); - } - - for (auto &SG : make_early_inc_range(src.aliases())) { - GlobalAlias *dG = cast_or_null(dest.getNamedValue(SG.getName())); - if (SG.hasLocalLinkage()) { - dG = nullptr; - } - if (dG && !dG->hasLocalLinkage()) { - if (!dG->isDeclaration()) { // aliases are always definitions, so this test is reversed from the above two - SG.replaceAllUsesWith(dG); - SG.eraseFromParent(); - continue; - } - else { - dG->replaceAllUsesWith(&SG); - dG->eraseFromParent(); - } - } - SG.removeFromParent(); - dest.insertAlias(&SG); - } - - // metadata nodes need to be explicitly merged not just copied - // so there are special passes here for each known type of metadata - NamedMDNode *sNMD = src.getNamedMetadata("llvm.dbg.cu"); - if (sNMD) { - NamedMDNode *dNMD = dest.getOrInsertNamedMetadata("llvm.dbg.cu"); - for (MDNode *I : sNMD->operands()) { - dNMD->addOperand(I); - } - } - }); - }); -} - //TargetMachine pass-through methods std::unique_ptr JuliaOJIT::cloneTargetMachine() const diff --git a/src/jitlayers.h b/src/jitlayers.h index 35b00891c4944..d8f41d13b6910 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -1,5 +1,6 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license +#include "llvm/ADT/SmallSet.h" #include #include #include @@ -72,7 +73,6 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeContext, LLVMOrcThreadSafeCont DEFINE_SIMPLE_CONVERSION_FUNCTIONS(orc::ThreadSafeModule, LLVMOrcThreadSafeModuleRef) void addTargetPasses(legacy::PassManagerBase *PM, const Triple &triple, TargetIRAnalysis analysis) JL_NOTSAFEPOINT; -void jl_merge_module(orc::ThreadSafeModule &dest, orc::ThreadSafeModule src) JL_NOTSAFEPOINT; GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M) JL_NOTSAFEPOINT; DataLayout jl_create_datalayout(TargetMachine &TM) JL_NOTSAFEPOINT; @@ -210,15 +210,17 @@ struct jl_codegen_call_target_t { llvm::Function *decl; llvm::Function *oc; bool specsig; + bool external_linkage; // whether codegen would like this edge to be externally-available + bool private_linkage; // whether codegen would like this edge to be internally-available + // external = ExternalLinkage (similar to "extern") + // private = InternalLinkage (similar to "static") + // external+private = AvailableExternallyLinkage+ExternalLinkage or ExternalLinkage (similar to "static inline") + // neither = unused }; // reification of a call to jl_jit_abi_convert, so that it isn't necessary to parse the Modules to recover this info struct cfunc_decl_t { - jl_pinned_ref(jl_value_t) declrt; - jl_pinned_ref(jl_value_t) sigt; - size_t nargs; - bool specsig; - llvm::GlobalVariable *theFptr; + jl_abi_t abi; llvm::GlobalVariable *cfuncdata; }; @@ -231,7 +233,7 @@ struct jl_codegen_params_t { DataLayout DL; Triple TargetTriple; - inline LLVMContext &getContext() { + inline LLVMContext &getContext() JL_NOTSAFEPOINT { return *tsctx.getContext(); } typedef StringMap SymMapGV; @@ -241,6 +243,7 @@ struct jl_codegen_params_t { // This map may hold Julia obj ref in the native heap. We need to pin the void*. std::map global_targets; jl_array_t *temporary_roots = nullptr; + SmallSet temporary_roots_set; std::map, GlobalVariable*> external_fns; std::map ditypes; std::map llvmtypes; @@ -269,6 +272,7 @@ struct jl_codegen_params_t { bool cache = false; bool external_linkage = false; bool imaging_mode; + bool safepoint_on_entry = true; bool use_swiftcc = true; jl_codegen_params_t(orc::ThreadSafeContext ctx, DataLayout DL, Triple triple) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER : tsctx(std::move(ctx)), @@ -306,6 +310,9 @@ jl_llvm_functions_t jl_emit_codedecls( jl_code_instance_t *codeinst, jl_codegen_params_t ¶ms); +void linkFunctionBody(Function &Dst, Function &Src) JL_NOTSAFEPOINT; +void emit_always_inline(orc::ThreadSafeModule &result_m, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT_LEAVE JL_NOTSAFEPOINT_ENTER; + enum CompilationPolicy { Default = 0, Extern = 1, @@ -315,10 +322,10 @@ Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tupletype_t * jl_codegen_params_t ¶ms); extern "C" JL_DLLEXPORT_CODEGEN -void *jl_jit_abi_convert(jl_task_t *ct, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data); -std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *invoke); -std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_code_instance_t *codeinst, Value *target, bool target_specsig); -std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, bool specsig, jl_value_t *rettype_const); +void *jl_jit_abi_convert(jl_task_t *ct, jl_abi_t from_abi, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data); +std::string emit_abi_dispatcher(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_code_instance_t *codeinst, Value *invoke); +std::string emit_abi_converter(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_code_instance_t *codeinst, Value *target, bool target_specsig); +std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, jl_abi_t from_abi, jl_value_t *rettype_const); std::string emit_abi_constreturn(Module *M, jl_codegen_params_t ¶ms, bool specsig, jl_code_instance_t *codeinst); Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, Module *M, jl_codegen_params_t ¶ms) JL_NOTSAFEPOINT; @@ -662,8 +669,8 @@ class JuliaOJIT { OptSelLayerT OptSelLayer; }; extern JuliaOJIT *jl_ExecutionEngine; -std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &ctx, const DataLayout &DL = jl_ExecutionEngine->getDataLayout(), const Triple &triple = jl_ExecutionEngine->getTargetTriple()) JL_NOTSAFEPOINT; -inline orc::ThreadSafeModule jl_create_ts_module(StringRef name, orc::ThreadSafeContext ctx, const DataLayout &DL = jl_ExecutionEngine->getDataLayout(), const Triple &triple = jl_ExecutionEngine->getTargetTriple()) JL_NOTSAFEPOINT { +std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &ctx, const DataLayout &DL, const Triple &triple) JL_NOTSAFEPOINT; +inline orc::ThreadSafeModule jl_create_ts_module(StringRef name, orc::ThreadSafeContext ctx, const DataLayout &DL, const Triple &triple) JL_NOTSAFEPOINT { auto lock = ctx.getLock(); return orc::ThreadSafeModule(jl_create_llvm_module(name, *ctx.getContext(), DL, triple), ctx); } diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index df3b9c121837c..76e8368132424 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -65,7 +65,7 @@ XX(jl_interconditional_type) \ XX(jl_interrupt_exception) \ XX(jl_intrinsic_type) \ - XX(jl_kwcall_func) \ + XX(jl_kwcall_type) \ XX(jl_libdl_module) \ XX(jl_libdl_dlopen_func) \ XX(jl_lineinfonode_type) \ @@ -91,13 +91,13 @@ XX(jl_method_match_type) \ XX(jl_method_type) \ XX(jl_methtable_type) \ + XX(jl_methcache_type) \ XX(jl_missingcodeerror_type) \ XX(jl_module_type) \ XX(jl_n_threads_per_pool) \ XX(jl_namedtuple_type) \ XX(jl_namedtuple_typename) \ XX(jl_newvarnode_type) \ - XX(jl_nonfunction_mt) \ XX(jl_nothing) \ XX(jl_nothing_type) \ XX(jl_number_type) \ @@ -135,7 +135,6 @@ XX(jl_typename_type) \ XX(jl_typeofbottom_type) \ XX(jl_type_type) \ - XX(jl_type_type_mt) \ XX(jl_type_typename) \ XX(jl_uint16_type) \ XX(jl_uint32_type) \ diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 56c067b453443..1eb7dbe08b524 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -2,7 +2,6 @@ #define JL_RUNTIME_EXPORTED_FUNCS(XX) \ XX(jl_active_task_stack) \ - XX(jl_add_standard_imports) \ XX(jl_adopt_thread) \ XX(jl_alignment) \ XX(jl_alloc_array_1d) \ @@ -98,7 +97,6 @@ XX(jl_cstr_to_string) \ XX(jl_current_exception) \ XX(jl_debug_method_invalidation) \ - XX(jl_defines_or_exports_p) \ XX(jl_deprecate_binding) \ XX(jl_dlclose) \ XX(jl_dlopen) \ @@ -127,11 +125,6 @@ XX(jl_exit) \ XX(jl_exit_on_sigint) \ XX(jl_exit_threaded_region) \ - XX(jl_expand) \ - XX(jl_expand_stmt) \ - XX(jl_expand_stmt_with_loc) \ - XX(jl_expand_with_loc) \ - XX(jl_expand_with_loc_warn) \ XX(jl_field_index) \ XX(jl_field_isdefined) \ XX(jl_gc_add_finalizer) \ @@ -187,6 +180,7 @@ XX(jl_gdblookup) \ XX(jl_generating_output) \ XX(jl_declare_const_gf) \ + XX(jl_declare_constant_val) \ XX(jl_gensym) \ XX(jl_getaffinity) \ XX(jl_getallocationgranularity) \ @@ -199,6 +193,7 @@ XX(jl_check_binding_currently_writable) \ XX(jl_get_cpu_name) \ XX(jl_get_cpu_features) \ + XX(jl_get_sysimage_cpu_target) \ XX(jl_cpu_has_fma) \ XX(jl_get_current_task) \ XX(jl_get_default_sysimg_path) \ @@ -244,9 +239,12 @@ XX(jl_hrtime) \ XX(jl_idtable_rehash) \ XX(jl_init) \ + XX(jl_init_) \ XX(jl_init_options) \ XX(jl_init_restored_module) \ XX(jl_init_with_image) \ + XX(jl_init_with_image_file) \ + XX(jl_init_with_image_handle) \ XX(jl_install_sigint_handler) \ XX(jl_instantiate_type_in_env) \ XX(jl_instantiate_unionall) \ @@ -271,6 +269,7 @@ XX(jl_istopmod) \ XX(jl_is_binding_deprecated) \ XX(jl_is_char_signed) \ + XX(jl_is_compilable) \ XX(jl_is_const) \ XX(jl_is_assertsbuild) \ XX(jl_is_debugbuild) \ @@ -293,6 +292,7 @@ XX(jl_load_dynamic_library) \ XX(jl_load_file_string) \ XX(jl_lookup_code_address) \ + XX(jl_lower) \ XX(jl_lseek) \ XX(jl_lstat) \ XX(jl_macroexpand) \ @@ -317,9 +317,7 @@ XX(jl_module_names) \ XX(jl_module_parent) \ XX(jl_module_getloc) \ - XX(jl_module_public) \ XX(jl_module_public_p) \ - XX(jl_module_use) \ XX(jl_module_using) \ XX(jl_module_usings) \ XX(jl_module_uuid) \ @@ -522,6 +520,7 @@ YY(jl_get_LLVM_VERSION) \ YY(jl_dump_native) \ YY(jl_get_llvm_gvs) \ + YY(jl_get_llvm_gv_inits) \ YY(jl_get_llvm_external_fns) \ YY(jl_get_llvm_mis) \ YY(jl_dump_function_asm) \ diff --git a/src/jl_uv.c b/src/jl_uv.c index 3ab1961457918..13ad74c957111 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -8,6 +8,9 @@ #include #include +// Needs to come before windows platform headers +#include "support/dtypes.h" + #ifdef _OS_WINDOWS_ #include #include @@ -160,10 +163,11 @@ static void jl_uv_call_close_callback(jl_value_t *val) { jl_value_t **args; JL_GC_PUSHARGS(args, 2); // val is "rooted" in the finalizer list only right now - args[0] = jl_get_global(jl_base_relative_to(((jl_datatype_t*)jl_typeof(val))->name->module), - jl_symbol("_uv_hook_close")); // topmod(typeof(val))._uv_hook_close + args[0] = jl_eval_global_var( + jl_base_relative_to(((jl_datatype_t*)jl_typeof(val))->name->module), + jl_symbol("_uv_hook_close"), + jl_current_task->world_age); // topmod(typeof(val))._uv_hook_close args[1] = val; - assert(args[0]); jl_apply(args, 2); // TODO: wrap in try-catch? JL_GC_POP(); } diff --git a/src/jlapi.c b/src/jlapi.c index 53585555b8a23..dfe5648afe17e 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -26,11 +26,13 @@ extern "C" { #include #endif +static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel, const char* julia_bindir); + /** * @brief Check if Julia is already initialized. * - * Determine if Julia has been previously initialized - * via `jl_init` or `jl_init_with_image`. + * Determine if Julia has been previously initialized via `jl_init` or + * `jl_init_with_image_file` or `jl_init_with_image_handle`. * * @return Returns 1 if Julia is initialized, 0 otherwise. */ @@ -48,26 +50,44 @@ JL_DLLEXPORT int jl_is_initialized(void) * @param argc The number of command line arguments. * @param argv Array of command line arguments. */ -JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv) +JL_DLLEXPORT jl_value_t *jl_set_ARGS(int argc, char **argv) { - if (jl_core_module != NULL) { - jl_array_t *args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS")); - if (args == NULL) { - args = jl_alloc_vec_any(0); - JL_GC_PUSH1(&args); + jl_array_t *args = NULL; + jl_value_t *vecstr = NULL; + JL_GC_PUSH2(&args, &vecstr); + if (jl_core_module != NULL) + args = (jl_array_t*)jl_get_global(jl_core_module, jl_symbol("ARGS")); + if (args == NULL) { + vecstr = jl_apply_array_type((jl_value_t*)jl_string_type, 1); + args = jl_alloc_array_1d(vecstr, 0); + if (jl_core_module != NULL) jl_set_const(jl_core_module, jl_symbol("ARGS"), (jl_value_t*)args); - JL_GC_POP(); - } - assert(jl_array_nrows(args) == 0); - jl_array_grow_end(args, argc); - int i; - for (i = 0; i < argc; i++) { - jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); - jl_array_ptr_set(args, i, s); - } } + assert(jl_array_nrows(args) == 0); + jl_array_grow_end(args, argc); + int i; + for (i = 0; i < argc; i++) { + jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); + jl_array_ptr_set(args, i, s); + } + JL_GC_POP(); + return (jl_value_t*)args; } +JL_DLLEXPORT void jl_init_with_image_handle(void *handle) { + if (jl_is_initialized()) + return; + + const char *image_path = jl_pathname_for_handle(handle); + jl_options.image_file = image_path; + + jl_resolve_sysimg_location(JL_IMAGE_JULIA_HOME, NULL); + jl_image_buf_t sysimage = jl_set_sysimg_so(handle); + + jl_init_(sysimage); + + jl_exception_clear(); +} /** * @brief Initialize Julia with a specified system image file. * @@ -82,21 +102,31 @@ JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv) * @param image_path The path of a system image file (*.so). Interpreted as relative to julia_bindir * or the default Julia home directory if not an absolute path. */ -JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, - const char *image_path) +JL_DLLEXPORT void jl_init_with_image_file(const char *julia_bindir, + const char *image_path) { if (jl_is_initialized()) return; - libsupport_init(); - jl_options.julia_bindir = julia_bindir; if (image_path != NULL) jl_options.image_file = image_path; else jl_options.image_file = jl_get_default_sysimg_path(); - julia_init(JL_IMAGE_JULIA_HOME); + + jl_resolve_sysimg_location(JL_IMAGE_JULIA_HOME, julia_bindir); + jl_image_buf_t sysimage = jl_preload_sysimg(jl_options.image_file); + + jl_init_(sysimage); + jl_exception_clear(); } +// Deprecated function, kept for backward compatibility +JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, + const char *image_path) +{ + jl_init_with_image_file(julia_bindir, image_path); +} + /** * @brief Initialize the Julia runtime. * @@ -105,18 +135,7 @@ JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, */ JL_DLLEXPORT void jl_init(void) { - char *libbindir = NULL; -#ifdef _OS_WINDOWS_ - libbindir = strdup(jl_get_libdir()); -#else - (void)asprintf(&libbindir, "%s" PATHSEPSTRING ".." PATHSEPSTRING "%s", jl_get_libdir(), "bin"); -#endif - if (!libbindir) { - printf("jl_init unable to find libjulia!\n"); - abort(); - } - jl_init_with_image(libbindir, jl_get_default_sysimg_path()); - free(libbindir); + jl_init_with_image_file(NULL, jl_get_default_sysimg_path()); } static void _jl_exception_clear(jl_task_t *ct) JL_NOTSAFEPOINT @@ -943,12 +962,14 @@ static NOINLINE int true_main(int argc, char *argv[]) ct->world_age = jl_get_world_counter(); jl_function_t *start_client = jl_base_module ? - (jl_function_t*)jl_get_global(jl_base_module, jl_symbol("_start")) : NULL; + (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("_start"), ct->world_age) : NULL; if (start_client) { int ret = 1; JL_TRY { + JL_GC_PUSH1(&start_client); jl_value_t *r = jl_apply(&start_client, 1); + JL_GC_POP(); if (jl_typeof(r) != (jl_value_t*)jl_int32_type) jl_type_error("typeassert", (jl_value_t*)jl_int32_type, r); ret = jl_unbox_int32(r); @@ -975,8 +996,8 @@ static NOINLINE int true_main(int argc, char *argv[]) while (!ios_eof(ios_stdin)) { char *volatile line = NULL; JL_TRY { - ios_puts("\njulia> ", ios_stdout); - ios_flush(ios_stdout); + jl_printf(JL_STDOUT, "\njulia> "); + jl_uv_flush(JL_STDOUT); line = ios_readline(ios_stdin); jl_value_t *val = (jl_value_t*)jl_eval_string(line); JL_GC_PUSH1(&val); @@ -992,7 +1013,6 @@ static NOINLINE int true_main(int argc, char *argv[]) jl_printf(JL_STDOUT, "\n"); free(line); line = NULL; - jl_process_events(); } JL_CATCH { if (line) { @@ -1102,7 +1122,14 @@ JL_DLLEXPORT int jl_repl_entrypoint(int argc, char *argv[]) jl_error("Failed to self-execute"); } - julia_init(jl_options.image_file_specified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME); + JL_IMAGE_SEARCH rel = jl_options.image_file_specified ? JL_IMAGE_CWD : JL_IMAGE_JULIA_HOME; + jl_resolve_sysimg_location(rel, NULL); + jl_image_buf_t sysimage = { JL_IMAGE_KIND_NONE }; + if (jl_options.image_file) + sysimage = jl_preload_sysimg(jl_options.image_file); + + jl_init_(sysimage); + if (lisp_prompt) { jl_current_task->world_age = jl_get_world_counter(); jl_lisp_prompt(); @@ -1113,6 +1140,180 @@ JL_DLLEXPORT int jl_repl_entrypoint(int argc, char *argv[]) return ret; } +// create an absolute-path copy of the input path format string +// formed as `joinpath(replace(pwd(), "%" => "%%"), in)` +// unless `in` starts with `%` +static const char *absformat(const char *in) +{ + if (in[0] == '%' || jl_isabspath(in)) + return in; + // get an escaped copy of cwd + size_t path_size = JL_PATH_MAX; + char path[JL_PATH_MAX]; + if (uv_cwd(path, &path_size)) { + jl_error("fatal error: unexpected error while retrieving current working directory"); + } + size_t sz = strlen(in) + 1; + size_t i, fmt_size = 0; + for (i = 0; i < path_size; i++) + fmt_size += (path[i] == '%' ? 2 : 1); + char *out = (char*)malloc_s(fmt_size + 1 + sz); + fmt_size = 0; + for (i = 0; i < path_size; i++) { // copy-replace pwd portion + char c = path[i]; + out[fmt_size++] = c; + if (c == '%') + out[fmt_size++] = '%'; + } + out[fmt_size++] = PATHSEPSTRING[0]; // path sep + memcpy(out + fmt_size, in, sz); // copy over format, including nul + return out; +} + +static char *absrealpath(const char *in, int nprefix) +{ // compute an absolute realpath location, so that chdir doesn't change the file reference + // ignores (copies directly over) nprefix characters at the start of abspath +#ifndef _OS_WINDOWS_ + char *out = realpath(in + nprefix, NULL); + if (out) { + if (nprefix > 0) { + size_t sz = strlen(out) + 1; + char *cpy = (char*)malloc_s(sz + nprefix); + memcpy(cpy, in, nprefix); + memcpy(cpy + nprefix, out, sz); + free(out); + out = cpy; + } + } + else { + size_t sz = strlen(in + nprefix) + 1; + if (in[nprefix] == PATHSEPSTRING[0]) { + out = (char*)malloc_s(sz + nprefix); + memcpy(out, in, sz + nprefix); + } + else { + size_t path_size = JL_PATH_MAX; + char *path = (char*)malloc_s(JL_PATH_MAX); + if (uv_cwd(path, &path_size)) { + jl_error("fatal error: unexpected error while retrieving current working directory"); + } + out = (char*)malloc_s(path_size + 1 + sz + nprefix); + memcpy(out, in, nprefix); + memcpy(out + nprefix, path, path_size); + out[nprefix + path_size] = PATHSEPSTRING[0]; + memcpy(out + nprefix + path_size + 1, in + nprefix, sz); + free(path); + } + } +#else + // GetFullPathName intentionally errors if given an empty string so manually insert `.` to invoke cwd + char *in2 = (char*)malloc_s(JL_PATH_MAX); + if (strlen(in) - nprefix == 0) { + memcpy(in2, in, nprefix); + in2[nprefix] = '.'; + in2[nprefix+1] = '\0'; + in = in2; + } + DWORD n = GetFullPathName(in + nprefix, 0, NULL, NULL); + if (n <= 0) { + jl_error("fatal error: jl_options.image_file path too long or GetFullPathName failed"); + } + char *out = (char*)malloc_s(n + nprefix); + DWORD m = GetFullPathName(in + nprefix, n, out + nprefix, NULL); + if (n != m + 1) { + jl_error("fatal error: jl_options.image_file path too long or GetFullPathName failed"); + } + memcpy(out, in, nprefix); + free(in2); +#endif + return out; +} + +static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel, const char* julia_bindir) +{ + libsupport_init(); + jl_init_timing(); + + // this function resolves the paths in jl_options to absolute file locations as needed + // and it replaces the pointers to `julia_bindir`, `julia_bin`, `image_file`, and output file paths + // it may fail, print an error, and exit(1) if any of these paths are longer than JL_PATH_MAX + // + // note: if you care about lost memory, you should call the appropriate `free()` function + // on the original pointer for each `char*` you've inserted into `jl_options`, after + // calling `jl_init_()` + char *free_path = (char*)malloc_s(JL_PATH_MAX); + size_t path_size = JL_PATH_MAX; + if (uv_exepath(free_path, &path_size)) { + jl_error("fatal error: unexpected error while retrieving exepath"); + } + if (path_size >= JL_PATH_MAX) { + jl_error("fatal error: jl_options.julia_bin path too long"); + } + jl_options.julia_bin = (char*)malloc_s(path_size + 1); + memcpy((char*)jl_options.julia_bin, free_path, path_size); + ((char*)jl_options.julia_bin)[path_size] = '\0'; + if (julia_bindir == NULL) { + jl_options.julia_bindir = getenv("JULIA_BINDIR"); + if (!jl_options.julia_bindir) { +#ifdef _OS_WINDOWS_ + jl_options.julia_bindir = strdup(jl_get_libdir()); +#else + int written = asprintf((char**)&jl_options.julia_bindir, "%s" PATHSEPSTRING ".." PATHSEPSTRING "%s", jl_get_libdir(), "bin"); + if (written < 0) + abort(); // unexpected: memory allocation failed +#endif + } + } else { + jl_options.julia_bindir = julia_bindir; + } + if (jl_options.julia_bindir) + jl_options.julia_bindir = absrealpath(jl_options.julia_bindir, 0); + free(free_path); + free_path = NULL; + if (jl_options.image_file) { + if (rel == JL_IMAGE_JULIA_HOME && !jl_isabspath(jl_options.image_file)) { + // build time path, relative to JULIA_BINDIR + free_path = (char*)malloc_s(JL_PATH_MAX); + int n = snprintf(free_path, JL_PATH_MAX, "%s" PATHSEPSTRING "%s", + jl_options.julia_bindir, jl_options.image_file); + if (n >= JL_PATH_MAX || n < 0) { + jl_error("fatal error: jl_options.image_file path too long"); + } + jl_options.image_file = free_path; + } + if (jl_options.image_file) + jl_options.image_file = absrealpath(jl_options.image_file, 0); + if (free_path) { + free(free_path); + free_path = NULL; + } + } + if (jl_options.outputo) + jl_options.outputo = absrealpath(jl_options.outputo, 0); + if (jl_options.outputji) + jl_options.outputji = absrealpath(jl_options.outputji, 0); + if (jl_options.outputbc) + jl_options.outputbc = absrealpath(jl_options.outputbc, 0); + if (jl_options.outputasm) + jl_options.outputasm = absrealpath(jl_options.outputasm, 0); + if (jl_options.machine_file) + jl_options.machine_file = absrealpath(jl_options.machine_file, 0); + if (jl_options.output_code_coverage) + jl_options.output_code_coverage = absformat(jl_options.output_code_coverage); + if (jl_options.tracked_path) + jl_options.tracked_path = absrealpath(jl_options.tracked_path, 0); + + const char **cmdp = jl_options.cmds; + if (cmdp) { + for (; *cmdp; cmdp++) { + const char *cmd = *cmdp; + if (cmd[0] == 'L') { + *cmdp = absrealpath(cmd, 1); + } + } + } +} + #ifdef __cplusplus } #endif diff --git a/src/jlfrontend.scm b/src/jlfrontend.scm index c313a1e9b0db5..a678d481eab1f 100644 --- a/src/jlfrontend.scm +++ b/src/jlfrontend.scm @@ -109,7 +109,7 @@ ;; return a lambda expression representing a thunk for a top-level expression ;; note: expansion of stuff inside module is delayed, so the contents obey ;; toplevel expansion order (don't expand until stuff before is evaluated). -(define (expand-toplevel-expr-- e file line) +(define (lower-toplevel-expr-- e file line) (let ((lno (first-lineno e)) (ex0 (julia-expand-macroscope e))) (if (and lno (or (not (length= lno 3)) (not (atom? (caddr lno))))) (set! lno #f)) @@ -118,8 +118,8 @@ ex0 (if lno `(toplevel ,lno ,ex0) ex0)) (let* ((linenode (if (and lno (or (= line 0) (eq? file 'none))) lno `(line ,line ,file))) - (ex (julia-expand0 ex0 linenode)) - (th (julia-expand1 + (ex (julia-lower0 ex0 linenode)) + (th (julia-lower1 `(lambda () () (scope-block ,(blockify ex lno))) @@ -142,20 +142,20 @@ error incomplete)) (and (memq (car e) '(global const)) (every symbol? (cdr e)))))) -(define *in-expand* #f) +(define *in-lowering* #f) -(define (expand-toplevel-expr e file line) +(define (lower-toplevel-expr e file line) (cond ((or (atom? e) (toplevel-only-expr? e)) (if (underscore-symbol? e) (error "all-underscore identifiers are write-only and their values cannot be used in expressions")) e) (else - (let ((last *in-expand*)) + (let ((last *in-lowering*)) (if (not last) (begin (reset-gensyms) - (set! *in-expand* #t))) - (begin0 (expand-toplevel-expr-- e file line) - (set! *in-expand* last)))))) + (set! *in-lowering* #t))) + (begin0 (lower-toplevel-expr-- e file line) + (set! *in-lowering* last)))))) ;; used to collect warnings during lowering, which are usually discarded ;; unless logging is requested @@ -163,17 +163,15 @@ ;; expand a piece of raw surface syntax to an executable thunk -(define (expand-to-thunk- expr file line) +(define (lower-to-thunk- expr file line) (error-wrap (lambda () - (expand-toplevel-expr expr file line)))) + (lower-toplevel-expr expr file line)))) -(define (expand-to-thunk-stmt- expr file line) - (expand-to-thunk- (if (toplevel-only-expr? expr) - expr - `(block ,expr (null))) - file line)) - -(define (jl-expand-to-thunk-warn expr file line stmt) +;; Returns a list `(,lowered-code ,warnings) where +;; - warnings (currently only ambiguous soft scope assignments) may be ignored, +;; e.g. when running interactively +;; - more items may be added to the list later +(define (jl-lower-to-thunk expr file line) (let ((warnings '())) (with-bindings ;; Abuse scm_to_julia here to convert arguments to warn. This is meant for @@ -183,30 +181,22 @@ (let ((line (if (= warn_line 0) line warn_line)) (file (if (eq? warn_file 'none) file warn_file))) (set! warnings (cons (list* 'warn level group (symbol (string file line)) file line lst) warnings)))))) - (let ((thunk (if stmt - (expand-to-thunk-stmt- expr file line) - (expand-to-thunk- expr file line)))) - (if (pair? warnings) `(warn ,@(reverse warnings) ,thunk) thunk))))) - -(define (jl-expand-to-thunk expr file line) - (expand-to-thunk- expr file line)) - -(define (jl-expand-to-thunk-stmt expr file line) - (expand-to-thunk-stmt- expr file line)) + `(,(lower-to-thunk- expr file line) + ,(reverse warnings))))) (define (jl-expand-macroscope expr) (error-wrap (lambda () (julia-expand-macroscope expr)))) (define (jl-default-inner-ctor-body field-kinds file line) - (expand-to-thunk- (default-inner-ctor-body (cdr field-kinds) file line) file line)) + (lower-to-thunk- (default-inner-ctor-body (cdr field-kinds) file line) file line)) (define (jl-default-outer-ctor-body args file line) - (expand-to-thunk- (default-outer-ctor-body (cadr args) (caddr args) (cadddr args) file line) file line)) + (lower-to-thunk- (default-outer-ctor-body (cadr args) (caddr args) (cadddr args) file line) file line)) ; run whole frontend on a string. useful for testing. (define (fe str) - (expand-toplevel-expr (julia-parse str) 'none 0)) + (lower-toplevel-expr (julia-parse str) 'none 0)) (define (profile-e s) (with-exception-catcher diff --git a/src/jloptions.c b/src/jloptions.c index a583eb0844f8f..5b943f404e3b4 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -107,6 +107,7 @@ JL_DLLEXPORT void jl_init_options(void) 0, // nprocs NULL, // machine_file NULL, // project + NULL, // program_file 0, // isinteractive 0, // color JL_OPTIONS_HISTORYFILE_ON, // history file @@ -159,6 +160,7 @@ JL_DLLEXPORT void jl_init_options(void) JL_TRIM_NO, // trim 0, // task_metrics -1, // timeout_for_safepoint_straggler_s + 0, // gc_sweep_always_full }; jl_options_initialized = 1; } @@ -174,11 +176,13 @@ static const char opts[] = " --help-hidden Print uncommon options not shown by `-h`\n\n" // startup options - " --project[={|@temp|@.}] Set as the active project/environment.\n" + " --project[={|@temp|@.|@script[]}] Set as the active project/environment.\n" " Or, create a temporary environment with `@temp`\n" " The default @. option will search through parent\n" " directories until a Project.toml or JuliaProject.toml\n" - " file is found.\n" + " file is found. @script is similar, but searches up\n" + " from the programfile or a path relative to\n" + " programfile.\n" " -J, --sysimage Start up with the given system image file\n" " -H, --home Set location of `julia` executable\n" " --startup-file={yes*|no} Load `JULIA_DEPOT_PATH/config/startup.jl`; \n" @@ -325,24 +329,38 @@ static const char opts_hidden[] = " --output-asm Generate an assembly file (.s)\n" " --output-incremental={yes|no*} Generate an incremental output file (rather than\n" " complete)\n" - " --timeout-for-safepoint-straggler If this value is set, then we will dump the backtrace for a thread\n" - " that fails to reach a safepoint within the specified time\n" + " --timeout-for-safepoint-straggler If this value is set, then we will dump the backtrace\n" + " for a thread that fails to reach a safepoint within\n" + " the specified time\n" " --trace-compile={stderr|name} Print precompile statements for methods compiled\n" - " during execution or save to stderr or a path. Methods that\n" - " were recompiled are printed in yellow or with a trailing\n" - " comment if color is not supported\n" - " --trace-compile-timing If --trace-compile is enabled show how long each took to\n" - " compile in ms\n" + " during execution or save to stderr or a path. Methods\n" + " that were recompiled are printed in yellow or with\n" + " a trailing comment if color is not supported\n" + " --trace-compile-timing If --trace-compile is enabled show how long each took\n" + " to compile in ms\n" " --task-metrics={yes|no*} Enable collection of per-task timing data.\n" " --image-codegen Force generate code in imaging mode\n" - " --permalloc-pkgimg={yes|no*} Copy the data section of package images into memory\n" - " --trim={no*|safe|unsafe|unsafe-warn}\n" - " Build a sysimage including only code provably reachable\n" - " from methods marked by calling `entrypoint`. In unsafe\n" - " mode, the resulting binary might be missing needed code\n" - " and can throw errors. With unsafe-warn warnings will be\n" - " printed for dynamic call sites that might lead to such\n" - " errors. In safe mode compile-time errors are given instead.\n" + " --permalloc-pkgimg={yes|no*} Copy the data section of package images into memory\n\n" + + " --trim={no*|safe|unsafe|unsafe-warn} Build a sysimage including only code provably\n" + " reachable from methods marked by calling\n" + " `entrypoint`. In unsafe mode, the resulting binary\n" + " might be missing needed code and can throw errors.\n" + " With unsafe-warn warnings will be printed for\n" + " dynamic call sites that might lead to such errors.\n" + " In safe mode compile-time errors are given instead.\n" + " --hard-heap-limit=[] Set a hard limit on the heap size: if we ever\n" + " go above this limit, we will abort. The value\n" + " may be specified as a number of bytes,\n" + " optionally in units of: B, K (kibibytes),\n" + " M (mebibytes), G (gibibytes) or T (tebibytes).\n" + " --heap-target-increment=[] Set an upper bound on how much the heap\n" + " target can increase between consecutive\n" + " collections. The value may be specified as\n" + " a number of bytes, optionally in units of:\n" + " B, K (kibibytes), M (mebibytes), G (gibibytes)\n" + " or T (tebibytes).\n" + " --gc-sweep-always-full Makes the GC always do a full sweep of the heap\n" ; JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) @@ -393,6 +411,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_heap_size_hint, opt_hard_heap_limit, opt_heap_target_increment, + opt_gc_sweep_always_full, opt_gc_threads, opt_permalloc_pkgimg, opt_trim, @@ -466,6 +485,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, { "hard-heap-limit", required_argument, 0, opt_hard_heap_limit }, { "heap-target-increment", required_argument, 0, opt_heap_target_increment }, + { "gc-sweep-always-full", no_argument, 0, opt_gc_sweep_always_full }, { "trim", optional_argument, 0, opt_trim }, { 0, 0, 0, 0 } }; @@ -604,12 +624,19 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_options.use_experimental_features = JL_OPTIONS_USE_EXPERIMENTAL_FEATURES_YES; break; case opt_sysimage_native_code: - if (!strcmp(optarg,"yes")) + if (!strcmp(optarg,"yes")) { jl_options.use_sysimage_native_code = JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES; - else if (!strcmp(optarg,"no")) + } + else if (!strcmp(optarg,"no")) { jl_options.use_sysimage_native_code = JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_NO; - else + if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) + jl_errorf("julia: --sysimage-native-code=no is deprecated"); + else if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ON) + jl_printf(JL_STDERR, "WARNING: --sysimage-native-code=no is deprecated\n"); + } + else { jl_errorf("julia: invalid argument to --sysimage-native-code={yes|no} (%s)", optarg); + } break; case opt_compiled_modules: if (!strcmp(optarg,"yes")) @@ -677,6 +704,9 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) if (nthreadsi == 0) jl_options.nthreadpools = 1; } + } else if (nthreads == 1) { // User asked for 1 thread so don't add an interactive one + jl_options.nthreadpools = 1; + nthreadsi = 0; } jl_options.nthreads = nthreads + nthreadsi; } @@ -1016,6 +1046,9 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_errorf("julia: --timeout-for-safepoint-straggler=; seconds must be an integer between 1 and %d", INT16_MAX); jl_options.timeout_for_safepoint_straggler_s = (int16_t)timeout; break; + case opt_gc_sweep_always_full: + jl_options.gc_sweep_always_full = 1; + break; case opt_trim: if (optarg == NULL || !strcmp(optarg,"safe")) jl_options.trim = JL_TRIM_SAFE; @@ -1041,6 +1074,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) "This is a bug, please report it.", c); } } + jl_options.program_file = optind < argc ? strdup(argv[optind]) : ""; parsing_args_done: if (!jl_options.use_experimental_features) { if (jl_options.trim != JL_TRIM_NO) diff --git a/src/jloptions.h b/src/jloptions.h index 0084a3a43cf9a..2a1733a54d59a 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -21,6 +21,7 @@ typedef struct { int32_t nprocs; const char *machine_file; const char *project; + const char *program_file; int8_t isinteractive; int8_t color; int8_t historyfile; @@ -69,6 +70,7 @@ typedef struct { int8_t trim; int8_t task_metrics; int16_t timeout_for_safepoint_straggler_s; + int8_t gc_sweep_always_full; } jl_options_t; #endif diff --git a/src/jltypes.c b/src/jltypes.c index 734290cd37756..c19fd5cc2ebc9 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2954,7 +2954,9 @@ void jl_init_types(void) JL_GC_DISABLED XX(symbol); jl_simplevector_type = jl_new_uninitialized_datatype(); XX(simplevector); + jl_methcache_type = jl_new_uninitialized_datatype(); jl_methtable_type = jl_new_uninitialized_datatype(); + jl_method_table = jl_new_method_table(jl_symbol("methodtable"), core); jl_emptysvec = (jl_svec_t*)jl_gc_permobj(sizeof(void*), jl_simplevector_type, 0); jl_set_typetagof(jl_emptysvec, jl_simplevector_tag, GC_OLD_MARKED); @@ -2962,14 +2964,10 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type = (jl_datatype_t*)jl_new_abstracttype((jl_value_t*)jl_symbol("Any"), core, NULL, jl_emptysvec); jl_any_type->super = jl_any_type; - jl_nonfunction_mt = jl_any_type->name->mt; - jl_any_type->name->mt = NULL; jl_datatype_t *type_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Type"), core, jl_any_type, jl_emptysvec); jl_type_type = (jl_unionall_t*)type_type; jl_type_typename = type_type->name; - jl_type_type_mt = jl_new_method_table(jl_type_typename->name, core); - jl_type_typename->mt = jl_type_type_mt; // initialize them. lots of cycles. // NOTE: types are not actually mutable, but we want to ensure they are heap-allocated with stable addresses @@ -3004,57 +3002,61 @@ void jl_init_types(void) JL_GC_DISABLED jl_typename_type->name = jl_new_typename_in(jl_symbol("TypeName"), core, 0, 1); jl_typename_type->name->wrapper = (jl_value_t*)jl_typename_type; - jl_typename_type->name->mt = jl_nonfunction_mt; jl_typename_type->super = jl_any_type; jl_typename_type->parameters = jl_emptysvec; - jl_typename_type->name->n_uninitialized = 17 - 2; - jl_typename_type->name->names = jl_perm_symsvec(17, "name", "module", + jl_typename_type->name->n_uninitialized = 19 - 2; + jl_typename_type->name->names = jl_perm_symsvec(19, "name", "module", "singletonname", "names", "atomicfields", "constfields", "wrapper", "Typeofwrapper", "cache", "linearcache", - "mt", "partial", - "hash", "n_uninitialized", + "partial", "hash", "max_args", "n_uninitialized", "flags", // "abstract", "mutable", "mayinlinealloc", - "max_methods", "constprop_heuristic", + "cache_entry_count", "max_methods", "constprop_heuristic", "hiddenptrfields"); - const static uint32_t typename_constfields[1] = { 0x00003a27 }; // (1<<0)|(1<<1)|(1<<2)|(1<<5)|(1<<9)|(1<<11)|(1<<12)|(1<<13) ; TODO: put back (1<<3)|(1<<4) in this list - const static uint32_t typename_atomicfields[1] = { 0x00000180 }; // (1<<7)|(1<<8) + const static uint32_t typename_constfields[1] = { 0b000110100001001011 }; // TODO: put back atomicfields and constfields in this list + const static uint32_t typename_atomicfields[1] = { 0b001001001110000000 }; jl_typename_type->name->constfields = typename_constfields; jl_typename_type->name->atomicfields = typename_atomicfields; jl_precompute_memoized_dt(jl_typename_type, 1); - jl_typename_type->types = jl_svec(17, jl_symbol_type, jl_any_type /*jl_module_type*/, - jl_simplevector_type, jl_any_type/*jl_voidpointer_type*/, jl_any_type/*jl_voidpointer_type*/, - jl_type_type, jl_type_type, jl_simplevector_type, jl_simplevector_type, - jl_methtable_type, jl_any_type, - jl_any_type /*jl_long_type*/, jl_any_type /*jl_int32_type*/, + jl_typename_type->types = jl_svec(19, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_symbol_type, + jl_simplevector_type, + jl_any_type/*jl_voidpointer_type*/, jl_any_type/*jl_voidpointer_type*/, + jl_type_type, jl_simplevector_type, jl_simplevector_type, + jl_methcache_type, jl_any_type, + jl_any_type /*jl_long_type*/, + jl_any_type /*jl_int32_type*/, + jl_any_type /*jl_int32_type*/, + jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/, jl_any_type /*jl_uint8_type*/, jl_any_type/*jl_voidpointer_type*/); + jl_methcache_type->name = jl_new_typename_in(jl_symbol("MethodCache"), core, 0, 1); + jl_methcache_type->name->wrapper = (jl_value_t*)jl_methcache_type; + jl_methcache_type->super = jl_any_type; + jl_methcache_type->parameters = jl_emptysvec; + jl_methcache_type->name->n_uninitialized = 4 - 2; + jl_methcache_type->name->names = jl_perm_symsvec(4, "leafcache", "cache", "", ""); + const static uint32_t methcache_atomicfields[1] = { 0b1111 }; + jl_methcache_type->name->atomicfields = methcache_atomicfields; + jl_precompute_memoized_dt(jl_methcache_type, 1); + jl_methcache_type->types = jl_svec(4, jl_any_type, jl_any_type, jl_any_type/*voidpointer*/, jl_any_type/*int32*/); + jl_methtable_type->name = jl_new_typename_in(jl_symbol("MethodTable"), core, 0, 1); jl_methtable_type->name->wrapper = (jl_value_t*)jl_methtable_type; - jl_methtable_type->name->mt = jl_nonfunction_mt; jl_methtable_type->super = jl_any_type; jl_methtable_type->parameters = jl_emptysvec; - jl_methtable_type->name->n_uninitialized = 11 - 6; - jl_methtable_type->name->names = jl_perm_symsvec(11, "name", "defs", - "leafcache", "cache", "max_args", - "module", "backedges", - "", "", "offs", ""); - const static uint32_t methtable_constfields[1] = { 0x00000020 }; // (1<<5); - const static uint32_t methtable_atomicfields[1] = { 0x0000001e }; // (1<<1)|(1<<2)|(1<<3)|(1<<4); + jl_methtable_type->name->n_uninitialized = 0; + jl_methtable_type->name->names = jl_perm_symsvec(5, "defs", "cache", "name", "module", "backedges"); + const static uint32_t methtable_constfields[1] = { 0b01110 }; + const static uint32_t methtable_atomicfields[1] = { 0b00001 }; jl_methtable_type->name->constfields = methtable_constfields; jl_methtable_type->name->atomicfields = methtable_atomicfields; jl_precompute_memoized_dt(jl_methtable_type, 1); - jl_methtable_type->types = jl_svec(11, jl_symbol_type, jl_any_type, jl_any_type, - jl_any_type, jl_any_type/*jl_long*/, - jl_any_type/*module*/, jl_any_type/*any vector*/, - jl_any_type/*voidpointer*/, jl_any_type/*int32*/, - jl_any_type/*uint8*/, jl_any_type/*uint8*/); + jl_methtable_type->types = jl_svec(5, jl_any_type, jl_methcache_type, jl_symbol_type, jl_any_type /*jl_module_type*/, jl_any_type); jl_symbol_type->name = jl_new_typename_in(jl_symbol("Symbol"), core, 0, 1); jl_symbol_type->name->wrapper = (jl_value_t*)jl_symbol_type; - jl_symbol_type->name->mt = jl_nonfunction_mt; jl_symbol_type->super = jl_any_type; jl_symbol_type->parameters = jl_emptysvec; jl_symbol_type->name->n_uninitialized = 0; @@ -3064,7 +3066,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_simplevector_type->name = jl_new_typename_in(jl_symbol("SimpleVector"), core, 0, 1); jl_simplevector_type->name->wrapper = (jl_value_t*)jl_simplevector_type; - jl_simplevector_type->name->mt = jl_nonfunction_mt; jl_simplevector_type->super = jl_any_type; jl_simplevector_type->parameters = jl_emptysvec; jl_simplevector_type->name->n_uninitialized = 0; @@ -3250,8 +3251,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_function_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Function"), core, jl_any_type, jl_emptysvec); jl_builtin_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Builtin"), core, jl_function_type, jl_emptysvec); - jl_function_type->name->mt = NULL; // subtypes of Function have independent method tables - jl_builtin_type->name->mt = NULL; // so they don't share the Any type table jl_svec_t *tv; @@ -3268,8 +3267,10 @@ void jl_init_types(void) JL_GC_DISABLED jl_svec(5, jl_any_type, jl_ulong_type, jl_ulong_type, jl_any_type/*jl_binding_partition_type*/, jl_ulong_type), jl_emptysvec, 0, 1, 0); - const static uint32_t binding_partition_atomicfields[] = { 0b01101 }; // Set fields 1, 3, 4 as atomic + const static uint32_t binding_partition_atomicfields[] = { 0b01110 }; // Set fields 2, 3, 4 as atomic jl_binding_partition_type->name->atomicfields = binding_partition_atomicfields; + const static uint32_t binding_partition_constfields[] = { 0b10001 }; // Set fields 1, 5 as constant + jl_binding_partition_type->name->constfields = binding_partition_constfields; jl_binding_type = jl_new_datatype(jl_symbol("Binding"), core, jl_any_type, jl_emptysvec, @@ -3287,12 +3288,9 @@ void jl_init_types(void) JL_GC_DISABLED jl_perm_symsvec(3, "mod", "name", "binding"), jl_svec(3, jl_module_type, jl_symbol_type, jl_binding_type), jl_emptysvec, 0, 0, 3); + jl_globalref_type->name->mayinlinealloc = 0; // not at all worthwhile, since the only constructor returns a boxed object - core = jl_new_module(jl_symbol("Core"), NULL); - core->parent = core; - jl_type_typename->mt->module = core; - jl_core_module = core; - core = NULL; // not ready yet to use + jl_core_module = jl_new_module(jl_symbol("Core"), NULL); tv = jl_svec1(tvar("Backend")); jl_addrspace_typename = @@ -3377,16 +3375,16 @@ void jl_init_types(void) JL_GC_DISABLED jl_array_uint64_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint64_type, jl_box_long(1)); jl_an_empty_vec_any = (jl_value_t*)jl_alloc_vec_any(0); // used internally jl_an_empty_memory_any = (jl_value_t*)jl_alloc_memory_any(0); // used internally - jl_atomic_store_relaxed(&jl_nonfunction_mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); - jl_atomic_store_relaxed(&jl_type_type_mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); // finish initializing module Core core = jl_core_module; + jl_method_table->module = core; + jl_atomic_store_relaxed(&jl_method_table->cache->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); + jl_method_table->backedges = (jl_genericmemory_t*)jl_an_empty_memory_any; jl_atomic_store_relaxed(&core->bindingkeyset, (jl_genericmemory_t*)jl_an_empty_memory_any); // export own name, so "using Foo" makes "Foo" itself visible - jl_set_const(core, core->name, (jl_value_t*)core); - jl_module_public(core, core->name, 1); - jl_set_const(core, jl_symbol("CPU"), (jl_value_t*)cpumem); + jl_set_initial_const(core, core->name, (jl_value_t*)core, 1); + jl_set_initial_const(core, jl_symbol("CPU"), (jl_value_t*)cpumem, 0); core = NULL; jl_expr_type = @@ -3542,13 +3540,14 @@ void jl_init_types(void) JL_GC_DISABLED jl_method_type = jl_new_datatype(jl_symbol("Method"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(32, + jl_perm_symsvec(33, "name", "module", "file", "line", + "dispatch_status", // atomic + "interferences", // atomic "primary_world", // atomic - "deleted_world", // atomic "sig", "specializations", // !const "speckeyset", // !const @@ -3575,12 +3574,13 @@ void jl_init_types(void) JL_GC_DISABLED "constprop", "max_varargs", "purity"), - jl_svec(32, + jl_svec(33, jl_symbol_type, jl_module_type, jl_symbol_type, jl_int32_type, - jl_ulong_type, + jl_uint8_type, + jl_memory_any_type, jl_ulong_type, jl_type_type, jl_any_type, // union(jl_simplevector_type, jl_method_instance_type), @@ -3610,35 +3610,37 @@ void jl_init_types(void) JL_GC_DISABLED jl_uint16_type), jl_emptysvec, 0, 1, 10); - //const static uint32_t method_constfields[1] = { 0b0 }; // (1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<6)|(1<<9)|(1<<10)|(1<<17)|(1<<21)|(1<<22)|(1<<23)|(1<<24)|(1<<25)|(1<<26)|(1<<27)|(1<<28)|(1<<29)|(1<<30); + //const static uint32_t method_constfields[] = { 0b0, 0b0 }; // (1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<6)|(1<<9)|(1<<10)|(1<<17)|(1<<21)|(1<<22)|(1<<23)|(1<<24)|(1<<25)|(1<<26)|(1<<27)|(1<<28)|(1<<29)|(1<<30); //jl_method_type->name->constfields = method_constfields; - const static uint32_t method_atomicfields[1] = { 0x00000030 }; // (1<<4)|(1<<5) + const static uint32_t method_atomicfields[] = { 0x20000070, 0x0 }; // (1<<4)|(1<<5)|(1<<6)|(1<<29) jl_method_type->name->atomicfields = method_atomicfields; jl_method_instance_type = jl_new_datatype(jl_symbol("MethodInstance"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(7, + jl_perm_symsvec(8, "def", "specTypes", "sparam_vals", "backedges", "cache", "cache_with_orig", - "flags"), - jl_svec(7, + "flags", + "dispatch_status"), + jl_svec(8, jl_new_struct(jl_uniontype_type, jl_method_type, jl_module_type), jl_any_type, jl_simplevector_type, jl_array_any_type, jl_any_type/*jl_code_instance_type*/, jl_bool_type, - jl_bool_type), + jl_bool_type, + jl_uint8_type), jl_emptysvec, 0, 1, 3); // These fields should be constant, but Serialization wants to mutate them in initialization - //const static uint32_t method_instance_constfields[1] = { 0b0000111 }; // fields 1, 2, 3 - const static uint32_t method_instance_atomicfields[1] = { 0b1010000 }; // fields 5, 7 + //const static uint32_t method_instance_constfields[1] = { 0b00000111 }; // fields 1, 2, 3 + const static uint32_t method_instance_atomicfields[1] = { 0b11010000 }; // fields 5, 7, 8 //Fields 4 and 5 must be protected by method->write_lock, and thus all operations on jl_method_instance_t are threadsafe. //jl_method_instance_type->name->constfields = method_instance_constfields; jl_method_instance_type->name->atomicfields = method_instance_atomicfields; @@ -3720,13 +3722,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_svec(4, jl_type_type, jl_simplevector_type, jl_method_type, jl_bool_type), jl_emptysvec, 0, 0, 4); - // all Kinds share the Type method table (not the nonfunction one) - jl_unionall_type->name->mt = - jl_uniontype_type->name->mt = - jl_datatype_type->name->mt = - jl_typeofbottom_type->name->mt = - jl_type_type_mt; - jl_intrinsic_type = jl_new_primitivetype((jl_value_t*)jl_symbol("IntrinsicFunction"), core, jl_builtin_type, jl_emptysvec, 32); @@ -3850,24 +3845,22 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_datatype_type->types, 6, jl_int32_type); jl_svecset(jl_datatype_type->types, 7, jl_uint16_type); jl_svecset(jl_typename_type->types, 1, jl_module_type); - jl_svecset(jl_typename_type->types, 3, jl_voidpointer_type); jl_svecset(jl_typename_type->types, 4, jl_voidpointer_type); - jl_svecset(jl_typename_type->types, 5, jl_type_type); + jl_svecset(jl_typename_type->types, 5, jl_voidpointer_type); jl_svecset(jl_typename_type->types, 6, jl_type_type); + jl_svecset(jl_typename_type->types, 7, jl_type_type); jl_svecset(jl_typename_type->types, 11, jl_long_type); jl_svecset(jl_typename_type->types, 12, jl_int32_type); - jl_svecset(jl_typename_type->types, 13, jl_uint8_type); + jl_svecset(jl_typename_type->types, 13, jl_int32_type); jl_svecset(jl_typename_type->types, 14, jl_uint8_type); jl_svecset(jl_typename_type->types, 15, jl_uint8_type); - jl_svecset(jl_typename_type->types, 16, jl_voidpointer_type); // hiddenptrfields - jl_svecset(jl_methtable_type->types, 4, jl_long_type); - jl_svecset(jl_methtable_type->types, 5, jl_module_type); - jl_svecset(jl_methtable_type->types, 6, jl_array_any_type); - jl_svecset(jl_methtable_type->types, 7, jl_long_type); // voidpointer - jl_svecset(jl_methtable_type->types, 8, jl_long_type); // uint32_t plus alignment - jl_svecset(jl_methtable_type->types, 9, jl_uint8_type); - jl_svecset(jl_methtable_type->types, 10, jl_uint8_type); - jl_svecset(jl_method_type->types, 13, jl_method_instance_type); + jl_svecset(jl_typename_type->types, 16, jl_uint8_type); + jl_svecset(jl_typename_type->types, 17, jl_uint8_type); + jl_svecset(jl_typename_type->types, 18, jl_voidpointer_type); // hiddenptrfields + jl_svecset(jl_methcache_type->types, 2, jl_long_type); // voidpointer + jl_svecset(jl_methcache_type->types, 3, jl_long_type); // uint32_t plus alignment + jl_svecset(jl_methtable_type->types, 3, jl_module_type); + jl_svecset(jl_method_type->types, 14, jl_method_instance_type); //jl_svecset(jl_debuginfo_type->types, 0, jl_method_instance_type); // union(jl_method_instance_type, jl_method_type, jl_symbol_type) jl_svecset(jl_method_instance_type->types, 4, jl_code_instance_type); jl_svecset(jl_code_instance_type->types, 19, jl_voidpointer_type); @@ -3881,6 +3874,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_compute_field_offsets(jl_uniontype_type); jl_compute_field_offsets(jl_tvar_type); jl_compute_field_offsets(jl_methtable_type); + jl_compute_field_offsets(jl_methcache_type); jl_compute_field_offsets(jl_method_instance_type); jl_compute_field_offsets(jl_code_instance_type); jl_compute_field_offsets(jl_unionall_type); @@ -3891,7 +3885,7 @@ void jl_init_types(void) JL_GC_DISABLED // override ismutationfree for builtin types that are mutable for identity jl_string_type->ismutationfree = jl_string_type->isidentityfree = 1; jl_symbol_type->ismutationfree = jl_symbol_type->isidentityfree = 1; - jl_simplevector_type->ismutationfree = jl_simplevector_type->isidentityfree = 1; + jl_simplevector_type->isidentityfree = 1; jl_typename_type->ismutationfree = 1; jl_datatype_type->ismutationfree = 1; jl_uniontype_type->ismutationfree = 1; @@ -3968,9 +3962,9 @@ void post_boot_hooks(void) jl_trimfailure_type = (jl_datatype_t*)core("TrimFailure"); jl_pair_type = core("Pair"); - jl_kwcall_func = core("kwcall"); - jl_kwcall_mt = ((jl_datatype_t*)jl_typeof(jl_kwcall_func))->name->mt; - jl_atomic_store_relaxed(&jl_kwcall_mt->max_args, 0); + jl_value_t *kwcall_func = core("kwcall"); + jl_kwcall_type = (jl_datatype_t*)jl_typeof(kwcall_func); + jl_atomic_store_relaxed(&jl_kwcall_type->name->max_args, 0); jl_weakref_type = (jl_datatype_t*)core("WeakRef"); jl_vecelement_typename = ((jl_datatype_t*)jl_unwrap_unionall(core("VecElement")))->name; @@ -3978,27 +3972,6 @@ void post_boot_hooks(void) jl_abioverride_type = (jl_datatype_t*)core("ABIOverride"); jl_init_box_caches(); - - // set module field of primitive types - jl_svec_t *bindings = jl_atomic_load_relaxed(&jl_core_module->bindings); - jl_value_t **table = jl_svec_data(bindings); - for (size_t i = 0; i < jl_svec_len(bindings); i++) { - if (table[i] != jl_nothing) { - jl_binding_t *b = (jl_binding_t*)table[i]; - jl_value_t *v = jl_get_binding_value(b); - if (v) { - if (jl_is_unionall(v)) - v = jl_unwrap_unionall(v); - if (jl_is_datatype(v)) { - jl_datatype_t *tt = (jl_datatype_t*)v; - tt->name->module = jl_core_module; - if (tt->name->mt) - tt->name->mt->module = jl_core_module; - } - } - } - } - export_jl_small_typeof(); } diff --git a/src/julia-parser.scm b/src/julia-parser.scm index 4415dc8686065..1a11494b5c8e3 100644 --- a/src/julia-parser.scm +++ b/src/julia-parser.scm @@ -10,7 +10,7 @@ ;; comma - higher than assignment outside parentheses, lower when inside (define prec-pair (add-dots '(=>))) (define prec-conditional '(?)) -(define prec-arrow (add-dots '(← → ↔ ↚ ↛ ↞ ↠ ↢ ↣ ↦ ↤ ↮ ⇎ ⇍ ⇏ ⇐ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⥷ ⭄ ⥺ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ￩ ￫ ⇜ ⇝ ↜ ↝ ↩ ↪ ↫ ↬ ↼ ↽ ⇀ ⇁ ⇄ ⇆ ⇇ ⇉ ⇋ ⇌ ⇚ ⇛ ⇠ ⇢ ↷ ↶ ↺ ↻ --> <-- <-->))) +(define prec-arrow (add-dots '(← → ↔ ↚ ↛ ↞ ↠ ↢ ↣ ↦ ↤ ↮ ⇎ ⇍ ⇏ ⇐ ⇒ ⇔ ⇴ ⇶ ⇷ ⇸ ⇹ ⇺ ⇻ ⇼ ⇽ ⇾ ⇿ ⟵ ⟶ ⟷ ⟹ ⟺ ⟻ ⟼ ⟽ ⟾ ⟿ ⤀ ⤁ ⤂ ⤃ ⤄ ⤅ ⤆ ⤇ ⤌ ⤍ ⤎ ⤏ ⤐ ⤑ ⤔ ⤕ ⤖ ⤗ ⤘ ⤝ ⤞ ⤟ ⤠ ⥄ ⥅ ⥆ ⥇ ⥈ ⥊ ⥋ ⥎ ⥐ ⥒ ⥓ ⥖ ⥗ ⥚ ⥛ ⥞ ⥟ ⥢ ⥤ ⥦ ⥧ ⥨ ⥩ ⥪ ⥫ ⥬ ⥭ ⥰ ⧴ ⬱ ⬰ ⬲ ⬳ ⬴ ⬵ ⬶ ⬷ ⬸ ⬹ ⬺ ⬻ ⬼ ⬽ ⬾ ⬿ ⭀ ⭁ ⭂ ⭃ ⥷ ⭄ ⥺ ⭇ ⭈ ⭉ ⭊ ⭋ ⭌ ￩ ￫ ⇜ ⇝ ↜ ↝ ↩ ↪ ↫ ↬ ↼ ↽ ⇀ ⇁ ⇄ ⇆ ⇇ ⇉ ⇋ ⇌ ⇚ ⇛ ⇠ ⇢ ↷ ↶ ↺ ↻ --> <-- <--> 🢲))) (define prec-lazy-or (add-dots '(|\|\||))) (define prec-lazy-and (add-dots '(&&))) (define prec-comparison diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 301d9fd82e4c2..69869e3e923ec 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3,11 +3,16 @@ ;; pass 1: syntax desugaring -;; allow (:: T) => (:: #gensym T) in formal argument lists +;; unnamed or all-underscore arguments may still be read from internally, so +;; convert (:: T) => (:: #gensym T) and _ => #gensym in formal argument lists (define (fill-missing-argname a unused) - (if (and (pair? a) (eq? (car a) '|::|) (null? (cddr a))) - `(|::| ,(if unused UNUSED (gensy)) ,(cadr a)) - a)) + (define (replace-if-underscore u) + (if (underscore-symbol? u) (if unused UNUSED (gensy)) u)) + (if (and (pair? a) (eq? (car a) '|::|)) + (cond ((null? (cddr a)) `(|::| ,(if unused UNUSED (gensy)) ,(cadr a))) + ((null? (cdddr a)) `(|::| ,(replace-if-underscore (cadr a)) ,(caddr a))) + (else a)) + (replace-if-underscore a))) (define (fix-arglist l (unused #t)) (if (any vararg? (butlast l)) (error "invalid \"...\" on non-final argument")) @@ -165,10 +170,7 @@ ;; GF method does not need to keep decl expressions on lambda args ;; except for rest arg (define (method-lambda-expr argl body rett) - (let ((argl (map (lambda (x) - (let ((n (arg-name x))) - (if (underscore-symbol? n) UNUSED n))) - argl)) + (let ((argl (map arg-name argl)) (body (blockify body))) `(lambda ,argl () (scope-block @@ -374,7 +376,7 @@ (append req opt vararg) rett))))) ;; no optional positional args (let* ((names (map car sparams)) - (anames (map (lambda (x) (if (underscore-symbol? x) UNUSED x)) (llist-vars argl))) + (anames (llist-vars argl)) (unused_anames (filter (lambda (x) (not (eq? x UNUSED))) anames)) (ename (if (nodot-sym-ref? name) name (if (overlay? name) (cadr name) `(null))))) @@ -439,9 +441,11 @@ ,body)))) (if (or (symbol? name) (globalref? name)) `(block ,@generator (method ,name) (latestworld-if-toplevel) ,mdef (unnecessary ,name)) ;; return the function - (if (not (null? generator)) - `(block ,@generator ,mdef) - mdef)))))) + (if (overlay? name) + (if (not (null? generator)) + `(block ,@generator ,mdef) + mdef) + `(block ,@generator ,mdef (null)))))))) ;; wrap expr in nested scopes assigning names to vals (define (scopenest names vals expr) @@ -541,14 +545,13 @@ ;; call with keyword args pre-sorted - original method code goes here ,(method-def-expr- mangled sparams - `((|::| ,mangled (call (core typeof) ,mangled)) ,@vars ,@restkw - ;; strip type off function self argument if not needed for a static param. - ;; then it is ok for cl-convert to move this definition above the original def. - ,@not-optional ,@vararg) + `((|::| ,mangled (call (core typeof) ,mangled)) ,@vars ,@restkw ,@not-optional ,@vararg) (insert-after-meta `(block ,@stmts) (cons `(meta nkw ,(+ (length vars) (length restkw))) - annotations)) + (if (has-thisfunction? `(block ,@stmts)) + (cons `(meta thisfunction-original ,(arg-name (car not-optional))) annotations) + annotations))) rett) ;; call with no keyword args @@ -1642,9 +1645,9 @@ (expand-forms ;; TODO: This behaviour (`const _:T = ...` does not call convert, ;; but still evaluates RHS) should be documented. - `(const ,(car e) ,(if (underscore-symbol? (car e)) - rhs - (convert-for-type-decl rhs T #t #f)))) + `(const (= ,(car e) ,(if (underscore-symbol? (car e)) + rhs + (convert-for-type-decl rhs T #t #f))))) (expand-forms `(block ,@(cdr e) ;; TODO: When x is a complex expression, this acts as a @@ -2519,7 +2522,7 @@ `(= ,lhs ,rhs))) (define (expand-forms e) - (if (or (atom? e) (memq (car e) '(quote inert top core globalref module toplevel ssavalue null true false meta using import export public thismodule toplevel-only))) + (if (or (atom? e) (memq (car e) '(quote inert top core globalref module toplevel ssavalue null true false meta export public thismodule toplevel-only))) e (let ((ex (get expand-table (car e) #f))) (if ex @@ -2539,6 +2542,20 @@ (define (something e) (find (lambda (x) (not (equal? x '(null)))) e)) +(define (check-import-paths what e) + (define (check-dot-path e) + (and (list? e) (eq? (car e) '|.|) (every symbol? (cdr e)))) + (define (check-path e) + (and (pair? e) + (or (check-dot-path e) + (and (eq? (car e) 'as) + (check-dot-path (cadr e)) (symbol? (caddr e)))))) + (unless (and (list? e) + (or (every check-path e) + (and (list? (car e)) (eq? (caar e) ':) + (every check-path (cdar e))))) + (error (string "malformed \"" what "\" statement")))) + ;; table mapping expression head to a function expanding that form (define expand-table (table @@ -2583,11 +2600,13 @@ (typ-svec (caddr sig-svec)) (tvars (cddr (cadddr sig-svec))) (argtypes (cdddr typ-svec)) - (functionloc (cadr (caddddr sig-svec)))) - (let* ((argtype (foldl (lambda (var ex) `(call (core UnionAll) ,var ,ex)) - (expand-forms `(curly (core Tuple) ,@argtypes)) - (reverse tvars)))) - `(_opaque_closure ,(or argt argtype) ,rt_lb ,rt_ub ,isva ,(length argtypes) ,allow-partial ,functionloc ,lam)))) + (functionloc (cadr (caddddr sig-svec))) + (argtype (foldl (lambda (var ex) `(call (core UnionAll) ,var ,ex)) + (expand-forms `(curly (core Tuple) ,@argtypes)) + (reverse tvars))) + (argtype (or argt argtype)) + (argtype (if (null? stmts) argtype `(block ,@stmts ,argtype)))) + `(_opaque_closure ,argtype ,rt_lb ,rt_ub ,isva ,(length argtypes) ,allow-partial ,functionloc ,lam))) 'block (lambda (e) @@ -2894,6 +2913,7 @@ 'generator (lambda (e) (check-no-return e) + (check-no-thisfunction e) (expand-generator e #f '())) 'flatten @@ -2937,6 +2957,38 @@ (lambda (e) (set! *current-desugar-loc* e) e) + + ;; We insert (latestworld) after every call to _eval_import or _eval_using + ;; to avoid having to do it in eval_import_path (#57316) + 'import + (lambda (e) + (check-import-paths "import" (cdr e)) + `(block + (toplevel-only import) + ,.(if (eq? (caadr e) ':) + `((call (top _eval_import) (true) (thismodule) + ,.(map (lambda (x) `(inert ,x)) (cdadr e))) + (latestworld)) + (map (lambda (x) + `(block + (call (top _eval_import) (true) (thismodule) (null) (inert ,x)) + (latestworld))) + (cdr e))))) + + 'using + (lambda (e) + (check-import-paths "using" (cdr e)) + `(block + (toplevel-only using) + ,.(if (eq? (caadr e) ':) + `((call (top _eval_import) (false) (thismodule) + ,.(map (lambda (x) `(inert ,x)) (cdadr e))) + (latestworld)) + (map (lambda (x) + `(block + (call (top _eval_using) (thismodule) (inert ,x)) + (latestworld))) + (cdr e))))) )) (define (has-return? e) @@ -2946,6 +2998,13 @@ (if (has-return? e) (error "\"return\" not allowed inside comprehension or generator"))) +(define (has-thisfunction? e) + (expr-contains-p thisfunction? e (lambda (x) (not (function-def? x))))) + +(define (check-no-thisfunction e) + (if (has-thisfunction? e) + (error "\"@__FUNCTION__\" not allowed inside comprehension or generator"))) + (define (has-break-or-continue? e) (expr-contains-p (lambda (x) (and (pair? x) (memq (car x) '(break continue)))) e @@ -2954,6 +3013,7 @@ (define (lower-comprehension ty expr itrs) (check-no-return expr) + (check-no-thisfunction expr) (if (has-break-or-continue? expr) (error "break or continue outside loop")) (let ((result (make-ssavalue)) @@ -3181,7 +3241,7 @@ (check-valid-name (cadr e)) ;; remove local decls '(null)) - ((memq (car e) '(using import export public)) + ((memq (car e) '(export public)) ;; no scope resolution - identifiers remain raw symbols e) ((eq? (car e) 'require-existing-local) @@ -3357,11 +3417,11 @@ (define (lambda-all-vars e) (append (lam:argnames e) (caddr e))) -;; compute set of variables referenced in a lambda but not bound by it +;; compute set of non-global variables referenced in a lambda but not bound by it (define (free-vars- e tab) (cond ((or (eq? e UNUSED) (underscore-symbol? e)) tab) ((symbol? e) (put! tab e #t)) - ((and (pair? e) (eq? (car e) 'globalref)) tab) + ((and (pair? e) (memq (car e) '(global globalref))) tab) ((and (pair? e) (eq? (car e) 'break-block)) (free-vars- (caddr e) tab)) ((and (pair? e) (eq? (car e) 'with-static-parameters)) (free-vars- (cadr e) tab)) ((or (atom? e) (quoted? e)) tab) @@ -3385,9 +3445,15 @@ vi) tab)) +;; env: list of vinfo (should not include globals) +;; captvars: list of vinfo +;; sp: list of symbol +;; new-sp: list of symbol (static params declared here) +;; methsig: `(call (core svec) ...) +;; tab: table of (name . var-info) (define (analyze-vars-lambda e env captvars sp new-sp methsig tab) (let* ((args (lam:args e)) - (locl (caddr e)) + (locl (lam:vinfo e)) (allv (nconc (map arg-name args) locl)) (fv (let* ((fv (diff (free-vars (lam:body e)) allv)) ;; add variables referenced in declared types for free vars @@ -3397,27 +3463,23 @@ fv)))) (append (diff dv fv) fv))) (sig-fv (if methsig (free-vars methsig) '())) - (glo (find-global-decls (lam:body e))) ;; make var-info records for vars introduced by this lambda (vi (nconc (map (lambda (decl) (make-var-info (decl-var decl))) args) (map make-var-info locl))) - (capt-sp (filter (lambda (v) (or (and (memq v fv) (not (memq v glo)) (not (memq v new-sp))) + (capt-sp (filter (lambda (v) (or (and (memq v fv) (not (memq v new-sp))) (memq v sig-fv))) sp)) ;; captured vars: vars from the environment that occur ;; in our set of free variables (fv). (cv (append (filter (lambda (v) (and (memq (vinfo:name v) fv) - (not (memq (vinfo:name v) new-sp)) - (not (memq (vinfo:name v) glo)))) + (not (memq (vinfo:name v) new-sp)))) env) (map make-var-info capt-sp))) (new-env (append vi ;; new environment: add our vars - (filter (lambda (v) - (and (not (memq (vinfo:name v) allv)) - (not (memq (vinfo:name v) glo)))) + (filter (lambda (v) (not (memq (vinfo:name v) allv))) env)))) (analyze-vars (lam:body e) new-env @@ -3440,10 +3502,15 @@ (define (analyze-vars e env captvars sp tab) (if (or (atom? e) (quoted? e)) (begin - (if (symbol? e) - (let ((vi (get tab e #f))) - (if vi - (vinfo:set-read! vi #t)))) + (cond + ((symbol? e) + (let ((vi (get tab e #f))) + (if vi + (vinfo:set-read! vi #t)))) + ((nospecialize-meta? e) + (let ((vi (get tab (caddr e) #f))) + (if vi + (vinfo:set-nospecialize! vi #t))))) e) (case (car e) ((local-def) ;; a local that we know has an assignment that dominates all usages @@ -3461,6 +3528,11 @@ (let ((vi (get tab (cadr e) #f))) (if vi (vinfo:set-called! vi #t)) + ;; calls f(x...) go through `_apply_iterate` + (if (and (length> e 3) (equal? (cadr e) '(core _apply_iterate))) + (let ((vi2 (get tab (cadddr e) #f))) + (if vi2 + (vinfo:set-called! vi2 #t)))) ;; calls to functions with keyword args have head of `kwcall` first (if (and (length> e 3) (equal? (cadr e) '(core kwcall))) (let ((vi2 (get tab (cadddr e) #f))) @@ -3537,21 +3609,6 @@ f(x) = yt(x) (call (core _typebody!) (false) ,s (call (core svec) ,@types)) (return (null))))))))) -(define (type-for-closure name fields super) - (let ((s (make-ssavalue))) - `((thunk ,(linearize `(lambda () - (() () 0 ()) - (block (global ,name) - (= ,s (call (core _structtype) (thismodule) (inert ,name) (call (core svec)) - (call (core svec) ,@(map quotify fields)) - (call (core svec)) - (false) ,(length fields))) - (call (core _setsuper!) ,s ,super) - (const (globalref (thismodule) ,name) ,s) - (call (core _typebody!) (false) ,s - (call (core svec) ,@(map (lambda (v) '(core Box)) fields))) - (return (null))))))))) - ;; better versions of above, but they get handled wrong in many places ;; need to fix that in order to handle #265 fully (and use the definitions) @@ -3809,9 +3866,9 @@ f(x) = yt(x) (Set '(quote top core lineinfo line inert local-def unnecessary copyast meta inbounds boundscheck loopinfo decl aliasscope popaliasscope thunk with-static-parameters toplevel-only - global globalref global-if-global assign-const-if-global isglobal thismodule + global globalref global-if-global assign-const-if-global isglobal thismodule thisfunction const atomic null true false ssavalue isdefined toplevel module lambda - error gc_preserve_begin gc_preserve_end import using export public inline noinline purity))) + error gc_preserve_begin gc_preserve_end export public inline noinline purity))) (define (local-in? s lam (tab #f)) (or (and tab (has? tab s)) @@ -3965,6 +4022,10 @@ f(x) = yt(x) (let ((cv (assq v (cadr (lam:vinfo lam))))) (and cv (vinfo:asgn cv) (vinfo:capt cv))))) +(define (is-var-nospecialize? v lam) + (let ((vi (assq v (car (lam:vinfo lam))))) + (and vi (vinfo:nospecialize vi)))) + (define (toplevel-preserving? e) (and (pair? e) (memq (car e) '(if elseif block trycatch tryfinally trycatchelse)))) @@ -4043,7 +4104,7 @@ f(x) = yt(x) ((atom? e) e) (else (case (car e) - ((quote top core globalref thismodule lineinfo line break inert module toplevel null true false meta import using) e) + ((quote top core global globalref thismodule thisfunction lineinfo line break inert module toplevel null true false meta) e) ((toplevel-only) ;; hack to avoid generating a (method x) expr for struct types (if (eq? (cadr e) 'struct) @@ -4107,6 +4168,7 @@ f(x) = yt(x) (capt-var-access v fname opaq) v))) cvs))) + (set-car! (cdddr (lam:vinfo lam2)) '()) ;; must capture static_parameters as values inside opaque_closure `(new_opaque_closure ,(cadr e) ,(or (caddr e) '(call (core apply_type) (core Union))) ,(or (cadddr e) '(core Any)) ,allow-partial (opaque_closure_method (null) ,nargs ,isva ,functionloc ,(convert-lambda lam2 (car (lam:args lam2)) #f '() (symbol-to-idx-map cvs) parsed-method-stack)) @@ -4124,6 +4186,7 @@ f(x) = yt(x) '() (map-cl-convert (butlast (cdr sig)) fname lam namemap defined toplevel interp opaq parsed-method-stack globals locals))) + (r (make-ssavalue)) (sig (and sig (if (eq? (car sig) 'block) (last sig) sig)))) @@ -4147,7 +4210,7 @@ f(x) = yt(x) ((null? cvs) `(block ,@sp-inits - (method ,(cadr e) ,(cl-convert + (= ,r (method ,(cadr e) ,(cl-convert ;; anonymous functions with keyword args generate global ;; functions that refer to the type of a local function (rename-sig-types sig namemap) @@ -4159,17 +4222,19 @@ f(x) = yt(x) `(lambda ,(cadr lam2) (,(clear-capture-bits (car vis)) ,@(cdr vis)) - ,body))) - (latestworld))) + ,body)))) + (latestworld) + ,r)) (else (let* ((exprs (lift-toplevel (convert-lambda lam2 '|#anon| #t '() #f parsed-method-stack))) (top-stmts (cdr exprs)) (newlam (compact-and-renumber (linearize (car exprs)) 'none 0))) `(toplevel-butfirst (block ,@sp-inits - (method ,(cadr e) ,(cl-convert sig fname lam namemap defined toplevel interp opaq parsed-method-stack globals locals) - ,(julia-bq-macro newlam)) - (latestworld)) + (= ,r (method ,(cadr e) ,(cl-convert sig fname lam namemap defined toplevel interp opaq parsed-method-stack globals locals) + ,(julia-bq-macro newlam))) + (latestworld) + ,r) ,@top-stmts)))) ;; local case - lift to a new type at top level @@ -4252,16 +4317,14 @@ f(x) = yt(x) (closure-param-syms (map (lambda (s) (make-ssavalue)) closure-param-names)) (typedef ;; expression to define the type (let* ((fieldtypes (map (lambda (v) - (if (is-var-boxed? v lam) - '(core Box) - (make-ssavalue))) + (cond ((is-var-boxed? v lam) '(core Box)) + ((is-var-nospecialize? v lam) (vinfo:type (assq v (car (lam:vinfo lam))))) + (else (make-ssavalue)))) capt-vars)) (para (append closure-param-syms (filter ssavalue? fieldtypes))) (fieldnames (append closure-param-names (filter (lambda (v) (not (is-var-boxed? v lam))) capt-vars)))) - (if (null? para) - (type-for-closure type-name capt-vars '(core Function)) - (type-for-closure-parameterized type-name para fieldnames capt-vars fieldtypes '(core Function))))) + (type-for-closure-parameterized type-name para fieldnames capt-vars fieldtypes '(core Function)))) (mk-method ;; expression to make the method (if short '() (let* ((iskw ;; TODO jb/functions need more robust version of this @@ -4291,7 +4354,7 @@ f(x) = yt(x) (P (append closure-param-names (filter identity (map (lambda (v ve) - (if (is-var-boxed? v lam) + (if (or (is-var-boxed? v lam) (is-var-nospecialize? v lam)) #f `(call (core _typeof_captured_variable) ,ve))) capt-vars var-exprs))))) @@ -4966,7 +5029,7 @@ f(x) = yt(x) (if value (error "misplaced \"global\" declaration")) (if (or (length= e 2) (atom? (caddr e))) (emit e) (let ((rr (make-ssavalue))) - (emit `(= ,rr ,(caddr e))) + (emit `(= ,rr ,(compile (caddr e) break-labels #t #f))) (emit `(globaldecl ,(cadr e) ,rr)))) (if (null? (cadr lam)) (emit `(latestworld)))) @@ -4996,8 +5059,10 @@ f(x) = yt(x) (let ((l (make-ssavalue))) (emit `(= ,l ,(compile lam break-labels #t #f))) l)))) - (emit `(method ,(or (cadr e) '(false)) ,sig ,lam)) - (if value (compile '(null) break-labels value tail))) + (let ((val (make-ssavalue))) + (emit `(= ,val (method ,(or (cadr e) '(false)) ,sig ,lam))) + (if tail (emit-return tail val)) + val)) (cond (tail (emit-return tail e)) (value e) (else (emit e))))) @@ -5035,7 +5100,7 @@ f(x) = yt(x) '(null)) ;; other top level expressions - ((import using export public latestworld) + ((export public latestworld) (check-top-level e) (if (not (eq? (car e) 'latestworld)) (emit e)) @@ -5079,6 +5144,30 @@ f(x) = yt(x) ((error) (error (cadr e))) + + ;; thisfunction replaced with first argument name + ((thisfunction) + (let ((first-arg (and (pair? (lam:args lam)) (car (lam:args lam))))) + (if first-arg + (let* ((arg-name (arg-name first-arg)) + ;; Check for thisfunction-original metadata in keyword wrapper functions + (original-name (let ((body (lam:body lam))) + (and (pair? body) (pair? (cdr body)) + (let loop ((stmts (cdr body))) + (if (pair? stmts) + (let ((stmt (car stmts))) + (if (and (pair? stmt) (eq? (car stmt) 'meta) + (pair? (cdr stmt)) (eq? (cadr stmt) 'thisfunction-original) + (pair? (cddr stmt))) + (caddr stmt) + (loop (cdr stmts)))) + #f))))) + (final-name (or original-name arg-name))) + (cond (tail (emit-return tail final-name)) + (value final-name) + (else (emit final-name) #f))) + (error "\"@__FUNCTION__\" can only be used inside a function")))) + (else (error (string "invalid syntax " (deparse e))))))) ;; introduce new slots for assigned arguments @@ -5176,6 +5265,14 @@ f(x) = yt(x) (define (set-lineno! lineinfo num) (set-car! (cddr lineinfo) num)) +;; note that the 'list and 'block atoms make all lists 1-indexed. +;; returns a 5-element vector containing: +;; code: `(block ,@(n expressions)) +;; locs: list of line-table index, where code[i] has lineinfo line-table[locs[i]] +;; line-table: list of `(lineinfo file.jl 123 0)' +;; ssavalue-table: table of (ssa-num . code-index) +;; where ssavalue references in `code` need this remapping +;; label-table: table of (label . code-index) (define (compact-ir body file line) (let ((code '(block)) (locs '(list)) @@ -5278,11 +5375,11 @@ f(x) = yt(x) ((nospecialize-meta? e) ;; convert nospecialize vars to slot numbers `(meta ,(cadr e) ,@(map renumber-stuff (cddr e)))) - ((or (atom? e) (quoted? e) (memq (car e) '(using import export public global toplevel))) + ((or (atom? e) (quoted? e) (memq (car e) '(export public global toplevel))) e) ((ssavalue? e) (let ((idx (get ssavalue-table (cadr e) #f))) - (if (not idx) (begin (prn e) (prn lam) (error "ssavalue with no def"))) + (if (not idx) (error "internal bug: ssavalue with no def")) `(ssavalue ,idx))) ((eq? (car e) 'goto) `(goto ,(get label-table (cadr e)))) @@ -5321,7 +5418,7 @@ f(x) = yt(x) ;; expander entry point -(define (julia-expand1 ex file line) +(define (julia-lower1 ex file line) (compact-and-renumber (linearize (closure-convert @@ -5330,7 +5427,7 @@ f(x) = yt(x) (define *current-desugar-loc* #f) -(define (julia-expand0 ex lno) +(define (julia-lower0 ex lno) (with-bindings ((*current-desugar-loc* lno)) (trycatch (expand-forms ex) (lambda (e) @@ -5343,7 +5440,7 @@ f(x) = yt(x) (error (string (cadr e) (format-loc *current-desugar-loc*)))) (raise e))))) -(define (julia-expand ex (file 'none) (line 0)) - (julia-expand1 - (julia-expand0 +(define (julia-lower ex (file 'none) (line 0)) + (julia-lower1 + (julia-lower0 (julia-expand-macroscope ex) `(line ,line ,file)) file line)) diff --git a/src/julia.expmap.in b/src/julia.expmap.in index b28a714e75f69..5a3fbce0d1a82 100644 --- a/src/julia.expmap.in +++ b/src/julia.expmap.in @@ -30,7 +30,6 @@ _Z22jl_coverage_alloc_lineN4llvm9StringRefEi*; _Z22jl_malloc_data_pointerN4llvm9StringRefEi*; _jl_timing_*; - LLVMExtra*; JLJIT*; llvmGetPassPluginInfo*; diff --git a/src/julia.h b/src/julia.h index f664655cb9131..abe9b08c677d9 100644 --- a/src/julia.h +++ b/src/julia.h @@ -208,6 +208,7 @@ typedef struct _jl_datatype_t jl_tupletype_t; struct _jl_code_instance_t; typedef struct _jl_method_instance_t jl_method_instance_t; typedef struct _jl_globalref_t jl_globalref_t; +typedef struct _jl_typemap_entry_t jl_typemap_entry_t; // TypeMap is an implicitly defined type @@ -243,15 +244,6 @@ JL_DLLEXPORT extern const jl_callptr_t jl_f_opaque_closure_call_addr; JL_DLLEXPORT extern const jl_callptr_t jl_fptr_wait_for_compiled_addr; -typedef struct _jl_line_info_node_t { - JL_DATA_TYPE - struct _jl_module_t *module; - jl_value_t *method; // may contain a jl_symbol, jl_method_t, or jl_method_instance_t - jl_sym_t *file; - int32_t line; - int32_t inlined_at; -} jl_line_info_node_t; - struct jl_codeloc_t { int32_t line; int32_t to; @@ -336,14 +328,20 @@ typedef struct _jl_code_info_t { // This type describes a single method definition, and stores data // shared by the specializations of a function. +// +// Reading or writing requires `writelock` or exclusive ownership: +// roots, root_blocks, nroots_sysimg, ccallable +// No lock is required to read these fields, set once on construction: +// all other fields typedef struct _jl_method_t { JL_DATA_TYPE jl_sym_t *name; // for error reporting struct _jl_module_t *module; jl_sym_t *file; int32_t line; + _Atomic(uint8_t) dispatch_status; // bits defined in staticdata.jl + _Atomic(jl_genericmemory_t*) interferences; // set of intersecting methods not more specific _Atomic(size_t) primary_world; - _Atomic(size_t) deleted_world; // method's type signature. redundant with TypeMapEntry->specTypes jl_value_t *sig; @@ -387,6 +385,7 @@ typedef struct _jl_method_t { uint8_t nospecializeinfer; // bit flags, 0x01 = scanned // 0x02 = added to module scanned list (either from scanning or inference edge) + // 0x04 = Source was invalidated since jl_require_world _Atomic(uint8_t) did_scan_source; // uint8 settings @@ -399,13 +398,19 @@ typedef struct _jl_method_t { _jl_purity_overrides_t purity; // hidden fields: - // lock for modifications to the method jl_mutex_t writelock; } jl_method_t; // This type is a placeholder to cache data for a specType signature specialization of a Method // can can be used as a unique dictionary key representation of a call to a particular Method // with a particular set of argument types +// +// Reading or writing requires `def.method->writelock` or exclusive ownership: +// backedges +// Reading or writing requires the associated jl_methcache_t's `writelock`: +// cache_with_orig +// No lock is required to read these fields, set once on construction: +// def, specTypes, sparam_vals struct _jl_method_instance_t { JL_NON_MOVING // Non moving, as it is referenced in a map in JITDebugInfoRegistry JL_DATA_TYPE @@ -416,14 +421,18 @@ struct _jl_method_instance_t { } def; // pointer back to the context for this code jl_value_t *specTypes; // argument types this was specialized for jl_svec_t *sparam_vals; // static parameter values, indexed by def.method->sig - jl_array_t *backedges; // list of code-instances which call this method-instance; `invoke` records (invokesig, caller) pairs + // list of code-instances which call this method-instance; `invoke` records (invokesig, caller) pairs + jl_array_t *backedges; _Atomic(struct _jl_code_instance_t*) cache; uint8_t cache_with_orig; // !cache_with_specTypes // flags for this method instance // bit 0: generated by an explicit `precompile(...)` // bit 1: dispatched + // bit 2: The ->backedges field is currently being walked higher up the stack - entries may be deleted, but not moved + // bit 3: The ->backedges field was modified and should be compacted when clearing bit 2 _Atomic(uint8_t) flags; + _Atomic(uint8_t) dispatch_status; // bits defined in staticdata.jl }; #define JL_MI_FLAGS_MASK_PRECOMPILED 0x01 #define JL_MI_FLAGS_MASK_DISPATCHED 0x02 @@ -439,6 +448,11 @@ typedef struct _jl_opaque_closure_t { } jl_opaque_closure_t; // This type represents an executable operation +// +// No lock is required to read these fields, which are set while we have +// exclusive ownership of the CodeInstance: +// def, owner, rettype, exctype, rettype_const, analysis_results, +// time_infer_total, time_infer_self typedef struct _jl_code_instance_t { JL_NON_MOVING // Pin codeinst, as they are referenced by vectors and maps in _jl_codegen_params_t JL_DATA_TYPE @@ -456,10 +470,11 @@ typedef struct _jl_code_instance_t { jl_value_t *rettype_const; // inferred constant return value, or null // Inferred result. When part of the runtime cache, either - // - A jl_code_info_t (may be compressed) containing the inferred IR + // - A jl_code_info_t (may be compressed as a String) containing the inferred IR // - jl_nothing, indicating that inference was completed, but the result was // deleted to save space. - // - null, indicating that inference was not yet completed or did not succeed + // - UInt8, indicating that inference recorded the estimated inlining cost, but deleted the result to save space + // - NULL, indicating that inference was not yet completed or did not succeed _Atomic(jl_value_t *) inferred; _Atomic(jl_debuginfo_t *) debuginfo; // stored information about edges from this object (set once, with a happens-before both source and invoke) _Atomic(jl_svec_t *) edges; // forward edge info @@ -508,17 +523,17 @@ typedef struct _jl_abi_override_t { typedef struct { JL_DATA_TYPE - jl_sym_t *name; - jl_value_t *lb; // lower bound - jl_value_t *ub; // upper bound + jl_sym_t *JL_NONNULL name; + jl_value_t *JL_NONNULL lb; // lower bound + jl_value_t *JL_NONNULL ub; // upper bound } jl_tvar_t; // UnionAll type (iterated union over all values of a variable in certain bounds) // written `body where lb<:var<:ub` typedef struct { JL_DATA_TYPE - jl_tvar_t *var; - jl_value_t *body; + jl_tvar_t *JL_NONNULL var; + jl_value_t *JL_NONNULL body; } jl_unionall_t; // represents the "name" part of a DataType, describing the syntactic structure @@ -529,6 +544,7 @@ typedef struct { JL_DATA_TYPE jl_sym_t *name; struct _jl_module_t *module; + jl_sym_t *singletonname; // sometimes used for debug printing jl_svec_t *names; // field names const uint32_t *atomicfields; // if any fields are atomic, we record them here const uint32_t *constfields; // if any fields are const, we record them here @@ -538,15 +554,16 @@ typedef struct { _Atomic(jl_value_t*) Typeofwrapper; // cache for Type{wrapper} _Atomic(jl_svec_t*) cache; // sorted array _Atomic(jl_svec_t*) linearcache; // unsorted array - struct _jl_methtable_t *mt; jl_array_t *partial; // incomplete instantiations of this type intptr_t hash; + _Atomic(int32_t) max_args; // max # of non-vararg arguments in a signature with this type as the function int32_t n_uninitialized; // type properties uint8_t abstract:1; uint8_t mutabl:1; uint8_t mayinlinealloc:1; - uint8_t _reserved:5; + uint8_t _unused:5; + _Atomic(uint8_t) cache_entry_count; // (approximate counter of TypeMapEntry for heuristics) uint8_t max_methods; // override for inference's max_methods setting (0 = no additional limit or relaxation) uint8_t constprop_heustic; // override for inference's constprop heuristic const uint32_t *hiddenptrfields; // if any fields are hidden pointers, we record them here @@ -554,8 +571,8 @@ typedef struct { typedef struct { JL_DATA_TYPE - jl_value_t *a; - jl_value_t *b; + jl_value_t *JL_NONNULL a; + jl_value_t *JL_NONNULL b; } jl_uniontype_t; // in little-endian, isptr is always the first bit, avoiding the need for a branch in computing isptr @@ -591,10 +608,12 @@ typedef struct { // metadata bit only for GenericMemory eltype layout uint16_t arrayelem_isboxed : 1; uint16_t arrayelem_isunion : 1; + uint16_t arrayelem_isatomic : 1; + uint16_t arrayelem_islocked : 1; // If set, this type's egality can be determined entirely by comparing // the non-padding bits of this datatype. uint16_t isbitsegal : 1; - uint16_t padding : 10; + uint16_t padding : 8; } flags; // union { // jl_fielddesc8_t field8[nfields]; @@ -780,7 +799,7 @@ typedef struct JL_ALIGNED_ATTR(8) _jl_binding_partition_t { * } restriction; */ jl_value_t *restriction; - size_t min_world; + _Atomic(size_t) min_world; _Atomic(size_t) max_world; _Atomic(struct _jl_binding_partition_t *) next; size_t kind; @@ -815,6 +834,15 @@ typedef struct { uint64_t lo; } jl_uuid_t; +// Reading or writing requires `lock`: +// scanned_methods, usings +// Reading or writing requires `Base.require_lock`: +// uuid +// Reading or writing requires `world_counter_lock`: +// usings_backedges (TODO) +// No lock is required to read these fields, set once on construction: +// name, parent, file, line, build_id, uuid, nospecialize, optlevel, compile, +// infer, iistopmod, max_methods typedef struct _jl_module_t { JL_NON_MOVING // modules are referenced in jl_current_modules (htable). They cannot move. JL_DATA_TYPE @@ -857,7 +885,7 @@ struct _jl_globalref_t { }; // one Type-to-Value entry -typedef struct _jl_typemap_entry_t { +struct _jl_typemap_entry_t { JL_DATA_TYPE _Atomic(struct _jl_typemap_entry_t*) next; // invasive linked list jl_tupletype_t *sig; // the type signature for this entry @@ -874,7 +902,7 @@ typedef struct _jl_typemap_entry_t { int8_t isleafsig; // isleaftype(sig) & !any(isType, sig) : unsorted and very fast int8_t issimplesig; // all(isleaftype | isAny | isType | isVararg, sig) : sorted and fast int8_t va; // isVararg(sig) -} jl_typemap_entry_t; +}; // one level in a TypeMap tree (each level splits on a type at a given offset) typedef struct _jl_typemap_level_t { @@ -894,19 +922,21 @@ typedef struct _jl_typemap_level_t { _Atomic(jl_typemap_t*) any; } jl_typemap_level_t; -// contains the TypeMap for one Type -typedef struct _jl_methtable_t { +typedef struct _jl_methcache_t { JL_DATA_TYPE - jl_sym_t *name; // sometimes used for debug printing - _Atomic(jl_typemap_t*) defs; _Atomic(jl_genericmemory_t*) leafcache; _Atomic(jl_typemap_t*) cache; - _Atomic(intptr_t) max_args; // max # of non-vararg arguments in a signature - jl_module_t *module; // sometimes used for debug printing - jl_array_t *backedges; // (sig, caller::CodeInstance) pairs jl_mutex_t writelock; - uint8_t offs; // 0, or 1 to skip splitting typemap on first (function) argument - uint8_t frozen; // whether this accepts adding new methods +} jl_methcache_t; + +// contains global MethodTable +typedef struct _jl_methtable_t { + JL_DATA_TYPE + _Atomic(jl_typemap_t*) defs; + jl_methcache_t *cache; + jl_sym_t *name; // sometimes used for debug printing + jl_module_t *module; // sometimes used for debug printing + jl_genericmemory_t *backedges; // IdDict{top typenames, Vector{uncovered (sig => caller::CodeInstance)}} } jl_methtable_t; typedef struct { @@ -1129,16 +1159,17 @@ extern JL_DLLIMPORT jl_datatype_t *jl_upsilonnode_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_quotenode_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_newvarnode_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_intrinsic_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_methcache_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_methtable_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_typemap_level_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_typemap_entry_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_kwcall_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_svec_t *jl_emptysvec JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_emptytuple JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_true JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_false JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_nothing JL_GLOBALLY_ROOTED; -extern JL_DLLIMPORT jl_value_t *jl_kwcall_func JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_libdl_dlopen_func JL_GLOBALLY_ROOTED; @@ -1603,9 +1634,7 @@ STATIC_INLINE void jl_array_uint32_set(void *a, size_t i, uint32_t x) JL_NOTSAFE #define jl_string_data(s) ((char*)s + sizeof(void*)) #define jl_string_len(s) (*(size_t*)s) -#define jl_gf_ft_mtable(ft) (((jl_datatype_t*)ft)->name->mt) -#define jl_gf_mtable(f) (jl_gf_ft_mtable(jl_typeof(f))) -#define jl_gf_name(f) (jl_gf_mtable(f)->name) +#define jl_gf_name(f) (((jl_datatype_t*)jl_typeof(f))->name->singletonname) // struct type info JL_DLLEXPORT jl_svec_t *jl_compute_fieldtypes(jl_datatype_t *st JL_PROPAGATES_ROOT, void *stack, int cacheable); @@ -1634,7 +1663,7 @@ JL_DLLEXPORT jl_value_t *jl_unwrap_unionall(jl_value_t *v JL_PROPAGATES_ROOT) JL #define jl_inlinedatatype_layout(t) (((jl_datatype_t*)t)->layout) STATIC_INLINE const jl_datatype_layout_t *jl_datatype_layout(jl_datatype_t *t) JL_NOTSAFEPOINT { - if (jl_is_layout_opaque(t->layout)) // e.g. GenericMemory + if (t->layout == NULL || jl_is_layout_opaque(t->layout)) // e.g. GenericMemory t = (jl_datatype_t*)jl_unwrap_unionall(t->name->wrapper); return t->layout; } @@ -1798,7 +1827,7 @@ static inline int jl_field_ishiddenptr(jl_datatype_t *st, int i) JL_NOTSAFEPOINT #define jl_is_mutable(t) (((jl_datatype_t*)t)->name->mutabl) #define jl_is_mutable_datatype(t) (jl_is_datatype(t) && (((jl_datatype_t*)t)->name->mutabl)) #define jl_is_immutable(t) (!((jl_datatype_t*)t)->name->mutabl) -#define jl_is_immutable_datatype(t) (jl_is_datatype(t) && (!((jl_datatype_t*)t)->name->mutabl)) +#define jl_may_be_immutable_datatype(t) (jl_is_datatype(t) && (!((jl_datatype_t*)t)->name->mutabl)) #define jl_is_uniontype(v) jl_typetagis(v,jl_uniontype_tag<<4) #define jl_is_typevar(v) jl_typetagis(v,jl_tvar_tag<<4) #define jl_is_unionall(v) jl_typetagis(v,jl_unionall_tag<<4) @@ -1839,6 +1868,7 @@ static inline int jl_field_ishiddenptr(jl_datatype_t *st, int i) JL_NOTSAFEPOINT #define jl_is_method(v) jl_typetagis(v,jl_method_type) #define jl_is_module(v) jl_typetagis(v,jl_module_tag<<4) #define jl_is_mtable(v) jl_typetagis(v,jl_methtable_type) +#define jl_is_mcache(v) jl_typetagis(v,jl_methcache_type) #define jl_is_task(v) jl_typetagis(v,jl_task_tag<<4) #define jl_is_string(v) jl_typetagis(v,jl_string_tag<<4) #define jl_is_cpointer(v) jl_is_cpointer_type(jl_typeof(v)) @@ -1849,6 +1879,8 @@ static inline int jl_field_ishiddenptr(jl_datatype_t *st, int i) JL_NOTSAFEPOINT #define jl_is_addrspacecore(v) jl_typetagis(v,jl_addrspacecore_type) #define jl_is_abioverride(v) jl_typetagis(v,jl_abioverride_type) #define jl_genericmemory_isbitsunion(a) (((jl_datatype_t*)jl_typetagof(a))->layout->flags.arrayelem_isunion) +#define jl_genericmemory_isatomic(a) (((jl_datatype_t*)jl_typetagof(a))->layout->flags.arrayelem_isatomic) +#define jl_genericmemory_islocked(a) (((jl_datatype_t*)jl_typetagof(a))->layout->flags.arrayelem_islocked) #define jl_is_array_any(v) jl_typetagis(v,jl_array_any_type) JL_DLLEXPORT int jl_subtype(jl_value_t *a, jl_value_t *b); @@ -2133,9 +2165,9 @@ JL_DLLEXPORT jl_sym_t *jl_tagged_gensym(const char *str, size_t len); JL_DLLEXPORT jl_sym_t *jl_get_root_symbol(void); JL_DLLEXPORT jl_value_t *jl_get_binding_value(jl_binding_t *b JL_PROPAGATES_ROOT); JL_DLLEXPORT jl_value_t *jl_get_binding_value_in_world(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world); -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT); -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_and_const(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT); +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_resolved_debug_only(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_resolved_and_const_debug_only(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name); JL_DLLEXPORT jl_method_t *jl_method_def(jl_svec_t *argdata, jl_methtable_t *mt, jl_code_info_t *f, jl_module_t *module); JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo, size_t world, jl_code_instance_t **cache); @@ -2288,13 +2320,12 @@ JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b JL_PROPAGATES_ROOT, size_t new_world); JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import); -JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr); -JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr); JL_DLLEXPORT jl_value_t *jl_get_global(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT); +void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported); JL_DLLEXPORT void jl_checked_assignment(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_value_t *jl_checked_swap(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_value_t *jl_checked_replace(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *expected, jl_value_t *rhs); @@ -2302,15 +2333,12 @@ JL_DLLEXPORT jl_value_t *jl_checked_modify(jl_binding_t *b, jl_module_t *mod, jl JL_DLLEXPORT jl_value_t *jl_checked_assignonce(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *rhs JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val2(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, enum jl_partition_kind); +JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici); +JL_DLLEXPORT void jl_import_module(jl_task_t *ct, jl_module_t *m, jl_module_t *import, jl_sym_t *asname); JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from); -JL_DLLEXPORT void jl_module_use(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s); -JL_DLLEXPORT void jl_module_use_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); -JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s); -JL_DLLEXPORT void jl_module_import_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname); -JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported); +int jl_module_public_(jl_module_t *from, jl_sym_t *s, int exported, size_t new_world); JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *s); JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var); -JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m); // eq hash tables JL_DLLEXPORT jl_genericmemory_t *jl_eqtable_put(jl_genericmemory_t *h JL_ROOTING_ARGUMENT, jl_value_t *key, jl_value_t *val JL_ROOTED_ARGUMENT, int *inserted); @@ -2417,10 +2445,10 @@ struct _jl_image_t; typedef struct _jl_image_t jl_image_t; JL_DLLIMPORT const char *jl_get_libdir(void); -JL_DLLEXPORT void julia_init(JL_IMAGE_SEARCH rel); JL_DLLEXPORT void jl_init(void); -JL_DLLEXPORT void jl_init_with_image(const char *julia_bindir, - const char *image_path); +JL_DLLEXPORT void jl_init_with_image_file(const char *julia_bindir, + const char *image_path); +JL_DLLEXPORT void jl_init_with_image_handle(void *handle); JL_DLLEXPORT const char *jl_get_default_sysimg_path(void); JL_DLLEXPORT int jl_is_initialized(void); JL_DLLEXPORT void jl_atexit_hook(int status); @@ -2449,16 +2477,9 @@ JL_DLLEXPORT jl_value_t *jl_parse_all(const char *text, size_t text_len, JL_DLLEXPORT jl_value_t *jl_parse_string(const char *text, size_t text_len, int offset, int greedy); // lowering -JL_DLLEXPORT jl_value_t *jl_expand(jl_value_t *expr, jl_module_t *inmodule); -JL_DLLEXPORT jl_value_t *jl_expand_with_loc(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line); -JL_DLLEXPORT jl_value_t *jl_expand_with_loc_warn(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line); -JL_DLLEXPORT jl_value_t *jl_expand_in_world(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line, size_t world); -JL_DLLEXPORT jl_value_t *jl_expand_stmt(jl_value_t *expr, jl_module_t *inmodule); -JL_DLLEXPORT jl_value_t *jl_expand_stmt_with_loc(jl_value_t *expr, jl_module_t *inmodule, - const char *file, int line); +JL_DLLEXPORT jl_value_t *jl_lower(jl_value_t *expr, jl_module_t *inmodule, + const char *file, int line, size_t world, + bool_t warn); // deprecated; use jl_parse_all JL_DLLEXPORT jl_value_t *jl_parse_input_line(const char *text, size_t text_len, const char *filename, size_t filename_len); @@ -2519,6 +2540,8 @@ JL_DLLEXPORT jl_value_t *jl_uncompress_argname_n(jl_value_t *syms, size_t i); JL_DLLEXPORT struct jl_codeloc_t jl_uncompress1_codeloc(jl_value_t *cl, size_t pc) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_compress_codelocs(int32_t firstline, jl_value_t *codelocs, size_t nstmts); JL_DLLEXPORT jl_value_t *jl_uncompress_codelocs(jl_value_t *cl, size_t nstmts); +JL_DLLEXPORT uint8_t jl_encode_inlining_cost(uint16_t inlining_cost) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint16_t jl_decode_inlining_cost(uint8_t inlining_cost) JL_NOTSAFEPOINT; JL_DLLEXPORT int jl_is_operator(const char *sym); JL_DLLEXPORT int jl_is_unary_operator(const char *sym); @@ -2791,7 +2814,7 @@ uint64_t parse_heap_size_option(const char *optarg, const char *option_name, int // Set julia-level ARGS array according to the arguments provided in // argc/argv -JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv); +JL_DLLEXPORT jl_value_t *jl_set_ARGS(int argc, char **argv); JL_DLLEXPORT int jl_generating_output(void) JL_NOTSAFEPOINT; @@ -2897,12 +2920,7 @@ JL_DLLEXPORT jl_task_t *jl_get_current_task(void) JL_GLOBALLY_ROOTED JL_NOTSAFEP STATIC_INLINE jl_function_t *jl_get_function(jl_module_t *m, const char *name) { - jl_task_t *ct = jl_get_current_task(); - size_t last_world = ct->world_age; - ct->world_age = jl_get_world_counter(); - jl_value_t *r = jl_get_global(m, jl_symbol(name)); - ct->world_age = last_world; - return (jl_function_t*)r; + return (jl_function_t*)jl_get_global(m, jl_symbol(name)); } // TODO: we need to pin the task while using this (set pure bit) diff --git a/src/julia_atomics.h b/src/julia_atomics.h index d05f0fafab28f..1d8fba3b44e33 100644 --- a/src/julia_atomics.h +++ b/src/julia_atomics.h @@ -190,7 +190,7 @@ T jl_atomic_exchange_explicit(std::atomic *ptr, S desired, std::memory_order { return std::atomic_exchange_explicit(ptr, desired, order); } -#define jl_atomic_exchange_release(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_reease) +#define jl_atomic_exchange_release(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_release) #define jl_atomic_exchange_relaxed(ptr, val) jl_atomic_exchange_explicit(ptr, val, memory_order_relaxed) extern "C" { #else diff --git a/src/julia_internal.h b/src/julia_internal.h index bc9b2a679eb9e..c0b18888588bd 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -3,6 +3,7 @@ #ifndef JL_INTERNAL_H #define JL_INTERNAL_H +#include "dtypes.h" #include "options.h" #include "julia_assert.h" #include "julia_locks.h" @@ -20,6 +21,9 @@ #include #include +#define STR(x) #x +#define XSTR(x) STR(x) + #if !defined(_WIN32) #include #else @@ -95,12 +99,6 @@ static inline void msan_unpoison(const volatile void *a, size_t size) JL_NOTSAFE static inline void msan_allocated_memory(const volatile void *a, size_t size) JL_NOTSAFEPOINT {} static inline void msan_unpoison_string(const volatile char *a) JL_NOTSAFEPOINT {} #endif -#ifdef _COMPILER_TSAN_ENABLED_ -JL_DLLIMPORT void *__tsan_create_fiber(unsigned flags); -JL_DLLIMPORT void *__tsan_get_current_fiber(void); -JL_DLLIMPORT void __tsan_destroy_fiber(void *fiber); -JL_DLLIMPORT void __tsan_switch_to_fiber(void *fiber, unsigned flags); -#endif #ifndef _OS_WINDOWS_ #if defined(_CPU_ARM_) || defined(_CPU_PPC_) || defined(_CPU_WASM_) @@ -178,7 +176,16 @@ JL_DLLIMPORT void __tsan_switch_to_fiber(void *fiber, unsigned flags); #endif #endif +#if defined(HAVE_SSP) && defined(_OS_DARWIN_) +// On Darwin, this is provided by libSystem and imported +extern JL_DLLIMPORT uintptr_t __stack_chk_guard; +#elif defined(HAVE_SSP) +// Added by compiler runtime in final link - not DLLIMPORT +extern uintptr_t __stack_chk_guard; +#else +// The system doesn't have it - we define our own extern JL_DLLEXPORT uintptr_t __stack_chk_guard; +#endif // If this is detected in a backtrace of segfault, it means the functions // that use this value must be reworked into their async form with cb arg @@ -192,6 +199,9 @@ void JL_UV_LOCK(void); extern _Atomic(unsigned) _threadedregion; extern _Atomic(uint16_t) io_loop_tid; +JL_DLLEXPORT void jl_init_(jl_image_buf_t sysimage); +JL_DLLEXPORT void jl_enter_threaded_region(void); +JL_DLLEXPORT void jl_exit_threaded_region(void); int jl_running_under_rr(int recheck) JL_NOTSAFEPOINT; //-------------------------------------------------- @@ -203,10 +213,9 @@ JL_DLLEXPORT double jl_get_profile_peek_duration(void); JL_DLLEXPORT void jl_set_profile_peek_duration(double); JL_DLLEXPORT void jl_init_profile_lock(void); -JL_DLLEXPORT uintptr_t jl_lock_profile_rd_held(void) JL_NOTSAFEPOINT; -JL_DLLEXPORT void jl_lock_profile(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER; +JL_DLLEXPORT int jl_lock_profile(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER; JL_DLLEXPORT void jl_unlock_profile(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_LEAVE; -JL_DLLEXPORT void jl_lock_profile_wr(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER; +JL_DLLEXPORT int jl_lock_profile_wr(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_ENTER; JL_DLLEXPORT void jl_unlock_profile_wr(void) JL_NOTSAFEPOINT JL_NOTSAFEPOINT_LEAVE; void jl_with_stackwalk_lock(void (*f)(void*) JL_NOTSAFEPOINT, void *ctx) JL_NOTSAFEPOINT; @@ -381,10 +390,18 @@ static inline void memassign_safe(int hasptr, char *dst, const jl_value_t *src, #define GC_OLD_MARKED (GC_OLD | GC_MARKED) // reachable and old #define GC_IN_IMAGE 4 +// data structures for runtime codegen +typedef struct _jl_abi_t { + jl_pinned_ref(jl_value_t) sigt; + jl_pinned_ref(jl_value_t) rt; + size_t nargs; + int specsig; // bool + // OpaqueClosure Methods override the first argument of their signature + int is_opaque_closure; +} jl_abi_t; + // useful constants -extern JL_DLLIMPORT jl_methtable_t *jl_type_type_mt JL_GLOBALLY_ROOTED; -extern JL_DLLIMPORT jl_methtable_t *jl_nonfunction_mt JL_GLOBALLY_ROOTED; -extern jl_methtable_t *jl_kwcall_mt JL_GLOBALLY_ROOTED; +extern jl_methtable_t *jl_method_table JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_method_t *jl_opaque_closure_method JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT _Atomic(size_t) jl_world_counter; extern jl_debuginfo_t *jl_nulldebuginfo JL_GLOBALLY_ROOTED; @@ -407,8 +424,8 @@ extern _Atomic(jl_typemap_entry_t*) call_cache[N_CALL_CACHE] JL_GLOBALLY_ROOTED; void free_stack(void *stkbuf, size_t bufsz) JL_NOTSAFEPOINT; -JL_DLLEXPORT extern int jl_lineno; -JL_DLLEXPORT extern const char *jl_filename; +JL_DLLEXPORT extern _Atomic(int) jl_lineno; +JL_DLLEXPORT extern _Atomic(const char *) jl_filename; jl_value_t *jl_gc_small_alloc_noinline(jl_ptls_t ptls, int offset, int osize); @@ -677,12 +694,16 @@ typedef union { #define SOURCE_MODE_NOT_REQUIRED 0x0 #define SOURCE_MODE_ABI 0x1 +#define METHOD_SIG_LATEST_WHICH 0b0001 +#define METHOD_SIG_LATEST_ONLY 0b0010 +#define METHOD_SIG_PRECOMPILE_MANY 0b0100 + JL_DLLEXPORT jl_code_instance_t *jl_engine_reserve(jl_method_instance_t *m, jl_value_t *owner); JL_DLLEXPORT void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src); void jl_engine_sweep(jl_ptls_t *gc_all_tls_states) JL_NOTSAFEPOINT; int jl_engine_hasreserved(jl_method_instance_t *m, jl_value_t *owner) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_code_instance_t *jl_type_infer(jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t world, uint8_t source_mode); +JL_DLLEXPORT jl_code_instance_t *jl_type_infer(jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t world, uint8_t source_mode, uint8_t trim_mode); JL_DLLEXPORT jl_code_info_t *jl_gdbcodetyped1(jl_method_instance_t *mi, size_t world); JL_DLLEXPORT jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *meth JL_PROPAGATES_ROOT, size_t world); JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred( @@ -701,7 +722,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( int32_t const_flags, size_t min_world, size_t max_world, uint32_t effects, jl_value_t *analysis_results, jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/); -JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, int compiled) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_code_instance_t *jl_get_ci_equiv(jl_code_instance_t *ci JL_PROPAGATES_ROOT, size_t target_world) JL_NOTSAFEPOINT; STATIC_INLINE jl_method_instance_t *jl_get_ci_mi(jl_code_instance_t *ci JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { @@ -732,9 +753,29 @@ JL_DLLEXPORT void jl_resolve_definition_effects_in_ir(jl_array_t *stmts, jl_modu JL_DLLEXPORT int jl_maybe_add_binding_backedge(jl_binding_t *b, jl_value_t *edge, jl_method_t *in_method); JL_DLLEXPORT void jl_add_binding_backedge(jl_binding_t *b, jl_value_t *edge); +static const uint8_t MI_FLAG_BACKEDGES_INUSE = 0b0100; +static const uint8_t MI_FLAG_BACKEDGES_DIRTY = 0b1000; +static const uint8_t MI_FLAG_BACKEDGES_ALL = 0b1100; + +STATIC_INLINE jl_array_t *jl_mi_get_backedges_mutate(jl_method_instance_t *mi JL_PROPAGATES_ROOT, uint8_t *flags) { + *flags = jl_atomic_load_relaxed(&mi->flags) & (MI_FLAG_BACKEDGES_ALL); + jl_array_t *ret = mi->backedges; + if (ret) + jl_atomic_fetch_or_relaxed(&mi->flags, MI_FLAG_BACKEDGES_INUSE); + return ret; +} + +STATIC_INLINE jl_array_t *jl_mi_get_backedges(jl_method_instance_t *mi JL_PROPAGATES_ROOT) { + assert(!(jl_atomic_load_relaxed(&mi->flags) & MI_FLAG_BACKEDGES_ALL)); + jl_array_t *ret = mi->backedges; + return ret; +} + int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_code_instance_t **caller) JL_NOTSAFEPOINT; int set_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instance_t *caller); +int clear_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instance_t *caller); void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_code_instance_t *caller); +void jl_mi_done_backedges(jl_method_instance_t *mi JL_PROPAGATES_ROOT, uint8_t old_flags); JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_t* root); void jl_append_method_roots(jl_method_t *m, uint64_t modid, jl_array_t* roots); @@ -754,16 +795,11 @@ JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t); #define JL_CALLABLE(name) \ JL_DLLEXPORT jl_value_t *name(jl_value_t *F, jl_value_t **args, uint32_t nargs) -JL_CALLABLE(jl_f_svec); JL_CALLABLE(jl_f_tuple); -JL_CALLABLE(jl_f_intrinsic_call); -JL_CALLABLE(jl_f_opaque_closure_call); void jl_install_default_signal_handlers(void); void restore_signals(void); void jl_install_thread_signal_handler(jl_ptls_t ptls); -JL_DLLEXPORT jl_fptr_args_t jl_get_builtin_fptr(jl_datatype_t *dt); - extern uv_loop_t *jl_io_loop; JL_DLLEXPORT void jl_uv_flush(uv_stream_t *stream); @@ -788,9 +824,9 @@ int jl_has_concrete_subtype(jl_value_t *typ); jl_tupletype_t *jl_inst_arg_tuple_type(jl_value_t *arg1, jl_value_t **args, size_t nargs, int leaf); jl_tupletype_t *jl_lookup_arg_tuple_type(jl_value_t *arg1 JL_PROPAGATES_ROOT, jl_value_t **args, size_t nargs, int leaf); JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); -void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry); +void jl_method_table_activate(jl_typemap_entry_t *newentry); jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); -jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED; +jl_method_t *jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *name, jl_fptr_args_t fptr) JL_GC_DISABLED; int jl_obviously_unequal(jl_value_t *a, jl_value_t *b); int jl_has_bound_typevars(jl_value_t *v, jl_typeenv_t *env) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_array_t *jl_find_free_typevars(jl_value_t *v); @@ -840,8 +876,7 @@ int setonce_bits(jl_datatype_t *rty, char *p, jl_value_t *owner, jl_value_t *rhs jl_expr_t *jl_exprn(jl_sym_t *head, size_t n); jl_function_t *jl_new_generic_function(jl_sym_t *name, jl_module_t *module, size_t new_world); jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_t *module, jl_datatype_t *st, size_t new_world); -int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), void *env); -int foreach_mtable_in_module(jl_module_t *m, int (*visit)(jl_methtable_t *mt, void *env), void *env); +int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), jl_array_t *mod_array, void *env); void jl_init_main_module(void); JL_DLLEXPORT int jl_is_submodule(jl_module_t *child, jl_module_t *parent) JL_NOTSAFEPOINT; jl_array_t *jl_get_loaded_modules(void); @@ -855,6 +890,7 @@ JL_DLLEXPORT void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3(jl_binding_t *b JL_ROOTING_ARGUMENT, jl_module_t *mod, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, enum jl_partition_kind, size_t new_world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int expanded, const char **toplevel_filename, int *toplevel_lineno); +void jl_module_initial_using(jl_module_t *to, jl_module_t *from); STATIC_INLINE struct _jl_module_using *module_usings_getidx(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; STATIC_INLINE jl_module_t *module_usings_getmod(jl_module_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to); @@ -884,7 +920,10 @@ STATIC_INLINE size_t module_usings_max(jl_module_t *m) JL_NOTSAFEPOINT { JL_DLLEXPORT jl_sym_t *jl_module_name(jl_module_t *m) JL_NOTSAFEPOINT; void jl_add_scanned_method(jl_module_t *m, jl_method_t *meth); -jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e); +jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e, size_t world); +JL_DLLEXPORT jl_value_t *jl_eval_globalref(jl_globalref_t *g, size_t world); +jl_value_t *jl_get_globalref_value(jl_globalref_t *gr, size_t world); +jl_value_t *jl_get_global_value(jl_module_t *m, jl_sym_t *var, size_t world); jl_value_t *jl_interpret_opaque_closure(jl_opaque_closure_t *clos, jl_value_t **args, size_t nargs); jl_value_t *jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t *src); jl_value_t *jl_interpret_toplevel_expr_in(jl_module_t *m, jl_value_t *e, @@ -909,10 +948,16 @@ jl_datatype_t *jl_nth_argument_datatype(jl_value_t *argtypes JL_PROPAGATES_ROOT, JL_DLLEXPORT jl_value_t *jl_argument_datatype(jl_value_t *argt JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_methtable_t *jl_method_table_for( jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_methcache_t *jl_method_cache_for( + jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; jl_methtable_t *jl_kwmethod_table_for( jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +jl_methcache_t *jl_kwmethod_cache_for( + jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_methtable_t *jl_method_get_table( jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_methcache_t *jl_method_get_cache( + jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT int jl_pointer_egal(jl_value_t *t); JL_DLLEXPORT jl_value_t *jl_nth_slot_type(jl_value_t *sig JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; @@ -927,7 +972,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t *bpart); extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; -extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; +extern jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; extern jl_genericmemory_t *jl_global_roots_list JL_GLOBALLY_ROOTED; extern jl_genericmemory_t *jl_global_roots_keyset JL_GLOBALLY_ROOTED; extern arraylist_t *jl_entrypoint_mis; @@ -967,6 +1012,10 @@ STATIC_INLINE int jl_bkind_is_defined_constant(enum jl_partition_kind kind) JL_N return kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_BACKDATED_CONST; } +STATIC_INLINE int jl_bkind_is_real_constant(enum jl_partition_kind kind) JL_NOTSAFEPOINT { + return kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT; +} + JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition_with_hint(jl_binding_t *b JL_PROPAGATES_ROOT, jl_binding_partition_t *previous_part, size_t world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition_all(jl_binding_t *b JL_PROPAGATES_ROOT, size_t min_world, size_t max_world) JL_GLOBALLY_ROOTED; @@ -1261,21 +1310,21 @@ _Atomic(jl_value_t*) *jl_table_peek_bp(jl_genericmemory_t *a, jl_value_t *key) J JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*); -JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent); -JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name); -JL_DLLEXPORT void jl_add_default_names(jl_module_t *m, uint8_t default_using_core, uint8_t self_name); +jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name); +jl_module_t *jl_add_standard_imports(jl_module_t *m); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); +JL_DLLEXPORT jl_methcache_t *jl_new_method_cache(void); JL_DLLEXPORT jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES_ROOT, size_t world, int mt_cache); jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_svec_t *sp) JL_PROPAGATES_ROOT; JL_DLLEXPORT jl_value_t *jl_rettype_inferred(jl_value_t *owner, jl_method_instance_t *li JL_PROPAGATES_ROOT, size_t min_world, size_t max_world); JL_DLLEXPORT jl_value_t *jl_rettype_inferred_native(jl_method_instance_t *mi, size_t min_world, size_t max_world) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_code_instance_t *jl_method_compiled(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt JL_PROPAGATES_ROOT, jl_value_t *type, size_t world); +JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_value_t *type, size_t world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo( jl_method_t *m JL_PROPAGATES_ROOT, jl_value_t *type, jl_svec_t *sparams); jl_method_instance_t *jl_specializations_get_or_insert(jl_method_instance_t *mi_ins JL_PROPAGATES_ROOT); JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_value_t *invokesig, jl_code_instance_t *caller); -JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *typ, jl_code_instance_t *caller); +JL_DLLEXPORT void jl_method_table_add_backedge(jl_value_t *typ, jl_code_instance_t *caller); JL_DLLEXPORT void jl_mi_cache_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, jl_code_instance_t *ci JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); JL_DLLEXPORT int jl_mi_try_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, @@ -1599,9 +1648,8 @@ JL_DLLEXPORT jl_value_t *jl_get_cfunction_trampoline( jl_value_t *fobj, jl_datatype_t *result, htable_t *cache, jl_svec_t *fill, void *(*init_trampoline)(void *tramp, void **nval), jl_unionall_t *env, jl_value_t **vals); -JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data); -JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, void *unspecialized, jl_value_t *declrt, jl_value_t *sigt, size_t nargs, int specsig, - jl_code_instance_t *codeinst, jl_callptr_t invoke, void *target, int target_specsig); +JL_DLLEXPORT void *jl_get_abi_converter(jl_task_t *ct, void *data); +JL_DLLIMPORT void *jl_jit_abi_converter(jl_task_t *ct, jl_abi_t from_abi, jl_code_instance_t *codeinst); // Special filenames used to refer to internal julia libraries @@ -1716,6 +1764,7 @@ JL_DLLEXPORT jl_value_t *jl_have_fma(jl_value_t *a); JL_DLLEXPORT int jl_stored_inline(jl_value_t *el_type); JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a); JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary); +JL_DLLEXPORT jl_genericmemory_t *jl_genericmemory_copy(jl_genericmemory_t *mem); JL_DLLEXPORT uintptr_t jl_object_id_(uintptr_t tv, jl_value_t *v) JL_NOTSAFEPOINT; JL_DLLEXPORT void jl_set_next_task(jl_task_t *task) JL_NOTSAFEPOINT; @@ -1855,7 +1904,6 @@ extern JL_DLLEXPORT jl_sym_t *jl_module_sym; extern JL_DLLEXPORT jl_sym_t *jl_slot_sym; extern JL_DLLEXPORT jl_sym_t *jl_export_sym; extern JL_DLLEXPORT jl_sym_t *jl_public_sym; -extern JL_DLLEXPORT jl_sym_t *jl_import_sym; extern JL_DLLEXPORT jl_sym_t *jl_toplevel_sym; extern JL_DLLEXPORT jl_sym_t *jl_quote_sym; extern JL_DLLEXPORT jl_sym_t *jl_line_sym; @@ -1877,7 +1925,6 @@ extern JL_DLLEXPORT jl_sym_t *jl_pop_exception_sym; extern JL_DLLEXPORT jl_sym_t *jl_exc_sym; extern JL_DLLEXPORT jl_sym_t *jl_error_sym; extern JL_DLLEXPORT jl_sym_t *jl_new_sym; -extern JL_DLLEXPORT jl_sym_t *jl_using_sym; extern JL_DLLEXPORT jl_sym_t *jl_splatnew_sym; extern JL_DLLEXPORT jl_sym_t *jl_block_sym; extern JL_DLLEXPORT jl_sym_t *jl_new_opaque_closure_sym; @@ -2023,8 +2070,9 @@ JL_DLLIMPORT void jl_dump_native(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, ios_t *z, ios_t *s, jl_emission_params_t *params); JL_DLLIMPORT void jl_get_llvm_gvs(void *native_code, size_t *num_els, void **gvs); +JL_DLLIMPORT void jl_get_llvm_gv_inits(void *native_code, size_t *num_els, void **inits); JL_DLLIMPORT void jl_get_llvm_external_fns(void *native_code, size_t *num_els, - jl_code_instance_t *gvs); + jl_code_instance_t *fns); JL_DLLIMPORT void jl_get_function_id(void *native_code, jl_code_instance_t *ncode, int32_t *func_idx, int32_t *specfunc_idx); JL_DLLIMPORT void jl_register_fptrs(uint64_t image_base, const struct _jl_image_fptrs_t *fptrs, diff --git a/src/julia_locks.h b/src/julia_locks.h index 92d67b34b1692..a4b5fd96b8fb4 100644 --- a/src/julia_locks.h +++ b/src/julia_locks.h @@ -3,6 +3,10 @@ #ifndef JL_LOCKS_H #define JL_LOCKS_H +#ifdef _COMPILER_TSAN_ENABLED_ +#include +#endif + #ifdef __cplusplus extern "C" { #endif @@ -34,7 +38,13 @@ static inline void jl_mutex_lock_nogc(jl_mutex_t *lock) JL_NOTSAFEPOINT JL_NOTSA // Hide this body from the analyzer, otherwise it complains that we're calling // a non-safepoint from this function. The 0 arguments guarantees that we do // not reach the safepoint, but the analyzer can't figure that out +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_lock(lock, __tsan_mutex_write_reentrant); +#endif jl_mutex_wait(lock, 0); +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_lock(lock, __tsan_mutex_write_reentrant, 1); +#endif #endif } diff --git a/src/llvm-Compression.cpp b/src/llvm-Compression.cpp new file mode 100644 index 0000000000000..c83f626747c27 --- /dev/null +++ b/src/llvm-Compression.cpp @@ -0,0 +1,245 @@ +//===--- Compression.cpp - Compression implementation ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements compression functions. +// +//===----------------------------------------------------------------------===// + +#include "llvm-Compression.h" +#include "llvm/Support/Compression.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#if LLVM_ENABLE_ZLIB +#include +#endif +#if LLVM_ENABLE_ZSTD +#include +#endif + +using namespace llvm; +using namespace llvm::compression; + +const char *compression::getReasonIfUnsupported(compression::Format F) { + switch (F) { + case compression::Format::Zlib: + if (zlib::isAvailable()) + return nullptr; + return "LLVM was not built with LLVM_ENABLE_ZLIB or did not find zlib at " + "build time"; + case compression::Format::Zstd: + if (zstd::isAvailable()) + return nullptr; + return "LLVM was not built with LLVM_ENABLE_ZSTD or did not find zstd at " + "build time"; + } + llvm_unreachable(""); +} + +void compression::compress(Params P, ArrayRef Input, + SmallVectorImpl &Output) { + switch (P.format) { + case compression::Format::Zlib: + zlib::compress(Input, Output, P.level); + break; + case compression::Format::Zstd: + zstd::compress(Input, Output, P.level, P.zstdEnableLdm); + break; + } +} + +Error compression::decompress(DebugCompressionType T, ArrayRef Input, + uint8_t *Output, size_t UncompressedSize) { + switch (formatFor(T)) { + case compression::Format::Zlib: + return zlib::decompress(Input, Output, UncompressedSize); + case compression::Format::Zstd: + return zstd::decompress(Input, Output, UncompressedSize); + } + llvm_unreachable(""); +} + +Error compression::decompress(compression::Format F, ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + switch (F) { + case compression::Format::Zlib: + return zlib::decompress(Input, Output, UncompressedSize); + case compression::Format::Zstd: + return zstd::decompress(Input, Output, UncompressedSize); + } + llvm_unreachable(""); +} + +Error compression::decompress(DebugCompressionType T, ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + return decompress(formatFor(T), Input, Output, UncompressedSize); +} + +#if LLVM_ENABLE_ZLIB + +static StringRef convertZlibCodeToString(int Code) { + switch (Code) { + case Z_MEM_ERROR: + return "zlib error: Z_MEM_ERROR"; + case Z_BUF_ERROR: + return "zlib error: Z_BUF_ERROR"; + case Z_STREAM_ERROR: + return "zlib error: Z_STREAM_ERROR"; + case Z_DATA_ERROR: + return "zlib error: Z_DATA_ERROR"; + case Z_OK: + default: + llvm_unreachable("unknown or unexpected zlib status code"); + } +} + +bool zlib::isAvailable() { return true; } + +void zlib::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level) { + unsigned long CompressedSize = ::compressBound(Input.size()); + CompressedBuffer.resize_for_overwrite(CompressedSize); + int Res = ::compress2((Bytef *)CompressedBuffer.data(), &CompressedSize, + (const Bytef *)Input.data(), Input.size(), Level); + if (Res == Z_MEM_ERROR) + report_bad_alloc_error("Allocation failed"); + assert(Res == Z_OK); + // Tell MemorySanitizer that zlib output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(CompressedBuffer.data(), CompressedSize); + if (CompressedSize < CompressedBuffer.size()) + CompressedBuffer.truncate(CompressedSize); +} + +Error zlib::decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize) { + int Res = ::uncompress((Bytef *)Output, (uLongf *)&UncompressedSize, + (const Bytef *)Input.data(), Input.size()); + // Tell MemorySanitizer that zlib output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(Output, UncompressedSize); + return Res ? make_error(convertZlibCodeToString(Res), + inconvertibleErrorCode()) + : Error::success(); +} + +Error zlib::decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + Output.resize_for_overwrite(UncompressedSize); + Error E = zlib::decompress(Input, Output.data(), UncompressedSize); + if (UncompressedSize < Output.size()) + Output.truncate(UncompressedSize); + return E; +} + +#else +bool zlib::isAvailable() { return false; } +void zlib::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level) { + llvm_unreachable("zlib::compress is unavailable"); +} +Error zlib::decompress(ArrayRef Input, uint8_t *UncompressedBuffer, + size_t &UncompressedSize) { + llvm_unreachable("zlib::decompress is unavailable"); +} +Error zlib::decompress(ArrayRef Input, + SmallVectorImpl &UncompressedBuffer, + size_t UncompressedSize) { + llvm_unreachable("zlib::decompress is unavailable"); +} +#endif + +#if LLVM_ENABLE_ZSTD + +bool zstd::isAvailable() { return true; } + +#include // Ensure ZSTD library is included + +void zstd::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level, + bool EnableLdm) { + ZSTD_CCtx *Cctx = ZSTD_createCCtx(); + if (!Cctx) + report_bad_alloc_error("Failed to create ZSTD_CCtx"); + + if (ZSTD_isError(ZSTD_CCtx_setParameter( + Cctx, ZSTD_c_enableLongDistanceMatching, EnableLdm ? 1 : 0))) { + ZSTD_freeCCtx(Cctx); + report_bad_alloc_error("Failed to set ZSTD_c_enableLongDistanceMatching"); + } + + if (ZSTD_isError( + ZSTD_CCtx_setParameter(Cctx, ZSTD_c_compressionLevel, Level))) { + ZSTD_freeCCtx(Cctx); + report_bad_alloc_error("Failed to set ZSTD_c_compressionLevel"); + } + + unsigned long CompressedBufferSize = ZSTD_compressBound(Input.size()); + CompressedBuffer.resize_for_overwrite(CompressedBufferSize); + + size_t const CompressedSize = + ZSTD_compress2(Cctx, CompressedBuffer.data(), CompressedBufferSize, + Input.data(), Input.size()); + + ZSTD_freeCCtx(Cctx); + + if (ZSTD_isError(CompressedSize)) + report_bad_alloc_error("Compression failed"); + + __msan_unpoison(CompressedBuffer.data(), CompressedSize); + if (CompressedSize < CompressedBuffer.size()) + CompressedBuffer.truncate(CompressedSize); +} + +Error zstd::decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize) { + const size_t Res = ::ZSTD_decompress( + Output, UncompressedSize, (const uint8_t *)Input.data(), Input.size()); + UncompressedSize = Res; + if (ZSTD_isError(Res)) + return make_error(ZSTD_getErrorName(Res), + inconvertibleErrorCode()); + // Tell MemorySanitizer that zstd output buffer is fully initialized. + // This avoids a false report when running LLVM with uninstrumented ZLib. + __msan_unpoison(Output, UncompressedSize); + return Error::success(); +} + +Error zstd::decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + Output.resize_for_overwrite(UncompressedSize); + Error E = zstd::decompress(Input, Output.data(), UncompressedSize); + if (UncompressedSize < Output.size()) + Output.truncate(UncompressedSize); + return E; +} + +#else +bool zstd::isAvailable() { return false; } +void zstd::compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, int Level, + bool EnableLdm) { + llvm_unreachable("zstd::compress is unavailable"); +} +Error zstd::decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize) { + llvm_unreachable("zstd::decompress is unavailable"); +} +Error zstd::decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize) { + llvm_unreachable("zstd::decompress is unavailable"); +} +#endif diff --git a/src/llvm-Compression.h b/src/llvm-Compression.h new file mode 100644 index 0000000000000..246ccbd6f6dcf --- /dev/null +++ b/src/llvm-Compression.h @@ -0,0 +1,136 @@ +//===-- llvm/Support/Compression.h ---Compression----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains basic functions for compression/decompression. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_COMPRESSION_H +#define LLVM_SUPPORT_COMPRESSION_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/DataTypes.h" + +namespace llvm { +template class SmallVectorImpl; +class Error; + +// None indicates no compression. The other members are a subset of +// compression::Format, which is used for compressed debug sections in some +// object file formats (e.g. ELF). This is a separate class as we may add new +// compression::Format members for non-debugging purposes. +enum class DebugCompressionType { + None, ///< No compression + Zlib, ///< zlib + Zstd, ///< Zstandard +}; + +namespace compression { +namespace zlib { + +constexpr int NoCompression = 0; +constexpr int BestSpeedCompression = 1; +constexpr int DefaultCompression = 6; +constexpr int BestSizeCompression = 9; + +LLVM_ABI bool isAvailable(); + +LLVM_ABI void compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, + int Level = DefaultCompression); + +LLVM_ABI Error decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize); + +LLVM_ABI Error decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize); + +} // End of namespace zlib + +namespace zstd { + +constexpr int NoCompression = -5; +constexpr int BestSpeedCompression = 1; +constexpr int DefaultCompression = 5; +constexpr int BestSizeCompression = 12; + +LLVM_ABI bool isAvailable(); + +LLVM_ABI void compress(ArrayRef Input, + SmallVectorImpl &CompressedBuffer, + int Level = DefaultCompression, bool EnableLdm = false); + +LLVM_ABI Error decompress(ArrayRef Input, uint8_t *Output, + size_t &UncompressedSize); + +LLVM_ABI Error decompress(ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize); + +} // End of namespace zstd + +enum class Format { + Zlib, + Zstd, +}; + +inline Format formatFor(DebugCompressionType Type) { + switch (Type) { + case DebugCompressionType::None: + llvm_unreachable("not a compression type"); + case DebugCompressionType::Zlib: + return Format::Zlib; + case DebugCompressionType::Zstd: + return Format::Zstd; + } + llvm_unreachable(""); +} + +struct Params { + constexpr Params(Format F) + : format(F), level(F == Format::Zlib ? zlib::DefaultCompression + : zstd::DefaultCompression) {} + constexpr Params(Format F, int L, bool Ldm = false) + : format(F), level(L), zstdEnableLdm(Ldm) {} + Params(DebugCompressionType Type) : Params(formatFor(Type)) {} + + Format format; + int level; + bool zstdEnableLdm = false; // Enable zstd long distance matching + // This may support multi-threading for zstd in the future. Note that + // different threads may produce different output, so be careful if certain + // output determinism is desired. +}; + +// Return nullptr if LLVM was built with support (LLVM_ENABLE_ZLIB, +// LLVM_ENABLE_ZSTD) for the specified compression format; otherwise +// return a string literal describing the reason. +LLVM_ABI const char *getReasonIfUnsupported(Format F); + +// Compress Input with the specified format P.Format. If Level is -1, use +// *::DefaultCompression for the format. +LLVM_ABI void compress(Params P, ArrayRef Input, + SmallVectorImpl &Output); + +// Decompress Input. The uncompressed size must be available. +LLVM_ABI Error decompress(DebugCompressionType T, ArrayRef Input, + uint8_t *Output, size_t UncompressedSize); +LLVM_ABI Error decompress(Format F, ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize); +LLVM_ABI Error decompress(DebugCompressionType T, ArrayRef Input, + SmallVectorImpl &Output, + size_t UncompressedSize); + +} // End of namespace compression + +} // End of namespace llvm + +#endif diff --git a/src/llvm-alloc-helpers.cpp b/src/llvm-alloc-helpers.cpp index 194c6837860ca..a1ed66a190190 100644 --- a/src/llvm-alloc-helpers.cpp +++ b/src/llvm-alloc-helpers.cpp @@ -214,6 +214,7 @@ void jl_alloc::runEscapeAnalysis(llvm::CallInst *I, EscapeAnalysisRequiredArgs r } if (auto call = dyn_cast(inst)) { // TODO handle `memcmp` + // TODO handle `memcpy` which is used a lot more often since opaque pointers // None of the intrinsics should care if the memory is stack or heap allocated. auto callee = call->getCalledOperand(); if (auto II = dyn_cast(call)) { diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp index bfc1b42444cd1..56bb1ab7c706b 100644 --- a/src/llvm-alloc-opt.cpp +++ b/src/llvm-alloc-opt.cpp @@ -758,7 +758,9 @@ void Optimizer::moveToStack(CallInst *orig_inst, size_t sz, bool has_ref, AllocF auto replace_inst = [&] (Instruction *user) { Instruction *orig_i = cur.orig_i; Instruction *new_i = cur.new_i; - if (isa(user) || isa(user)) { + if (isa(user) || isa(user) || + isa(user) || isa(user)) { + // TODO: these atomics are likely removable if the user is the first argument user->replaceUsesOfWith(orig_i, new_i); } else if (auto call = dyn_cast(user)) { @@ -1131,6 +1133,7 @@ void Optimizer::splitOnStack(CallInst *orig_inst) return; } else if (isa(user) || isa(user)) { + // TODO: Downgrade atomics here potentially auto slot_idx = find_slot(offset); auto &slot = slots[slot_idx]; assert(slot.offset <= offset && slot.offset + slot.size >= offset); diff --git a/src/llvm-codegen-shared.h b/src/llvm-codegen-shared.h index cfdb8eb5b1a99..d5d7ae3d50113 100644 --- a/src/llvm-codegen-shared.h +++ b/src/llvm-codegen-shared.h @@ -13,9 +13,6 @@ #include "julia.h" -#define STR(csym) #csym -#define XSTR(csym) STR(csym) - static constexpr std::nullopt_t None = std::nullopt; enum AddressSpace { @@ -200,6 +197,7 @@ static inline llvm::Value *get_current_signal_page_from_ptls(llvm::IRBuilder<> & llvm::Value *psafepoint = builder.CreateConstInBoundsGEP1_32(i8, ptls, nthfield); LoadInst *ptls_load = builder.CreateAlignedLoad( T_ptr, psafepoint, Align(sizeof(void *)), "safepoint"); + ptls_load->setOrdering(AtomicOrdering::Monotonic); tbaa_decorate(tbaa, ptls_load); return ptls_load; } diff --git a/src/llvm-expand-atomic-modify.cpp b/src/llvm-expand-atomic-modify.cpp new file mode 100644 index 0000000000000..e4152bb45fe42 --- /dev/null +++ b/src/llvm-expand-atomic-modify.cpp @@ -0,0 +1,489 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +// TODO: move this feature into AtomicExpandImpl + +#include "llvm-version.h" +#include "passes.h" + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include "llvm/IR/MemoryModelRelaxationAnnotations.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "julia.h" +#include "julia_assert.h" + +#define DEBUG_TYPE "expand-atomic-modify" +#undef DEBUG + +using namespace llvm; + +// This pass takes fake call instructions that look like this which were emitted by the front end: +// (oldval, newval) = call atomicmodify.iN(ptr %op, ptr align(N) %ptr, i8 immarg %SSID, i8 immarg %Ordering, ...) !rmwattributes +// where op is a function with a prototype of `iN (iN arg, ...)` +// Then rewrite that to +// oldval = atomicrmw op ptr, val ordering syncscope +// newval = op oldval, val +// Or to an equivalent RMWCmpXchgLoop if `op` isn't valid for atomicrmw + + +// from AtomicExpandImpl, with modification of failure order and added Attributes +using CreateWeakCmpXchgInstFun = + std::function; + +static void createWeakCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr, + Value *Loaded, Value *NewVal, Align AddrAlign, + AtomicOrdering MemOpOrder, SyncScope::ID SSID, Instruction &Attributes, + Value *&Success, Value *&NewLoaded) { + Type *OrigTy = NewVal->getType(); + + // This code can go away when cmpxchg supports FP types. + assert(!OrigTy->isPointerTy()); + bool NeedBitcast = OrigTy->isFloatingPointTy(); + if (NeedBitcast) { + IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits()); + NewVal = Builder.CreateBitCast(NewVal, IntTy); + Loaded = Builder.CreateBitCast(Loaded, IntTy); + } + + AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg( + Addr, Loaded, NewVal, AddrAlign, MemOpOrder, + AtomicOrdering::Monotonic, // why does LLVM use AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder) here + SSID); + Pair->copyMetadata(Attributes); + Success = Builder.CreateExtractValue(Pair, 1, "success"); + NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); + + if (NeedBitcast) + NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy); +} + +// from AtomicExpandImpl, with modification of values returned +std::pair insertRMWCmpXchgLoop( + IRBuilderBase &Builder, Type *ResultTy, Value *Addr, Align AddrAlign, + AtomicOrdering MemOpOrder, SyncScope::ID SSID, Instruction &Attributes, + const std::function &PerformOp, + const CreateWeakCmpXchgInstFun &CreateWeakCmpXchg) { + LLVMContext &Ctx = Builder.getContext(); + BasicBlock *BB = Builder.GetInsertBlock(); + Function *F = BB->getParent(); + + // Given: atomicrmw some_op iN* %addr, iN %incr ordering + // + // The standard expansion we produce is: + // [...] + // %init_loaded = load atomic iN* %addr + // br label %loop + // loop: + // %loaded = phi iN [ %init_loaded, %entry ], [ %new_loaded, %loop ] + // %new = some_op iN %loaded, %incr + // %pair = cmpxchg iN* %addr, iN %loaded, iN %new + // %new_loaded = extractvalue { iN, i1 } %pair, 0 + // %success = extractvalue { iN, i1 } %pair, 1 + // br i1 %success, label %atomicrmw.end, label %loop + // atomicrmw.end: + // [...] + BasicBlock *ExitBB = + BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end"); + BasicBlock *LoopBB = BasicBlock::Create(Ctx, "atomicrmw.start", F, ExitBB); + + // The split call above "helpfully" added a branch at the end of BB (to the + // wrong place), but we want a load. It's easiest to just remove + // the branch entirely. + std::prev(BB->end())->eraseFromParent(); + Builder.SetInsertPoint(BB); + LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign); + InitLoaded->setOrdering(AtomicOrdering::Unordered); // n.b. the original LLVM pass is missing this call so is actually mildly UB + Builder.CreateBr(LoopBB); + + // Start the main loop block now that we've taken care of the preliminaries. + Builder.SetInsertPoint(LoopBB); + PHINode *Loaded = Builder.CreatePHI(ResultTy, 2, "loaded"); + Loaded->addIncoming(InitLoaded, BB); + + Value *NewVal = PerformOp(Builder, Loaded); + + Value *NewLoaded = nullptr; + Value *Success = nullptr; + + CreateWeakCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign, + MemOpOrder == AtomicOrdering::Unordered + ? AtomicOrdering::Monotonic + : MemOpOrder, + SSID, Attributes, Success, NewLoaded); + assert(Success && NewLoaded); + + Loaded->addIncoming(NewLoaded, LoopBB); + + Builder.CreateCondBr(Success, ExitBB, LoopBB); + + Builder.SetInsertPoint(ExitBB, ExitBB->begin()); + return {NewLoaded, NewVal}; +} + +// from AtomicExpandImpl +// IRBuilder to be used for replacement atomic instructions. +struct ReplacementIRBuilder + : IRBuilder { + MDNode *MMRAMD = nullptr; + + // Preserves the DebugLoc from I, and preserves still valid metadata. + // Enable StrictFP builder mode when appropriate. + explicit ReplacementIRBuilder(Instruction *I, const DataLayout &DL) + : IRBuilder(I->getContext(), InstSimplifyFolder(DL), + IRBuilderCallbackInserter( + [this](Instruction *I) { addMMRAMD(I); })) { + SetInsertPoint(I); + this->CollectMetadataToCopy(I, {LLVMContext::MD_pcsections}); + if (BB->getParent()->getAttributes().hasFnAttr(Attribute::StrictFP)) + this->setIsFPConstrained(true); + + MMRAMD = I->getMetadata(LLVMContext::MD_mmra); + } + + void addMMRAMD(Instruction *I) { + if (canInstructionHaveMMRAs(*I)) + I->setMetadata(LLVMContext::MD_mmra, MMRAMD); + } +}; + +// Must check that either Target cannot observe or mutate global state +// or that no trailing instructions does so either. +// Depending on the choice, it can also decide whether it is better to move Target after RMW +// or to move RMW before Target (or meet somewhere in the middle). +// Currently conservatively implemented as there being no instruction in the +// function which writes memory (which includes any atomics). +// Excluding the Target itself, unless some other instruction might read memory to observe it. +static bool canReorderWithRMW(Instruction &Target, bool verifyop) +{ + if (!verifyop) + return true; + Function &Op = *Target.getFunction(); + // quick check: if Op is nosync and Target doesn't access any memory, then reordering is trivially valid + bool nosync = Op.hasNoSync(); + if (nosync && !Target.mayReadOrWriteMemory()) + return true; + // otherwise, scan the whole function to see if any function accesses memory + // in a way that would conflict with reordering the atomic read and write + bool mayRead = false; + for (auto &BB : Op) { + for (auto &I : BB) { + if (&I == &Target) + continue; + if (I.mayWriteToMemory()) + return false; + if (!mayRead) { + mayRead = I.mayReadFromMemory(); + if (!nosync && mayRead) + return false; + } + } + } + // if any other instruction read memory, then the ordering of any writes by the target instruction might be observed + return !(mayRead && Target.mayWriteToMemory()); +} + +static std::variant patternMatchAtomicRMWOp(Value *Old, Use **ValOp, Value *RetVal) +{ + bool verifyop = RetVal == nullptr; + assert(verifyop ? isa(Old) : isa(Old)); + Function *Op = verifyop ? cast(Old)->getParent() : nullptr; + if (verifyop && (Op->isDeclaration() || Op->isInterposable() || Op->isIntrinsic())) + return false; + // TODO: peek forward from Old through any trivial casts which don't affect the instruction (e.g. i64 to f64 and back) + if (RetVal == nullptr) { + if (Old->use_empty()) { + if (ValOp) *ValOp = nullptr; + return AtomicRMWInst::Xchg; + } + if (!Old->hasOneUse()) + return false; + ReturnInst *Ret = nullptr; + for (auto &BB : *Op) { + if (isa(BB.getTerminator())) { + if (Ret != nullptr) + return false; + Ret = cast(BB.getTerminator()); + } + } + if (Ret == nullptr) + return false; + // Now examine the instruction list + RetVal = Ret->getReturnValue(); + if (!RetVal->hasOneUse()) + return false; + } + if (RetVal == Old) { + // special token indicating to convert to an atomic fence + if (ValOp) *ValOp = nullptr; + return AtomicRMWInst::Or; + } + if (Old->use_empty()) { + if (ValOp) *ValOp = nullptr; + return AtomicRMWInst::Xchg; + } + if (auto BinOp = dyn_cast(RetVal)) { + if ((BinOp->getOperand(0) == Old || (BinOp->isCommutative() && BinOp->getOperand(1) == Old)) && canReorderWithRMW(*BinOp, verifyop)) { + if (ValOp) *ValOp = &BinOp->getOperandUse(BinOp->getOperand(0) == Old ? 1 : 0); + switch (BinOp->getOpcode()) { + case Instruction::Add: + return AtomicRMWInst::Add; + case Instruction::Sub: + return AtomicRMWInst::Sub; + case Instruction::And: + return AtomicRMWInst::And; + case Instruction::Or: + return AtomicRMWInst::Or; + case Instruction::Xor: + return AtomicRMWInst::Xor; + case Instruction::FAdd: + return AtomicRMWInst::FAdd; + case Instruction::FSub: + return AtomicRMWInst::FSub; + default: + break; + } + } + if (BinOp->getOpcode() == Instruction::Xor) { + if (auto CI = dyn_cast(BinOp->getOperand(1))) { + if (CI->isAllOnesValue()) { + BinOp = dyn_cast(BinOp->getOperand(0)); + if (BinOp && BinOp->hasOneUse() && BinOp->getOpcode() == Instruction::And) { + if ((BinOp->getOperand(0) == Old || (BinOp->isCommutative() && BinOp->getOperand(1) == Old)) && canReorderWithRMW(*BinOp, verifyop)) { + if (ValOp) *ValOp = &BinOp->getOperandUse(BinOp->getOperand(0) == Old ? 1 : 0); + return AtomicRMWInst::Nand; + } + } + } + } + } + return false; + } else if (auto Intr = dyn_cast(RetVal)) { + if (Intr->arg_size() == 2) { + if ((Intr->getOperand(0) == Old || (Intr->isCommutative() && Intr->getOperand(1) == Old)) && canReorderWithRMW(*Intr, verifyop)) { + if (ValOp) *ValOp = &Intr->getOperandUse(Intr->getOperand(0) == Old ? 1 : 0); + switch (Intr->getIntrinsicID()) { + case Intrinsic::minnum: + return AtomicRMWInst::FMin; + case Intrinsic::maxnum: + return AtomicRMWInst::FMax; + case Intrinsic::smax: + return AtomicRMWInst::Max; + case Intrinsic::umax: + return AtomicRMWInst::UMax; + case Intrinsic::smin: + return AtomicRMWInst::Min; + case Intrinsic::umin: + return AtomicRMWInst::UMin; +#if JL_LLVM_VERSION >= 200000 + case Intrinsic::usub_sat: + return AtomicRMWInst::USubSat; +#endif + } + } + } + return false; + } + else if (auto Intr = dyn_cast(RetVal)) { + // TODO: decide inlining cost of Op, or check alwaysinline/inlinehint, before this? + for (auto &Arg : Intr->args()) { + if (Arg == Old) { + if (canReorderWithRMW(*Intr, verifyop)) { + if (ValOp) *ValOp = &Arg; + return true; + } + return false; + } + } + } + // TODO: does this need to deal with F->hasFnAttribute(Attribute::StrictFP)? + // TODO: does Fneg and Neg have expansions? + // TODO: be able to ignore some simple bitcasts (particularly f64 to i64) + // TODO: handle longer sequences (UIncWrap, UDecWrap, USubCond, and target-specific ones for CUDA) + return false; +} + +void expandAtomicModifyToCmpXchg(CallInst &Modify, + const CreateWeakCmpXchgInstFun &CreateWeakCmpXchg) { + Value *Ptr = Modify.getOperand(0); + Function *Op = dyn_cast(Modify.getOperand(1)); + if (!Op) { + Modify.getParent()->getParent()->print(errs()); + llvm_unreachable("expected immarg for function argument"); + } + AtomicOrdering Ordering = (AtomicOrdering)cast(Modify.getOperand(2))->getZExtValue(); + SyncScope::ID SSID = (SyncScope::ID)cast(Modify.getOperand(3))->getZExtValue(); + MaybeAlign Alignment = Modify.getParamAlign(0); + unsigned user_arg_start = Modify.getFunctionType()->getNumParams(); + Type *Ty = Modify.getFunctionType()->getReturnType()->getStructElementType(0); + + ReplacementIRBuilder Builder(&Modify, Modify.getModule()->getDataLayout()); + + CallInst *ModifyOp; + { + SmallVector Args(1 + Modify.arg_size() - user_arg_start); + Args[0] = UndefValue::get(Ty); // Undef used as placeholder for Loaded / RMW; + for (size_t argi = 0; argi < Modify.arg_size() - user_arg_start; ++argi) { + Args[argi + 1] = Modify.getArgOperand(argi + user_arg_start); + } + SmallVector Defs; + Modify.getOperandBundlesAsDefs(Defs); + ModifyOp = Builder.CreateCall(Op, Args, Defs); + ModifyOp->setCallingConv(Op->getCallingConv()); + } + Use *LoadedOp = &ModifyOp->getOperandUse(0); + + Value *OldVal = nullptr; + Value *NewVal = nullptr; + auto BinOp = patternMatchAtomicRMWOp(Op->getArg(0), nullptr, nullptr); + if (BinOp != decltype(BinOp)(false)) { + Builder.SetInsertPoint(ModifyOp); + AtomicRMWInst *RMW = Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, Ptr, UndefValue::get(Ty), Alignment, Ordering, SSID); // Undef used as placeholder + RMW->copyMetadata(Modify); + Builder.SetInsertPoint(&Modify); + LoadedOp->set(RMW); + for (int attempts = 0; ; ) { + FreezeInst *TrackReturn = Builder.Insert(new FreezeInst(ModifyOp)); // Create a temporary TrackingVH so we can recover the NewVal after inlining + InlineFunctionInfo IFI; + if (!InlineFunction(*ModifyOp, IFI).isSuccess()) { + // Undo the attempt, since inlining failed + BinOp = false; + TrackReturn->eraseFromParent(); + break; + } + ModifyOp = nullptr; + NewVal = TrackReturn->getOperand(0); + TrackReturn->eraseFromParent(); + // NewVal might have been folded away by inlining so redo patternMatchAtomicRMWOp here + // tracing from RMW to NewVal, in case instsimplify folded something + Use *ValOp; + BinOp = patternMatchAtomicRMWOp(RMW, &ValOp, NewVal); + if (BinOp == decltype(BinOp)(true)) { + ModifyOp = cast(ValOp->getUser()); + LoadedOp = ValOp; + assert(LoadedOp->get() == RMW); + RMW->moveBeforePreserving(ModifyOp->getIterator()); // NewValInst is a user of RMW, and RMW has no other dependants (per patternMatchAtomicRMWOp) + BinOp = false; + if (++attempts > 3) + break; + if (auto FOp = ModifyOp->getCalledFunction()) + BinOp = patternMatchAtomicRMWOp(FOp->getArg(LoadedOp->getOperandNo()), nullptr, nullptr); + else + break; + if (BinOp == decltype(BinOp)(false)) + break; + } else { + assert(BinOp != decltype(BinOp)(true)); + auto RMWOp = std::get(BinOp); + assert(RMWOp != AtomicRMWInst::BAD_BINOP); + assert(isa(RMW->getOperand(1))); // RMW was previously being used as the placeholder for Val + Value *Val; + if (ValOp != nullptr) { + RMW->moveBeforePreserving(cast(ValOp->getUser())->getIterator()); // ValOp is a user of RMW, and RMW has no other dependants (per patternMatchAtomicRMWOp) + Val = ValOp->get(); + } else if (RMWOp == AtomicRMWInst::Xchg) { + Val = NewVal; + } else { + // convert to an atomic fence of the form: atomicrmw or %ptr, 0 + assert(RMWOp == AtomicRMWInst::Or); + Val = ConstantInt::getNullValue(Ty); + } + RMW->setOperation(RMWOp); + RMW->setOperand(1, Val); + OldVal = RMW; + break; + } + } + if (BinOp == decltype(BinOp)(false)) { + LoadedOp->set(UndefValue::get(Ty)); + RMW->eraseFromParent(); + } + } + + if (BinOp == decltype(BinOp)(false)) { + // FIXME: If FP exceptions are observable, we should force them off for the + // loop for the FP atomics. + std::tie(OldVal, NewVal) = insertRMWCmpXchgLoop( + Builder, Ty, Ptr, *Alignment, Ordering, SSID, Modify, + [&](IRBuilderBase &Builder, Value *Loaded) JL_NOTSAFEPOINT { + LoadedOp->set(Loaded); + ModifyOp->moveBeforePreserving(*Builder.GetInsertBlock(), Builder.GetInsertPoint()); + return ModifyOp; + }, + CreateWeakCmpXchg); + } + + for (auto user : make_early_inc_range(Modify.users())) { + if (auto EV = dyn_cast(user)) { + if (EV->getNumIndices() == 1) { + if (EV->use_empty()) { + EV->eraseFromParent(); + continue; + } + else if (EV->getIndices()[0] == 0) { + EV->replaceAllUsesWith(OldVal); + EV->eraseFromParent(); + continue; + } else if (EV->getIndices()[0] == 1) { + EV->replaceAllUsesWith(NewVal); + EV->eraseFromParent(); + continue; + } + } + } + } + if (!Modify.use_empty()) { + auto OldNewVal = Builder.CreateInsertValue(UndefValue::get(Modify.getType()), OldVal, 0); + OldNewVal = Builder.CreateInsertValue(OldNewVal, NewVal, 1); + Modify.replaceAllUsesWith(OldNewVal); + } + Modify.eraseFromParent(); +} + +static bool expandAtomicModify(Function &F) { + SmallVector AtomicInsts; + + // Changing control-flow while iterating through it is a bad idea, so gather a + // list of all atomic instructions before we start. + for (Instruction &I : instructions(F)) + if (auto CI = dyn_cast(&I)) { + auto callee = dyn_cast_or_null(CI->getCalledOperand()); + if (callee && callee->getName().starts_with("julia.atomicmodify.")) { + assert(CI->getFunctionType() == callee->getFunctionType()); + AtomicInsts.push_back(CI); + } + } + + bool MadeChange = !AtomicInsts.empty(); + for (auto *I : AtomicInsts) + expandAtomicModifyToCmpXchg(*I, createWeakCmpXchgInstFun); + return MadeChange; +} + +PreservedAnalyses ExpandAtomicModifyPass::run(Function &F, FunctionAnalysisManager &AM) +{ + if (expandAtomicModify(F)) { + return PreservedAnalyses::none(); + } + return PreservedAnalyses::all(); +} diff --git a/src/llvm-gc-interface-passes.h b/src/llvm-gc-interface-passes.h index fff20f85ec685..bd327ff8cc3d3 100644 --- a/src/llvm-gc-interface-passes.h +++ b/src/llvm-gc-interface-passes.h @@ -251,11 +251,13 @@ struct BBState { // These get updated during dataflow LargeSparseBitVector LiveIn; LargeSparseBitVector LiveOut; - SmallVector Safepoints; - int TopmostSafepoint = -1; + // auto Safepoints = std::range(LastSafepoint, FirstSafepoint); bool HasSafepoint = false; - // Have we gone through this basic block in our local scan yet? - bool Done = false; + // This lets us refine alloca tracking to avoid creating GC frames in + // some simple functions that only have the initial safepoint. + int FirstSafepoint = -1; + int LastSafepoint = -1; + int FirstSafepointAfterFirstDef = -1; }; struct State { @@ -292,21 +294,18 @@ struct State { // of its uses need to preserve the values listed in the map value. std::map> GCPreserves; - // The assignment of numbers to safepoints. The indices in the map - // are indices into the next three maps which store safepoint properties - std::map SafepointNumbering; + // The assignment of numbers to safepoints. These have the same ordering as + // LiveSets, LiveIfLiveOut, and CalleeRoots. + SmallVector SafepointNumbering; - // Reverse mapping index -> safepoint - SmallVector ReverseSafepointNumbering; - - // Instructions that can return twice. For now, all values live at these - // instructions will get their own, dedicated GC frame slots, because they - // have unobservable control flow, so we can't be sure where they're - // actually live. All of these are also considered safepoints. - SmallVector ReturnsTwice; + // Safepoint number of instructions that can return twice. For now, all + // values live at these instructions will get their own, dedicated GC frame + // slots, because they have unobservable control flow, so we can't be sure + // where they're actually live. + SmallVector ReturnsTwice; // The set of values live at a particular safepoint - SmallVector< LargeSparseBitVector , 0> LiveSets; + SmallVector LiveSets; // Those values that - if live out from our parent basic block - are live // at this safepoint. SmallVector> LiveIfLiveOut; @@ -332,7 +331,7 @@ struct LateLowerGCFrame: private JuliaPassContext { Value *pgcstack; Function *smallAllocFunc; - void MaybeNoteDef(State &S, BBState &BBS, Value *Def, const ArrayRef &SafepointsSoFar, + bool MaybeNoteDef(State &S, BBState &BBS, Value *Def, SmallVector &&RefinedPtr = SmallVector()); void NoteUse(State &S, BBState &BBS, Value *V, LargeSparseBitVector &Uses, Function &F); void NoteUse(State &S, BBState &BBS, Value *V, Function &F) { diff --git a/src/llvm-julia-licm.cpp b/src/llvm-julia-licm.cpp index 68fe41216bfd4..2fb17f26eb694 100644 --- a/src/llvm-julia-licm.cpp +++ b/src/llvm-julia-licm.cpp @@ -59,7 +59,11 @@ static void moveInstructionBefore(Instruction &I, Instruction &Dest, MemorySSAUpdater &MSSAU, ScalarEvolution *SE, MemorySSA::InsertionPlace Place = MemorySSA::BeforeTerminator) { +#if JL_LLVM_VERSION >= 200000 + I.moveBefore(Dest.getIterator()); +#else I.moveBefore(&Dest); +#endif if (MSSAU.getMemorySSA()) if (MemoryUseOrDef *OldMemAcc = cast_or_null( MSSAU.getMemorySSA()->getMemoryAccess(&I))) diff --git a/src/llvm-julia-passes.inc b/src/llvm-julia-passes.inc index 0cc36f799db00..bd223499f37af 100644 --- a/src/llvm-julia-passes.inc +++ b/src/llvm-julia-passes.inc @@ -16,6 +16,7 @@ FUNCTION_PASS("AllocOpt", AllocOptPass()) FUNCTION_PASS("PropagateJuliaAddrspaces", PropagateJuliaAddrspacesPass()) FUNCTION_PASS("GCInvariantVerifier", GCInvariantVerifierPass()) FUNCTION_PASS("FinalLowerGC", FinalLowerGCPass()) +FUNCTION_PASS("ExpandAtomicModify", ExpandAtomicModifyPass()) #endif //Loop passes diff --git a/src/llvm-julia-task-dispatcher.h b/src/llvm-julia-task-dispatcher.h new file mode 100644 index 0000000000000..dd4037378b6b6 --- /dev/null +++ b/src/llvm-julia-task-dispatcher.h @@ -0,0 +1,465 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +namespace { + +using namespace llvm::orc; + +template struct future_value_storage { + // Union disables default construction/destruction semantics, allowing us to + // use placement new/delete for precise control over value lifetime + union { + U value_; + }; + + future_value_storage() {} + ~future_value_storage() {} +}; + +template <> struct future_value_storage { + // No value_ member for void +}; + +struct JuliaTaskDispatcher : public TaskDispatcher { + /// Forward declarations + class future_base; + void dispatch(std::unique_ptr T) override; + void shutdown() override; + void work_until(future_base &F); +private: + /// C++ does not support non-static thread_local variables, so this needs to + /// store both the task and the associated dispatcher queue so that shutdown + /// can wait for the correct tasks to finish. + thread_local static SmallVector, JuliaTaskDispatcher*>> TaskQueue; + std::mutex DispatchMutex; + std::condition_variable WorkFinishedCV; + SmallVector WaitingFutures; + +public: + +/// @name ORC Promise/Future Classes +/// +/// ORC-aware promise/future implementation that integrates with the +/// TaskDispatcher system to allow efficient cooperative multitasking while +/// waiting for results (with certain limitations on what can be awaited). +/// Together they provide building blocks for a full async/await-like runtime +/// for llvm that supports multiple threads. +/// +/// Unlike std::promise/std::future alone, these classes can help dispatch other +/// tasks while waiting, preventing deadlocks and improving overall system +/// throughput. They have a similar API, though with some important differences +/// and some features simply not currently implemented. +/// +/// @{ + +template class promise; +template class future; + +/// Status for future/promise state +enum class FutureStatus : uint8_t { NotReady = 0, Ready = 1 }; + +/// @} + +/// Type-erased base class for futures, generally for scheduler use to avoid +/// needing virtual dispatches +class future_base { +public: + /// Check if the future is now ready with a value (precondition: get_promise() + /// must have been called) + bool ready() const { + if (!valid()) + report_fatal_error("ready() called before get_promise()"); + return state_->status_.load(std::memory_order_acquire) == FutureStatus::Ready; + } + + /// Check if the future is in a valid state (not moved-from and get_promise() called) + bool valid() const { return state_ != nullptr; } + + /// Wait for the future to be ready, helping with task dispatch + void wait(JuliaTaskDispatcher &D) { + // Keep helping with task dispatch until our future is ready + if (!ready()) { + D.work_until(*this); + if (state_->status_.load(std::memory_order_relaxed) != FutureStatus::Ready) + report_fatal_error( + "work_until() returned without this future being ready"); + } + } + +protected: + struct state_base { + std::atomic status_{FutureStatus::NotReady}; + }; + + future_base(state_base *state) : state_(state) {} + future_base() = default; + + /// Only allow deleting the future once it is invalid + ~future_base() { + if (state_) + report_fatal_error("get() must be called before future destruction (ensuring promise::set_value memory is valid)"); + } + + // Move constructor and assignment + future_base(future_base &&other) noexcept : state_(other.state_) { + other.state_ = nullptr; + } + + future_base &operator=(future_base &&other) noexcept { + if (this != &other) { + this->~future_base(); + state_ = other.state_; + other.state_ = nullptr; + } + return *this; + } + + state_base *state_; +}; + +/// TaskDispatcher-aware future class for cooperative await. +/// +/// @tparam T The type of value this future will provide. Use void for futures +/// that +/// signal completion without providing a value. +/// +/// This future implementation is similar to `std::future`, so most code can +/// transition to it easily. However, it differs from `std::future` in a few +/// key ways to be aware of: +/// - No exception support (or the overhead for it). +/// - The future is created before the promise, then the promise is created +/// from the future. +/// - The future is in an invalid state until get_promise() has been called. +/// - Waiting operations (get(&D), wait(&D)) help dispatch other tasks while +/// blocked, requiring an additional argument of which TaskDispatcher object +/// of where all associated work will be scheduled. +/// - While `wait` may be called multiple times and on multiple threads, all of +/// them must have returned before calling `get` on exactly one thread. +/// - Must call get() exactly once before destruction (enforced with +/// `report_fatal_error`) after each call to `get_promise`. Internal state is +/// freed when `get` returns, and allocated when `get_promise` is called. +/// +/// Other notable features, in common with `std::future`: +/// - Supports both value types and void specialization through the same +/// interface. +/// - Thread-safe through atomic operations. +/// - Provides acquire-release ordering with `std::promise::set_value()`. +/// - Concurrent access to any method (including to `ready`) on multiple threads +/// is not allowed. +/// - Holding any locks while calling `get()` is likely to lead to deadlock. +/// +/// @warning Users should avoid borrowing references to futures. References may +/// go out of scope and break the uniqueness contract, which may break the +/// soundness of the types. Always use move semantics or pass by value. + +template class future : public future_base { +public: + future() : future_base(nullptr) {} + future(const future &) = delete; + future &operator=(const future &) = delete; + future(future &&) = default; + future &operator=(future &&) = default; + + /// Get the value, helping with task dispatch while waiting. + /// This will destroy the underlying value, so this must be called exactly + /// once, which returns the future to the initial state. + T get(JuliaTaskDispatcher &D) { + if (!valid()) + report_fatal_error("get() must only be called once, after get_promise()"); + wait(D); + auto state_ = static_cast(this->state_); + this->state_ = nullptr; + return take_value(state_); + } + + /// Get the associated promise (must only be called once) + promise get_promise() { + if (valid()) + report_fatal_error("get_promise() can only be called once"); + auto state_ = new state(); + this->state_ = state_; + return promise(state_); + } + +private: + friend class promise; + + // Template the state struct with EBCO so that future has no wasted + // overhead for the value. The declaration of future_value_storage is far + // above here since GCC doesn't implement it properly when nested. + struct state : future_base::state_base, future_value_storage {}; + + template + typename std::enable_if::value, U>::type take_value(state *state_) { + T result = std::move(state_->value_); + state_->value_.~T(); + delete state_; + return result; + } + + template + typename std::enable_if::value, U>::type take_value(state *state_) { + delete state_; + } +}; + +/// TaskDispatcher-aware promise class that provides values to associated +/// futures. +/// +/// @tparam T The type of value this promise will provide. Use void for promises +/// that +/// signal completion without providing a value. +/// +/// This promise implementation provides the value-setting side of the +/// promise/future pair and integrates with the ORC TaskDispatcher system. Key +/// characteristics: +/// - Created from a future via get_promise() rather than creating the future from the promise. +/// - Must call get_future() on the thread that created it (it can be passed to another thread, but do not borrow a reference and use that to mutate it from another thread). +/// - Must call set_value() exactly once per `get_promise()` call to provide the result. +/// - Thread-safe from set_value to get. +/// - Move-only semantics to prevent accidental copying. +/// +/// The `promise` can usually be passed to another thread in one of two ways: +/// - With move semantics: +/// * `[P = F.get_promise()] () { P.set_value(); }` +/// * `[P = std::move(P)] () { P.set_value(); }` +/// * Advantages: clearer where `P` is owned, automatic deadlock detection +/// on destruction, +/// easier memory management if the future is returned from the function. +/// - By reference: +/// * `[&P] () { P.set_value(); }` +/// * Advantages: simpler memory management if the future is consumed in the +/// same function. +/// * Disadvantages: more difficult memory management if the future is +/// returned from the function, no deadlock detection. +/// +/// @warning Users should avoid borrowing references to promises. References may +/// go out of scope and break the uniqueness contract, which may break the +/// soundness of the types. Always use move semantics or pass by value. +/// +/// @par Error Handling: +/// The promise/future system uses report_fatal_error() for misuse: +/// - Calling set_value() more than once. +/// - Destroying a future without calling get(). +/// - Calling get() more than once on a future. +/// +/// @par Thread Safety: +/// - Each promise/future must only be accessed by one thread, as concurrent +/// calls to the API functions may result in crashes. +/// - Multiple threads can safely access different promise/future pairs. +/// - set_value() and get() operations are atomic and thread-safe. +/// - Move operations should only be performed by a single thread. +template class promise { + friend class future; + +public: + promise() : state_(nullptr) {} + + ~promise() { + // Assert proper promise lifecycle: ensure set_value was called if promise was valid. + // This can catch deadlocks where a promise is created but set_value() is + // never called, though only if the promise is moved from instead of + // borrowed from the frame with the future. + // Empty promises (state_ == nullptr) are allowed to be destroyed without calling set_value. + } + + promise(const promise &) = delete; + promise &operator=(const promise &) = delete; + + promise(promise &&other) noexcept + : state_(other.state_) { + other.state_ = nullptr; + } + + promise &operator=(promise &&other) noexcept { + if (this != &other) { + this->~promise(); + state_ = other.state_; + other.state_ = nullptr; + } + return *this; + } + + + /// Set the value (must only be called once) + // In C++20, this std::conditional weirdness can probably be replaced just + // with requires. It ensures that we don't try to define a method for `void&`, + // but that if the user calls set_value(v) for any value v that they get a + // member function error, instead of no member named 'value_'. + template + void + set_value(const typename std::conditional::value, + std::nullopt_t, T>::type &value) const { + assert(state_ && "set_value() can only be called once"); + new (&state_->value_) T(value); + state_->status_.store(FutureStatus::Ready, std::memory_order_release); + state_ = nullptr; + } + + template + void set_value(typename std::conditional::value, + std::nullopt_t, T>::type &&value) const { + assert(state_ && "set_value() can only be called once"); + new (&state_->value_) T(std::move(value)); + state_->status_.store(FutureStatus::Ready, std::memory_order_release); + state_ = nullptr; + } + + template + typename std::enable_if::value, void>::type + set_value(const std::nullopt_t &value) = delete; + + template + typename std::enable_if::value, void>::type + set_value(std::nullopt_t &&value) = delete; + + template + typename std::enable_if::value, void>::type set_value() const { + assert(state_ && "set_value() can only be called once"); + state_->status_.store(FutureStatus::Ready, std::memory_order_release); + state_ = nullptr; + } + + /// Swap with another promise + void swap(promise &other) noexcept { + using std::swap; + swap(state_, other.state_); + } + +private: + explicit promise(typename future::state *state) + : state_(state) {} + + mutable typename future::state *state_; +}; + +}; // class JuliaTaskDispatcher + +thread_local SmallVector, JuliaTaskDispatcher *>> JuliaTaskDispatcher::TaskQueue; + +void JuliaTaskDispatcher::dispatch(std::unique_ptr T) { + TaskQueue.push_back(std::pair(std::move(T), this)); +} + +void JuliaTaskDispatcher::shutdown() { + // Keep processing until no tasks belonging to this dispatcher remain + while (true) { + // Check if any task belongs to this dispatcher + auto it = std::find_if( + TaskQueue.begin(), TaskQueue.end(), + [this](const auto &TaskPair) { return TaskPair.second == this; }); + + // If no tasks belonging to this dispatcher, we're done + if (it == TaskQueue.end()) + return; + + // Create a future/promise pair to wait for completion of this task + future taskFuture; + // Replace the task with a GenericNamedTask that wraps the original task + // with a notification of completion that this thread can work_until. + auto originalTask = std::move(it->first); + it->first = makeGenericNamedTask( + [originalTask = std::move(originalTask), + taskPromise = taskFuture.get_promise()]() { + originalTask->run(); + taskPromise.set_value(); + }, + "Shutdown task marker"); + + // Wait for the task to complete + taskFuture.get(*this); + } +} + +void JuliaTaskDispatcher::work_until(future_base &F) { + while (!F.ready()) { + // First, process any tasks in our local queue + // Process in LIFO order (most recently added first) to avoid deadlocks + // when tasks have dependencies on each other + while (!TaskQueue.empty()) { + { + auto TaskPair = std::move(TaskQueue.back()); + TaskQueue.pop_back(); + TaskPair.first->run(); + } + + // Notify any threads that might be waiting for work to complete + { + std::lock_guard Lock(DispatchMutex); + bool ShouldNotify = llvm::any_of( + WaitingFutures, [](future_base *F) { return F->ready(); }); + if (ShouldNotify) { + WaitingFutures.clear(); + WorkFinishedCV.notify_all(); + } + } + + // Check if our future is now ready + if (F.ready()) + return; + } + + // If we get here, our queue is empty but the future isn't ready + // We need to wait for other threads to finish work that should complete our + // future + { + std::unique_lock Lock(DispatchMutex); + WaitingFutures.push_back(&F); + WorkFinishedCV.wait(Lock, [&F]() { return F.ready(); }); + } + } +} + +} // End namespace + +namespace std { +template +void swap(::JuliaTaskDispatcher::promise &lhs, ::JuliaTaskDispatcher::promise &rhs) noexcept { + lhs.swap(rhs); +} +} // End namespace std + +// n.b. this actually is sometimes a safepoint +Expected +safelookup(ExecutionSession &ES, + const JITDylibSearchOrder &SearchOrder, + SymbolLookupSet Symbols, LookupKind K = LookupKind::Static, + SymbolState RequiredState = SymbolState::Ready, + RegisterDependenciesFunction RegisterDependencies = NoDependenciesToRegister) JL_NOTSAFEPOINT { + JuliaTaskDispatcher::future> PromisedFuture; + auto NotifyComplete = [PromisedResult = PromisedFuture.get_promise()](Expected R) { + PromisedResult.set_value(std::move(R)); + }; + ES.lookup(K, SearchOrder, std::move(Symbols), RequiredState, + std::move(NotifyComplete), RegisterDependencies); + return PromisedFuture.get(static_cast(ES.getExecutorProcessControl().getDispatcher())); +} + +Expected +safelookup(ExecutionSession &ES, + const JITDylibSearchOrder &SearchOrder, + SymbolStringPtr Name, + SymbolState RequiredState = SymbolState::Ready) JL_NOTSAFEPOINT { + SymbolLookupSet Names({Name}); + + if (auto ResultMap = safelookup(ES, SearchOrder, std::move(Names), LookupKind::Static, + RequiredState, NoDependenciesToRegister)) { + assert(ResultMap->size() == 1 && "Unexpected number of results"); + assert(ResultMap->count(Name) && "Missing result for symbol"); + return std::move(ResultMap->begin()->second); + } else + return ResultMap.takeError(); +} + +Expected +safelookup(ExecutionSession &ES, + ArrayRef SearchOrder, SymbolStringPtr Name, + SymbolState RequiredState = SymbolState::Ready) JL_NOTSAFEPOINT { + return safelookup(ES, makeJITDylibSearchOrder(SearchOrder), Name, RequiredState); +} + +Expected +safelookup(ExecutionSession &ES, + ArrayRef SearchOrder, StringRef Name, + SymbolState RequiredState = SymbolState::Ready) JL_NOTSAFEPOINT { + return safelookup(ES, SearchOrder, ES.intern(Name), RequiredState); +} diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 86ef39457a845..2e86d8e3ab824 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -1,6 +1,8 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "llvm-gc-interface-passes.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/Support/Casting.h" #define DEBUG_TYPE "late_lower_gcroot" @@ -163,12 +165,12 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac (void)LI; break; } - else if (auto II = dyn_cast(CurrentV)) { - // Some intrinsics behave like LoadInst followed by a SelectInst - // This should never happen in a derived addrspace (since those cannot be stored to memory) - // so we don't need to lift these operations, but we do need to check if it's loaded and continue walking the base pointer + else if (auto *II = dyn_cast(CurrentV)) { if (II->getIntrinsicID() == Intrinsic::masked_load || II->getIntrinsicID() == Intrinsic::masked_gather) { + // Some intrinsics behave like LoadInst followed by a SelectInst + // This should never happen in a derived addrspace (since those cannot be stored to memory) + // so we don't need to lift these operations, but we do need to check if it's loaded and continue walking the base pointer if (auto VTy = dyn_cast(II->getType())) { if (hasLoadedTy(VTy->getElementType())) { Value *Mask = II->getOperand(2); @@ -197,6 +199,24 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac // In general a load terminates a walk break; } + else if (II->getIntrinsicID() == Intrinsic::vector_extract) { + if (auto VTy = dyn_cast(II->getType())) { + if (hasLoadedTy(VTy->getElementType())) { + Value *Idx = II->getOperand(1); + if (!isa(Idx)) { + assert(isa(Idx) && "unimplemented"); + (void)Idx; + } + CurrentV = II->getOperand(0); + fld_idx = -1; + continue; + } + } + break; + } else { + // Unknown Intrinsic + break; + } } else if (auto CI = dyn_cast(CurrentV)) { auto callee = CI->getCalledFunction(); @@ -204,9 +224,11 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac CurrentV = CI->getArgOperand(0); continue; } + // Unknown Call break; } else { + // Unknown Instruction break; } } @@ -522,6 +544,22 @@ SmallVector LateLowerGCFrame::NumberAllBase(State &S, Value *CurrentV) { Numbers = NumberAll(S, IEI->getOperand(0)); int ElNumber = Number(S, IEI->getOperand(1)); Numbers[idx] = ElNumber; + // C++17 + // } else if (auto *II = dyn_cast(CurrentV); II && II->getIntrinsicID() == Intrinsic::vector_insert) { + } else if (isa(CurrentV) && cast(CurrentV)->getIntrinsicID() == Intrinsic::vector_insert) { + auto *II = dyn_cast(CurrentV); + // Vector insert is a bit like a shuffle so use the same approach + SmallVector Numbers1 = NumberAll(S, II->getOperand(0)); + SmallVector Numbers2 = NumberAll(S, II->getOperand(1)); + unsigned first_idx = cast(II->getOperand(2))->getZExtValue(); + for (unsigned i = 0; i < Numbers1.size(); ++i) { + if (i < first_idx) + Numbers.push_back(Numbers1[i]); + else if (i - first_idx < Numbers2.size()) + Numbers.push_back(Numbers2[i - first_idx]); + else + Numbers.push_back(Numbers1[i]); + } } else if (auto *IVI = dyn_cast(CurrentV)) { Numbers = NumberAll(S, IVI->getAggregateOperand()); auto Tracked = TrackCompositeType(IVI->getType()); @@ -630,16 +668,6 @@ SmallVector LateLowerGCFrame::NumberAll(State &S, Value *V) { } -static void MaybeResize(BBState &BBS, unsigned Idx) { - /* - if (BBS.Defs.size() <= Idx) { - BBS.Defs.resize(Idx + 1); - BBS.UpExposedUses.resize(Idx + 1); - BBS.PhiOuts.resize(Idx + 1); - } - */ -} - static bool HasBitSet(const LargeSparseBitVector &BV, unsigned Bit) { return BV.test(Bit); } @@ -648,47 +676,47 @@ static bool HasBitSet(const BitVector &BV, unsigned Bit) { return Bit < BV.size() && BV[Bit]; } -static void NoteDef(State &S, BBState &BBS, int Num, const ArrayRef &SafepointsSoFar) { +static void NoteDef(State &S, BBState &BBS, int Num) { assert(Num >= 0); - MaybeResize(BBS, Num); assert(!BBS.Defs.test(Num) && "SSA Violation or misnumbering?"); BBS.Defs.set(Num); BBS.UpExposedUses.reset(Num); // This value could potentially be live at any following safe point // if it ends up live out, so add it to the LiveIfLiveOut lists for all // following safepoints. - for (int Safepoint : SafepointsSoFar) { - S.LiveIfLiveOut[Safepoint].push_back(Num); - } + if (BBS.HasSafepoint) + for (int Safepoint = BBS.FirstSafepoint; Safepoint >= BBS.LastSafepoint; --Safepoint) + S.LiveIfLiveOut[Safepoint].push_back(Num); } -void LateLowerGCFrame::MaybeNoteDef(State &S, BBState &BBS, Value *Def, - const ArrayRef &SafepointsSoFar, +bool LateLowerGCFrame::MaybeNoteDef(State &S, BBState &BBS, Value *Def, SmallVector &&RefinedPtr) { Type *RT = Def->getType(); if (isa(RT)) { if (!isSpecialPtr(RT)) - return; + return false; assert(isTrackedValue(Def) && "Returned value of GC interest, but not tracked?"); int Num = Number(S, Def); - NoteDef(S, BBS, Num, SafepointsSoFar); + NoteDef(S, BBS, Num); if (!RefinedPtr.empty()) S.Refinements[Num] = std::move(RefinedPtr); + return true; } else { SmallVector Nums = NumberAll(S, Def); for (int Num : Nums) { - NoteDef(S, BBS, Num, SafepointsSoFar); + NoteDef(S, BBS, Num); if (!RefinedPtr.empty()) S.Refinements[Num] = RefinedPtr; } + return !Nums.empty(); } } static int NoteSafepoint(State &S, BBState &BBS, CallInst *CI, SmallVectorImpl &CalleeRoots) { + assert(BBS.FirstSafepoint == -1 || BBS.FirstSafepoint == S.MaxSafepointNumber); int Number = ++S.MaxSafepointNumber; - S.SafepointNumbering[CI] = Number; - S.ReverseSafepointNumbering.push_back(CI); + S.SafepointNumbering.push_back(CI); // Note which pointers are upward exposed live here. They need to be // considered live at this safepoint even when they have a def earlier // in this BB (i.e. even when they don't participate in the dataflow @@ -696,6 +724,10 @@ static int NoteSafepoint(State &S, BBState &BBS, CallInst *CI, SmallVectorImpl{}); S.CalleeRoots.push_back(std::move(CalleeRoots)); + BBS.HasSafepoint = true; + if (BBS.LastSafepoint == -1) + BBS.LastSafepoint = Number; + BBS.FirstSafepoint = Number; return Number; } @@ -715,7 +747,6 @@ void LateLowerGCFrame::NoteUse(State &S, BBState &BBS, Value *V, LargeSparseBitV int Num = Number(S, V); if (Num < 0) return; - MaybeResize(BBS, Num); Uses.set(Num); } } else { @@ -723,7 +754,6 @@ void LateLowerGCFrame::NoteUse(State &S, BBState &BBS, Value *V, LargeSparseBitV for (int Num : Nums) { if (Num < 0) continue; - MaybeResize(BBS, Num); Uses.set(Num); } } @@ -744,7 +774,7 @@ void RecursivelyVisit(callback f, Value *V) { if (isa(TheUser) || isa(TheUser) || isa(TheUser) || isa(TheUser) || // TODO: should these be removed from this list? isa(TheUser) || isa(TheUser) || - isa(TheUser) || // ICmpEQ/ICmpNE can be used with ptr types + isa(TheUser) || isa(TheUser)|| // ICmpEQ/ICmpNE can be used with ptr types isa(TheUser) || isa(TheUser)) continue; if (isa(TheUser) || isa(TheUser) || isa(TheUser)) { @@ -924,15 +954,6 @@ static uint64_t getLoadValueAlign(LoadInst *LI) return mdconst::extract(md->getOperand(0))->getLimitedValue(); } -static bool LooksLikeFrameRef(Value *V) { - if (isSpecialPtr(V->getType())) - return false; - V = V->stripInBoundsOffsets(); - if (isSpecialPtr(V->getType())) - return false; - return isa(V); -} - SmallVector LateLowerGCFrame::GetPHIRefinements(PHINode *Phi, State &S) { // The returned vector can violate the domination property of the Refinements map. @@ -1167,6 +1188,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { SmallVector PHINumbers; for (BasicBlock &BB : F) { BBState &BBS = S.BBStates[&BB]; + // Avoid tracking safepoints until we reach the first instruction the defines a value. for (auto it = BB.rbegin(); it != BB.rend(); ++it) { Instruction &I = *it; if (CallInst *CI = dyn_cast(&I)) { @@ -1198,21 +1220,28 @@ State LateLowerGCFrame::LocalScan(Function &F) { } } } + if (II->getIntrinsicID() == Intrinsic::vector_extract || II->getIntrinsicID() == Intrinsic::vector_insert) { + // These are not real defs + continue; + } } auto callee = CI->getCalledFunction(); if (callee && callee == typeof_func) { - MaybeNoteDef(S, BBS, CI, BBS.Safepoints, SmallVector{-2}); + MaybeNoteDef(S, BBS, CI, SmallVector{-2}); } else if (callee && callee->getName() == "julia.gc_loaded") { continue; } else { - MaybeNoteDef(S, BBS, CI, BBS.Safepoints); + if (MaybeNoteDef(S, BBS, CI)) + BBS.FirstSafepointAfterFirstDef = BBS.FirstSafepoint; } + bool HasDefBefore = false; if (CI->hasStructRetAttr()) { Type *ElT = getAttributeAtIndex(CI->getAttributes(), 1, Attribute::StructRet).getValueAsType(); auto tracked = CountTrackedPointers(ElT, true); if (tracked.count) { + HasDefBefore = true; auto allocas_opt = FindSretAllocas((CI->arg_begin()[0])->stripInBoundsOffsets()); // We know that with the right optimizations we can forward a sret directly from an argument // This hasn't been seen without adding IPO effects to julia functions but it's possible we need to handle that too @@ -1255,57 +1284,54 @@ State LateLowerGCFrame::LocalScan(Function &F) { } } NoteOperandUses(S, BBS, I); - if (CI->canReturnTwice()) { - S.ReturnsTwice.push_back(CI); - } - if (callee) { - if (callee == gc_preserve_begin_func) { - SmallVector args; - for (Use &U : CI->args()) { - Value *V = U; - if (isa(V)) - continue; - if (isa(V->getType())) { - if (isSpecialPtr(V->getType())) { - int Num = Number(S, V); - if (Num >= 0) + if (!CI->canReturnTwice()) { + if (callee) { + if (callee == gc_preserve_begin_func) { + SmallVector args; + for (Use &U : CI->args()) { + Value *V = U; + if (isa(V)) + continue; + if (isa(V->getType())) { + if (isSpecialPtr(V->getType())) { + int Num = Number(S, V); + if (Num >= 0) + args.push_back(Num); + } + } else { + SmallVector Nums = NumberAll(S, V); + for (int Num : Nums) { + if (Num < 0) + continue; args.push_back(Num); - } - } else { - SmallVector Nums = NumberAll(S, V); - for (int Num : Nums) { - if (Num < 0) - continue; - args.push_back(Num); + } } } + S.GCPreserves[CI] = args; + continue; + } + // Known functions emitted in codegen that are not safepoints + if (callee == pointer_from_objref_func || callee == gc_preserve_begin_func || + callee == gc_preserve_end_func || callee == typeof_func || + callee == pgcstack_getter || callee->getName() == XSTR(jl_egal__unboxed) || + callee->getName() == XSTR(jl_lock_value) || callee->getName() == XSTR(jl_unlock_value) || + callee->getName() == XSTR(jl_lock_field) || callee->getName() == XSTR(jl_unlock_field) || + callee == write_barrier_func || callee == gc_loaded_func || callee == pop_handler_noexcept_func || + callee->getName() == "memcmp") { + continue; + } + if (callee->getMemoryEffects().onlyReadsMemory() || + callee->getMemoryEffects().onlyAccessesArgPointees()) { + continue; } - S.GCPreserves[CI] = args; - continue; } - // Known functions emitted in codegen that are not safepoints - if (callee == pointer_from_objref_func || callee == gc_preserve_begin_func || - callee == gc_preserve_end_func || callee == typeof_func || - callee == pgcstack_getter || callee->getName() == XSTR(jl_egal__unboxed) || - callee->getName() == XSTR(jl_lock_value) || callee->getName() == XSTR(jl_unlock_value) || - callee->getName() == XSTR(jl_lock_field) || callee->getName() == XSTR(jl_unlock_field) || - callee == write_barrier_func || callee == gc_loaded_func || callee == pop_handler_noexcept_func || - callee->getName() == "memcmp") { + if (isa(CI)) + // Intrinsics are never safepoints. continue; - } - if (callee->getMemoryEffects().onlyReadsMemory() || - callee->getMemoryEffects().onlyAccessesArgPointees()) { + auto effects = CI->getMemoryEffects(); + if (effects.onlyAccessesArgPointees() || effects.onlyReadsMemory()) + // Readonly functions and functions that cannot change GC state (which is inaccessiblemem) are not safepoints continue; - } - if (MemTransferInst *MI = dyn_cast(CI)) { - MaybeTrackDst(S, MI); - } - } - if (isa(CI) || - CI->getMemoryEffects().onlyAccessesArgPointees() || - CI->getMemoryEffects().onlyReadsMemory()) { - // Intrinsics are never safepoints. - continue; } SmallVector CalleeRoots; for (Use &U : CI->args()) { @@ -1326,10 +1352,15 @@ State LateLowerGCFrame::LocalScan(Function &F) { CalleeRoots.push_back(Num); } int SafepointNumber = NoteSafepoint(S, BBS, CI, CalleeRoots); - BBS.HasSafepoint = true; - BBS.TopmostSafepoint = SafepointNumber; - BBS.Safepoints.push_back(SafepointNumber); - } else if (LoadInst *LI = dyn_cast(&I)) { + if (CI->canReturnTwice()) { + S.ReturnsTwice.push_back(SafepointNumber); + HasDefBefore = true; + } + if (HasDefBefore) // With sret, the Def happens before the instruction instead of after + BBS.FirstSafepointAfterFirstDef = SafepointNumber; + continue; + } + if (LoadInst *LI = dyn_cast(&I)) { // If this is a load from an immutable, we know that // this object will always be rooted as long as the // object we're loading from is, so we can refine uses @@ -1337,14 +1368,10 @@ State LateLowerGCFrame::LocalScan(Function &F) { // from. SmallVector RefinedPtr{}; Type *Ty = LI->getType()->getScalarType(); + bool refined_globally = false; bool task_local = false; if (isLoadFromImmut(LI) && isSpecialPtr(LI->getPointerOperand()->getType())) { RefinedPtr.push_back(Number(S, LI->getPointerOperand())); - } else if (LI->getType()->isPointerTy() && - isSpecialPtr(Ty) && - LooksLikeFrameRef(LI->getPointerOperand())) { - // Loads from a jlcall argument array - RefinedPtr.push_back(-1); } else if (isLoadFromConstGV(LI, task_local)) { // If this is a const load from a global, @@ -1352,21 +1379,26 @@ State LateLowerGCFrame::LocalScan(Function &F) { // If this is a task local constant, we don't need to root it within the // task but we do need to issue write barriers for when the current task dies. RefinedPtr.push_back(task_local ? -1 : -2); + refined_globally = true; } if (!hasLoadedTy(Ty)) - MaybeNoteDef(S, BBS, LI, BBS.Safepoints, std::move(RefinedPtr)); + if (MaybeNoteDef(S, BBS, LI, std::move(RefinedPtr))) + if (!refined_globally) + BBS.FirstSafepointAfterFirstDef = BBS.FirstSafepoint; NoteOperandUses(S, BBS, I); } else if (auto *LI = dyn_cast(&I)) { Type *Ty = LI->getNewValOperand()->getType()->getScalarType(); if (!Ty->isPointerTy() || Ty->getPointerAddressSpace() != AddressSpace::Loaded) { - MaybeNoteDef(S, BBS, LI, BBS.Safepoints); + if (MaybeNoteDef(S, BBS, LI)) + BBS.FirstSafepointAfterFirstDef = BBS.FirstSafepoint; } NoteOperandUses(S, BBS, I); // TODO: do we need MaybeTrackStore(S, LI); } else if (auto *LI = dyn_cast(&I)) { Type *Ty = LI->getType()->getScalarType(); if (!Ty->isPointerTy() || Ty->getPointerAddressSpace() != AddressSpace::Loaded) { - MaybeNoteDef(S, BBS, LI, BBS.Safepoints); + if (MaybeNoteDef(S, BBS, LI)) + BBS.FirstSafepointAfterFirstDef = BBS.FirstSafepoint; } NoteOperandUses(S, BBS, I); // TODO: do we need MaybeTrackStore(S, LI); @@ -1382,7 +1414,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { Number(S, SI->getFalseValue()) }; } - MaybeNoteDef(S, BBS, SI, BBS.Safepoints, std::move(RefinedPtr)); + MaybeNoteDef(S, BBS, SI, std::move(RefinedPtr)); NoteOperandUses(S, BBS, I); } else if (tracked.count) { // We need to insert extra selects for the GC roots @@ -1396,7 +1428,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { if (isa(Phi->getType())) // TODO: Vector refinements PHIRefinements = GetPHIRefinements(Phi, S); - MaybeNoteDef(S, BBS, Phi, BBS.Safepoints, std::move(PHIRefinements)); + MaybeNoteDef(S, BBS, Phi, std::move(PHIRefinements)); if (isa(Phi->getType())) { PHINumbers.push_back(Number(S, Phi)); } else { @@ -1428,7 +1460,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { RefinedPtr.push_back(task_local ? -1 : -2); } } - MaybeNoteDef(S, BBS, ASCI, BBS.Safepoints, std::move(RefinedPtr)); + MaybeNoteDef(S, BBS, ASCI, std::move(RefinedPtr)); } } else if (auto *AI = dyn_cast(&I)) { Type *ElT = AI->getAllocatedType(); @@ -1439,7 +1471,6 @@ State LateLowerGCFrame::LocalScan(Function &F) { } // Pre-seed the dataflow variables; BBS.LiveIn = BBS.UpExposedUses; - BBS.Done = true; } FixUpRefinements(PHINumbers, S); return S; @@ -1538,35 +1569,6 @@ SmallVector ExtractTrackedValues(Value *Src, Type *STy, bool isptr, I // return Ptrs.size(); //} -// turn a memcpy into a set of loads -void LateLowerGCFrame::MaybeTrackDst(State &S, MemTransferInst *MI) { - //Value *Dst = MI->getRawDest()->stripInBoundsOffsets(); - //if (AllocaInst *AI = dyn_cast(Dst)) { - // Type *STy = AI->getAllocatedType(); - // if (!AI->isStaticAlloca() || (isa(STy) && STy->getPointerAddressSpace() == AddressSpace::Tracked) || S.ArrayAllocas.count(AI)) - // return; // already numbered this - // auto tracked = CountTrackedPointers(STy); - // unsigned nroots = tracked.count * cast(AI->getArraySize())->getZExtValue(); - // if (nroots) { - // assert(!tracked.derived); - // if (!tracked.all) { - // // materialize shadow LoadInst and StoreInst ops to make a copy of just the tracked values inside - // //assert(MI->getLength() == DL.getTypeAllocSize(AI->getAllocatedType()) && !AI->isArrayAllocation()); // XXX: handle partial copy - // Value *Src = MI->getSource(); - // Src = new BitCastInst(Src, STy->getPointerTo(MI->getSourceAddressSpace()), "", MI); - // auto &Shadow = S.ShadowAllocas[AI]; - // if (!Shadow) - // Shadow = new AllocaInst(ArrayType::get(T_prjlvalue, nroots), 0, "", MI); - // AI = Shadow; - // unsigned count = TrackWithShadow(Src, STy, true, AI, IRBuilder<>(MI)); - // assert(count == tracked.count); (void)count; - // } - // S.ArrayAllocas[AI] = nroots; - // } - //} - //// TODO: else??? -} - void LateLowerGCFrame::MaybeTrackStore(State &S, StoreInst *I) { Value *PtrBase = I->getPointerOperand()->stripInBoundsOffsets(); auto tracked = CountTrackedPointers(I->getValueOperand()->getType()); @@ -1640,10 +1642,11 @@ void LateLowerGCFrame::ComputeLiveness(State &S) { // For debugging JL_USED_FUNC static void dumpSafepointsForBBName(Function &F, State &S, const char *BBName) { - for (auto it : S.SafepointNumbering) { - if (it.first->getParent()->getName() == BBName) { - dbgs() << "Live at " << *it.first << "\n"; - LargeSparseBitVector &LS = S.LiveSets[it.second]; + for (Instruction *&it : S.SafepointNumbering) { + if (it->getParent()->getName() == BBName) { + int idx = &it - S.SafepointNumbering.begin(); + dbgs() << "Live at " << idx << "\n"; + LargeSparseBitVector &LS = S.LiveSets[idx]; for (auto Idx : LS) { dbgs() << "\t"; S.ReversePtrNumbering[Idx]->printAsOperand(dbgs()); @@ -1722,9 +1725,8 @@ void LateLowerGCFrame::RefineLiveSet(LargeSparseBitVector &LS, State &S, ArrayRe void LateLowerGCFrame::ComputeLiveSets(State &S) { // Iterate over all safe points. Add to live sets all those variables that // are now live across their parent block. - for (auto it : S.SafepointNumbering) { - int idx = it.second; - Instruction *Safepoint = it.first; + for (Instruction *&Safepoint : S.SafepointNumbering) { + int idx = &Safepoint - S.SafepointNumbering.begin(); BasicBlock *BB = Safepoint->getParent(); BBState &BBS = S.BBStates[BB]; LargeSparseBitVector LiveAcross = BBS.LiveIn; @@ -1764,8 +1766,9 @@ void LateLowerGCFrame::ComputeLiveSets(State &S) { } // Compute the interference graph S.Neighbors.resize(S.MaxPtrNumber+1); - for (auto it : S.SafepointNumbering) { - const LargeSparseBitVector &LS = S.LiveSets[it.second]; + for (Instruction *&Safepoint : S.SafepointNumbering) { + int idx = &Safepoint - S.SafepointNumbering.begin(); + const LargeSparseBitVector &LS = S.LiveSets[idx]; for (int idx : LS) { S.Neighbors[idx] |= LS; } @@ -1852,8 +1855,7 @@ std::pair, int> LateLowerGCFrame::ColorRoots(const State &S) int PreAssignedColors = 0; /* First assign permanent slots to things that need them due to returns_twice */ - for (auto it : S.ReturnsTwice) { - int Num = S.SafepointNumbering.at(it); + for (int Num : S.ReturnsTwice) { const LargeSparseBitVector &LS = S.LiveSets[Num]; for (int Idx : LS) { if (Colors[Idx] == -1) @@ -1914,11 +1916,6 @@ Value *LateLowerGCFrame::EmitLoadTag(IRBuilder<> &builder, Type *T_size, Value * return load; } -// Enable this optimization only on LLVM 4.0+ since this cause LLVM to optimize -// constant store loop to produce a `memset_pattern16` with a global variable -// that's initialized by `addrspacecast`. Such a global variable is not supported by the backend. -// This is not a problem on 4.0+ since that transformation (in loop-idiom) is disabled -// for NI pointers. static SmallVector *FindRefinements(Value *V, State *S) { if (!S) @@ -2313,7 +2310,9 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { return ChangesMade; } -static void AddInPredLiveOuts(BasicBlock *BB, LargeSparseBitVector &LiveIn, State &S) +// Compute the set of all objects that are live in from all predecessors +// TODO: reset any slots that contain values which are only live from some predecessors +static void AddInPredecessorLiveOuts(BasicBlock *BB, LargeSparseBitVector &LiveIn, State &S) { bool First = true; std::set Visited; @@ -2334,7 +2333,7 @@ static void AddInPredLiveOuts(BasicBlock *BB, LargeSparseBitVector &LiveIn, Stat WorkList.push_back(Pred); continue; } else { - int LastSP = S.BBStates[Pred].Safepoints.front(); + int LastSP = S.BBStates[Pred].LastSafepoint; if (First) { LiveIn |= S.LiveSets[LastSP]; First = false; @@ -2366,7 +2365,11 @@ void LateLowerGCFrame::PlaceGCFrameStore(State &S, unsigned R, unsigned MinColor // free to rewrite them if convenient. We need to change // it back here for the store. assert(Val->getType() == T_prjlvalue); +#if JL_LLVM_VERSION >= 200000 new StoreInst(Val, slotAddress, InsertBefore->getIterator()); +#else + new StoreInst(Val, slotAddress, InsertBefore); +#endif } void LateLowerGCFrame::PlaceGCFrameReset(State &S, unsigned R, unsigned MinColorRoot, @@ -2376,10 +2379,18 @@ void LateLowerGCFrame::PlaceGCFrameReset(State &S, unsigned R, unsigned MinColor auto slotAddress = CallInst::Create( getOrDeclare(jl_intrinsics::getGCFrameSlot), {GCFrame, ConstantInt::get(Type::getInt32Ty(InsertBefore->getContext()), Colors[R] + MinColorRoot)}, +#if JL_LLVM_VERSION >= 200000 + "gc_slot_addr_" + StringRef(std::to_string(Colors[R] + MinColorRoot)), InsertBefore->getIterator()); +#else "gc_slot_addr_" + StringRef(std::to_string(Colors[R] + MinColorRoot)), InsertBefore); +#endif // Reset the slot to NULL. Value *Val = ConstantPointerNull::get(T_prjlvalue); +#if JL_LLVM_VERSION >= 200000 + new StoreInst(Val, slotAddress, InsertBefore->getIterator()); +#else new StoreInst(Val, slotAddress, InsertBefore); +#endif } void LateLowerGCFrame::PlaceGCFrameStores(State &S, unsigned MinColorRoot, @@ -2387,20 +2398,18 @@ void LateLowerGCFrame::PlaceGCFrameStores(State &S, unsigned MinColorRoot, { for (auto &BB : *S.F) { const BBState &BBS = S.BBStates[&BB]; - if (!BBS.HasSafepoint) { + if (!BBS.HasSafepoint) continue; - } LargeSparseBitVector LiveIn; - AddInPredLiveOuts(&BB, LiveIn, S); + AddInPredecessorLiveOuts(&BB, LiveIn, S); const LargeSparseBitVector *LastLive = &LiveIn; - for(auto rit = BBS.Safepoints.rbegin(); - rit != BBS.Safepoints.rend(); ++rit ) { - const LargeSparseBitVector &NowLive = S.LiveSets[*rit]; + for (int Safepoint = BBS.FirstSafepoint; Safepoint >= BBS.LastSafepoint; --Safepoint) { + const LargeSparseBitVector &NowLive = S.LiveSets[Safepoint]; // reset slots which are no longer alive for (int Idx : *LastLive) { if (Colors[Idx] >= PreAssignedColors && !HasBitSet(NowLive, Idx)) { PlaceGCFrameReset(S, Idx, MinColorRoot, Colors, GCFrame, - S.ReverseSafepointNumbering[*rit]); + S.SafepointNumbering[Safepoint]); } } // store values which are alive in this safepoint but @@ -2408,7 +2417,7 @@ void LateLowerGCFrame::PlaceGCFrameStores(State &S, unsigned MinColorRoot, for (int Idx : NowLive) { if (!HasBitSet(*LastLive, Idx)) { PlaceGCFrameStore(S, Idx, MinColorRoot, Colors, GCFrame, - S.ReverseSafepointNumbering[*rit]); + S.SafepointNumbering[Safepoint]); } } LastLive = &NowLive; @@ -2432,7 +2441,7 @@ void LateLowerGCFrame::PlaceRootsAndUpdateCalls(ArrayRef Colors, int PreAss getOrDeclare(jl_intrinsics::newGCFrame), {ConstantInt::get(T_int32, 0)}, "gcframe"); - gcframe->insertBefore(&*F->getEntryBlock().begin()); + gcframe->insertBefore(F->getEntryBlock().begin()); auto pushGcframe = CallInst::Create( getOrDeclare(jl_intrinsics::pushGCFrame), @@ -2530,7 +2539,11 @@ void LateLowerGCFrame::PlaceRootsAndUpdateCalls(ArrayRef Colors, int PreAss assert(Elem->getType() == T_prjlvalue); //auto Idxs = ArrayRef(Tracked[i]); //Value *Elem = ExtractScalar(Base, true, Idxs, SI); +#if JL_LLVM_VERSION >= 200000 Value *shadowStore = new StoreInst(Elem, slotAddress, SI->getIterator()); +#else + Value *shadowStore = new StoreInst(Elem, slotAddress, SI); +#endif (void)shadowStore; // TODO: shadowStore->setMetadata(LLVMContext::MD_tbaa, tbaa_gcframe); AllocaSlot++; @@ -2548,7 +2561,11 @@ void LateLowerGCFrame::PlaceRootsAndUpdateCalls(ArrayRef Colors, int PreAss auto popGcframe = CallInst::Create( getOrDeclare(jl_intrinsics::popGCFrame), {gcframe}); +#if JL_LLVM_VERSION >= 200000 + popGcframe->insertBefore(BB.getTerminator()->getIterator()); +#else popGcframe->insertBefore(BB.getTerminator()); +#endif } } } @@ -2560,16 +2577,25 @@ bool LateLowerGCFrame::runOnFunction(Function &F, bool *CFGModified) { LLVM_DEBUG(dbgs() << "GC ROOT PLACEMENT: Processing function " << F.getName() << "\n"); pgcstack = getPGCstack(F); - if (!pgcstack) - return CleanupIR(F, nullptr, CFGModified); - - State S = LocalScan(F); - ComputeLiveness(S); - auto Colors = ColorRoots(S); - std::map> CallFrames; // = OptimizeCallFrames(S, Ordering); - PlaceRootsAndUpdateCalls(Colors.first, Colors.second, S, CallFrames); - CleanupIR(F, &S, CFGModified); - + if (pgcstack) { + State S = LocalScan(F); + // If there is no safepoint after the first reachable def, then we don't need any roots (even those for allocas) + if (std::any_of(S.BBStates.begin(), S.BBStates.end(), + [&F](auto BBS) { + if (BBS.first == &F.getEntryBlock()) + return BBS.second.FirstSafepointAfterFirstDef != -1; + return BBS.second.HasSafepoint; + })) { + ComputeLiveness(S); + auto Colors = ColorRoots(S); + std::map> CallFrames; // = OptimizeCallFrames(S, Ordering); + PlaceRootsAndUpdateCalls(Colors.first, Colors.second, S, CallFrames); + } + CleanupIR(F, &S, CFGModified); + } + else { + CleanupIR(F, nullptr, CFGModified); + } // We lower the julia.gc_alloc_bytes intrinsic in this pass to insert slowpath/fastpath blocks for MMTk // For now, we do nothing for the Stock GC diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index dbdc0ac375a75..1278a5475adbe 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -18,6 +18,9 @@ #include "julia_assert.h" #include "llvm-pass-helpers.h" +#define STR(csym) #csym +#define XSTR(csym) STR(csym) + using namespace llvm; JuliaPassContext::JuliaPassContext() @@ -85,10 +88,11 @@ llvm::Value *JuliaPassContext::getPGCstack(llvm::Function &F) const } } } - if (F.getCallingConv() == CallingConv::Swift) { - for (auto &arg : F.args()) { - if (arg.hasSwiftSelfAttr()) - return &arg; + for (auto &arg : F.args()) { + // Check for the "gcstack" attribute + AttributeSet attrs = F.getAttributes().getParamAttrs(arg.getArgNo()); + if (attrs.hasAttribute("gcstack")) { + return &arg; } } return nullptr; diff --git a/src/llvm-propagate-addrspaces.cpp b/src/llvm-propagate-addrspaces.cpp index 55c9e731e1525..5be8a30e3405e 100644 --- a/src/llvm-propagate-addrspaces.cpp +++ b/src/llvm-propagate-addrspaces.cpp @@ -289,7 +289,11 @@ bool propagateJuliaAddrspaces(Function &F) { PropagateJuliaAddrspacesVisitor visitor; visitor.visit(F); for (auto it : visitor.ToInsert) +#if JL_LLVM_VERSION >= 200000 + it.first->insertBefore(it.second->getIterator()); +#else it.first->insertBefore(it.second); +#endif for (Instruction *I : visitor.ToDelete) I->eraseFromParent(); visitor.ToInsert.clear(); diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 807d3cb95422d..a7bc79afd3eb4 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -179,7 +179,11 @@ void LowerPTLS::fix_pgcstack_use(CallInst *pgcstack, Function *pgcstack_getter, adoptFunc->copyMetadata(pgcstack_getter, 0); } adopt->setCalledFunction(adoptFunc); +#if JL_LLVM_VERSION >= 200000 + adopt->insertBefore(slowTerm->getIterator()); +#else adopt->insertBefore(slowTerm); +#endif phi->addIncoming(adopt, slowTerm->getParent()); // emit fast branch code builder.SetInsertPoint(fastTerm->getParent()); @@ -237,7 +241,11 @@ void LowerPTLS::fix_pgcstack_use(CallInst *pgcstack, Function *pgcstack_getter, builder.SetInsertPoint(pgcstack); auto phi = builder.CreatePHI(T_pppjlvalue, 2, "pgcstack"); pgcstack->replaceAllUsesWith(phi); +#if JL_LLVM_VERSION >= 200000 + pgcstack->moveBefore(slowTerm->getIterator()); +#else pgcstack->moveBefore(slowTerm); +#endif // refresh the basic block in the builder builder.SetInsertPoint(pgcstack); auto getter = builder.CreateLoad(T_pgcstack_getter, pgcstack_func_slot); @@ -336,7 +344,8 @@ bool LowerPTLS::run(bool *CFGModified) auto f = call->getCaller(); Value *pgcstack = NULL; for (Function::arg_iterator arg = f->arg_begin(); arg != f->arg_end(); ++arg) { - if (arg->hasSwiftSelfAttr()) { + AttributeSet attrs = f->getAttributes().getParamAttrs(arg->getArgNo()); + if (attrs.hasAttribute("gcstack")) { pgcstack = &*arg; break; } diff --git a/src/llvm-remove-addrspaces.cpp b/src/llvm-remove-addrspaces.cpp index bb492f467e74c..9e99cd3fcb25f 100644 --- a/src/llvm-remove-addrspaces.cpp +++ b/src/llvm-remove-addrspaces.cpp @@ -256,7 +256,7 @@ bool removeAddrspaces(Module &M, AddrspaceRemapFunction ASRemapper) Name, (GlobalVariable *)nullptr, GV->getThreadLocalMode(), - GV->getType()->getAddressSpace()); + cast(TypeRemapper.remapType(GV->getType()))->getAddressSpace()); NGV->copyAttributesFrom(GV); VMap[GV] = NGV; } @@ -276,7 +276,7 @@ bool removeAddrspaces(Module &M, AddrspaceRemapFunction ASRemapper) auto *NGA = GlobalAlias::create( TypeRemapper.remapType(GA->getValueType()), - GA->getType()->getPointerAddressSpace(), + cast(TypeRemapper.remapType(GA->getType()))->getAddressSpace(), GA->getLinkage(), Name, &M); @@ -334,7 +334,11 @@ bool removeAddrspaces(Module &M, AddrspaceRemapFunction ASRemapper) GV->setInitializer(nullptr); } - + // Same workaround as in CloneCtx::prepare_vmap to avoid LLVM bug when cloning + auto &MD = VMap.MD(); + for (auto cu: M.debug_compile_units()) { + MD[cu].reset(cu); + } // Similarly, copy over and rewrite function bodies for (Function *F : Functions) { Function *NF = cast(VMap[F]); @@ -414,6 +418,7 @@ bool removeAddrspaces(Module &M, AddrspaceRemapFunction ASRemapper) } } + return true; } diff --git a/src/macroexpand.scm b/src/macroexpand.scm index f67145317dc7a..2990d98eefe6e 100644 --- a/src/macroexpand.scm +++ b/src/macroexpand.scm @@ -513,7 +513,7 @@ (body (cadr e)) (m (caddr e)) (lno (cdddr e))) - (resolve-expansion-vars-with-new-env body env m lno parent-scope inarg #t))) + (resolve-expansion-vars-with-new-env body '() m lno parent-scope inarg #t))) ((tuple) (cons (car e) (map (lambda (x) diff --git a/src/method.c b/src/method.c index 38f963939d38c..5aaf221cc38a6 100644 --- a/src/method.c +++ b/src/method.c @@ -10,14 +10,12 @@ #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" +#include "builtin_proto.h" #ifdef __cplusplus extern "C" { #endif -extern jl_value_t *jl_builtin_getfield; -extern jl_value_t *jl_builtin_tuple; -jl_methtable_t *jl_kwcall_mt; jl_method_t *jl_opaque_closure_method; static void check_c_types(const char *where, jl_value_t *rt, jl_value_t *at) @@ -225,7 +223,7 @@ static jl_value_t *resolve_definition_effects(jl_value_t *expr, jl_module_t *mod jl_sym_t *fe_sym = jl_globalref_name(fe); // look at some known called functions jl_binding_t *b = jl_get_binding(fe_mod, fe_sym); - if (jl_get_binding_value_if_const(b) == jl_builtin_tuple) { + if (jl_get_latest_binding_value_if_const(b) == BUILTIN(tuple)) { size_t j; for (j = 1; j < nargs; j++) { if (!jl_is_quotenode(jl_exprarg(e, j))) @@ -611,6 +609,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_new_method_instance_uninit(void) jl_atomic_store_relaxed(&mi->cache, NULL); mi->cache_with_orig = 0; jl_atomic_store_relaxed(&mi->flags, 0); + jl_atomic_store_relaxed(&mi->dispatch_status, 0); return mi; } @@ -721,14 +720,14 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t jl_code_instance_t *ci = NULL; JL_GC_PUSH5(&ex, &func, &uninferred, &ci, &kind); jl_task_t *ct = jl_current_task; - int last_lineno = jl_lineno; + int last_lineno = jl_atomic_load_relaxed(&jl_lineno); int last_in = ct->ptls->in_pure_callback; size_t last_age = ct->world_age; JL_TRY { ct->ptls->in_pure_callback = 1; ct->world_age = jl_atomic_load_relaxed(&def->primary_world); - if (ct->world_age > jl_atomic_load_acquire(&jl_world_counter) || jl_atomic_load_relaxed(&def->deleted_world) < ct->world_age) + if (ct->world_age > jl_atomic_load_acquire(&jl_world_counter)) jl_error("The generator method cannot run until it is added to a method table."); // invoke code generator @@ -804,7 +803,8 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t else if (jl_is_mtable(kind)) { assert(i < l); ex = data[i++]; - jl_method_table_add_backedge((jl_methtable_t*)kind, ex, ci); + if ((jl_methtable_t*)kind == jl_method_table) + jl_method_table_add_backedge(ex, ci); } else { assert(i < l); @@ -818,12 +818,12 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *mi, size_t } ct->ptls->in_pure_callback = last_in; - jl_lineno = last_lineno; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); ct->world_age = last_age; } JL_CATCH { ct->ptls->in_pure_callback = last_in; - jl_lineno = last_lineno; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); jl_rethrow(); } JL_GC_POP(); @@ -958,6 +958,9 @@ JL_DLLEXPORT void jl_method_set_source(jl_method_t *m, jl_code_info_t *src) } src = jl_copy_code_info(src); src->isva = m->isva; // TODO: It would be nice to reverse this + // If nargs hasn't been set yet, do it now. This can happen if an old CodeInfo is deserialized. + if (src->nargs == 0) + src->nargs = m->nargs; assert(m->nargs == src->nargs); src->code = copy; jl_gc_wb(src, copy); @@ -1007,7 +1010,8 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) m->isva = 0; m->nargs = 0; jl_atomic_store_relaxed(&m->primary_world, ~(size_t)0); - jl_atomic_store_relaxed(&m->deleted_world, 1); + jl_atomic_store_relaxed(&m->dispatch_status, 0); + jl_atomic_store_relaxed(&m->interferences, (jl_genericmemory_t*)jl_an_empty_memory_any); m->is_for_opaque_closure = 0; m->nospecializeinfer = 0; jl_atomic_store_relaxed(&m->did_scan_source, 0); @@ -1028,7 +1032,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_code_instance_t **caller) JL_NOTSAFEPOINT { jl_value_t *item = jl_array_ptr_ref(list, i); - if (jl_is_code_instance(item)) { + if (!item || jl_is_code_instance(item)) { // Not an `invoke` call, it's just the CodeInstance if (invokesig != NULL) *invokesig = NULL; @@ -1053,6 +1057,14 @@ int set_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instan return i; } +int clear_next_edge(jl_array_t *list, int i, jl_value_t *invokesig, jl_code_instance_t *caller) +{ + if (invokesig) + jl_array_ptr_set(list, i++, NULL); + jl_array_ptr_set(list, i++, NULL); + return i; +} + void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_code_instance_t *caller) { if (invokesig) @@ -1061,6 +1073,33 @@ void push_edge(jl_array_t *list, jl_value_t *invokesig, jl_code_instance_t *call return; } +void jl_mi_done_backedges(jl_method_instance_t *mi JL_PROPAGATES_ROOT, uint8_t old_flags) { + uint8_t flags_now = 0; + jl_array_t *backedges = jl_mi_get_backedges_mutate(mi, &flags_now); + if (backedges && !old_flags) { + if (flags_now & MI_FLAG_BACKEDGES_DIRTY) { + size_t n = jl_array_nrows(backedges); + size_t i = 0; + size_t insb = 0; + while (i < n) { + jl_value_t *invokesig; + jl_code_instance_t *caller; + i = get_next_edge(backedges, i, &invokesig, &caller); + if (!caller) + continue; + insb = set_next_edge(backedges, insb, invokesig, caller); + } + if (insb == n) { + // All were deleted + mi->backedges = NULL; + } else { + jl_array_del_end(backedges, n - insb); + } + } + jl_atomic_fetch_and_relaxed(&mi->flags, ~MI_FLAG_BACKEDGES_ALL); + } +} + // method definition ---------------------------------------------------------- jl_method_t *jl_make_opaque_closure_method(jl_module_t *module, jl_value_t *name, @@ -1120,54 +1159,33 @@ JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name) return gf; } -static jl_methtable_t *nth_methtable(jl_value_t *a JL_PROPAGATES_ROOT, int n) JL_NOTSAFEPOINT -{ - if (jl_is_datatype(a)) { - if (n == 0) { - jl_methtable_t *mt = ((jl_datatype_t*)a)->name->mt; - if (mt != NULL) - return mt; - } - else if (jl_is_tuple_type(a)) { - if (jl_nparams(a) >= n) - return nth_methtable(jl_tparam(a, n - 1), 0); - } - } - else if (jl_is_typevar(a)) { - return nth_methtable(((jl_tvar_t*)a)->ub, n); - } - else if (jl_is_unionall(a)) { - return nth_methtable(((jl_unionall_t*)a)->body, n); - } - else if (jl_is_uniontype(a)) { - jl_uniontype_t *u = (jl_uniontype_t*)a; - jl_methtable_t *m1 = nth_methtable(u->a, n); - if ((jl_value_t*)m1 != jl_nothing) { - jl_methtable_t *m2 = nth_methtable(u->b, n); - if (m1 == m2) - return m1; - } - } - return (jl_methtable_t*)jl_nothing; -} // get the MethodTable for dispatch, or `nothing` if cannot be determined JL_DLLEXPORT jl_methtable_t *jl_method_table_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { - return nth_methtable(argtypes, 1); + return jl_method_table; } -jl_methtable_t *jl_kwmethod_table_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +// get a MethodCache for dispatch +JL_DLLEXPORT jl_methcache_t *jl_method_cache_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { - jl_methtable_t *kwmt = nth_methtable(argtypes, 3); - if ((jl_value_t*)kwmt == jl_nothing) - return NULL; - return kwmt; + return jl_method_table->cache; +} + +jl_methcache_t *jl_kwmethod_cache_for(jl_value_t *argtypes JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + return jl_method_table->cache; } JL_DLLEXPORT jl_methtable_t *jl_method_get_table(jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT { - return method->external_mt ? (jl_methtable_t*)method->external_mt : jl_method_table_for(method->sig); + return method->external_mt ? (jl_methtable_t*)method->external_mt : jl_method_table; +} + +// get an arbitrary MethodCache for dispatch optimizations of method +JL_DLLEXPORT jl_methcache_t *jl_method_get_cache(jl_method_t *method JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + return jl_method_get_table(method)->cache; } JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, @@ -1184,25 +1202,29 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, size_t nargs = jl_svec_len(atypes); assert(nargs > 0); int isva = jl_is_vararg(jl_svecref(atypes, nargs - 1)); - if (!jl_is_type(jl_svecref(atypes, 0)) || (isva && nargs == 1)) + jl_value_t *ft = jl_svecref(atypes, 0); + if (!jl_is_type(ft) || (isva && nargs == 1)) jl_error("function type in method definition is not a type"); jl_sym_t *name; jl_method_t *m = NULL; jl_value_t *argtype = NULL; - JL_GC_PUSH3(&f, &m, &argtype); + JL_GC_PUSH4(&ft, &f, &m, &argtype); size_t i, na = jl_svec_len(atypes); argtype = jl_apply_tuple_type(atypes, 1); if (!jl_is_datatype(argtype)) jl_error("invalid type in method definition (Union{})"); - jl_methtable_t *external_mt = mt; if (!mt) - mt = jl_method_table_for(argtype); - if ((jl_value_t*)mt == jl_nothing) - jl_error("Method dispatch is unimplemented currently for this method signature"); - if (mt->frozen) - jl_error("cannot add methods to a builtin function"); + mt = jl_method_table; + jl_methtable_t *external_mt = mt == jl_method_table ? NULL : mt; + + //if (!external_mt) { + // jl_value_t **ttypes = { jl_builtin_type, jl_tparam0(jl_anytuple_type) }; + // jl_value_t *invalidt = jl_apply_tuple_type_v(ttypes, 2); // Tuple{Union{Builtin,OpaqueClosure}, Vararg} + // if (!jl_has_empty_intersection(argtype, invalidt)) + // jl_error("cannot add methods to builtin function"); + //} assert(jl_is_linenode(functionloc)); jl_sym_t *file = (jl_sym_t*)jl_linenode_file(functionloc); @@ -1211,21 +1233,13 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, int32_t line = jl_linenode_line(functionloc); // TODO: derive our debug name from the syntax instead of the type - jl_methtable_t *kwmt = mt == jl_kwcall_mt ? jl_kwmethod_table_for(argtype) : mt; // if we have a kwcall, try to derive the name from the callee argument method table - name = (kwmt ? kwmt : mt)->name; - if (kwmt == jl_type_type_mt || kwmt == jl_nonfunction_mt || external_mt) { - // our value for `name` is bad, try to guess what the syntax might have had, - // like `jl_static_show_func_sig` might have come up with - jl_datatype_t *dt = jl_nth_argument_datatype(argtype, mt == jl_kwcall_mt ? 3 : 1); - if (dt != NULL) { - name = dt->name->name; - if (jl_is_type_type((jl_value_t*)dt)) { - dt = (jl_datatype_t*)jl_argument_datatype(jl_tparam0(dt)); - if ((jl_value_t*)dt != jl_nothing) { - name = dt->name->name; - } - } + jl_datatype_t *dtname = (jl_datatype_t*)jl_argument_datatype(jl_kwcall_type && ft == (jl_value_t*)jl_kwcall_type && nargs >= 3 ? jl_svecref(atypes, 2) : ft); + name = (jl_value_t*)dtname != jl_nothing ? dtname->name->singletonname : jl_any_type->name->singletonname; + if (jl_is_type_type((jl_value_t*)dtname)) { + dtname = (jl_datatype_t*)jl_argument_datatype(jl_tparam0(dtname)); + if ((jl_value_t*)dtname != jl_nothing) { + name = dtname->name->singletonname; } } @@ -1286,6 +1300,9 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, jl_symbol_name(file), line); } + ft = jl_rewrap_unionall(ft, argtype); + if (!external_mt && !jl_has_empty_intersection(ft, (jl_value_t*)jl_builtin_type)) // disallow adding methods to Any, Function, Builtin, and subtypes, or Unions of those + jl_errorf("cannot add methods to builtin function `%s`", jl_symbol_name(name)); m = jl_new_method_uninit(module); m->external_mt = (jl_value_t*)external_mt; diff --git a/src/module.c b/src/module.c index 2b9ee4367abba..7e542590327ae 100644 --- a/src/module.c +++ b/src/module.c @@ -20,7 +20,7 @@ static jl_binding_partition_t *new_binding_partition(void) jl_binding_partition_t *bpart = (jl_binding_partition_t*)jl_gc_alloc(jl_current_task->ptls, sizeof(jl_binding_partition_t), jl_binding_partition_type); bpart->restriction = NULL; bpart->kind = (size_t)PARTITION_KIND_GUARD; - bpart->min_world = 0; + jl_atomic_store_relaxed(&bpart->min_world, 0); jl_atomic_store_relaxed(&bpart->max_world, (size_t)-1); jl_atomic_store_relaxed(&bpart->next, NULL); return bpart; @@ -40,9 +40,12 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition__(jl_binding_t *b { // Iterate through the list of binding partitions, keeping track of where to insert a new one for an implicit // resolution if necessary. - while (gap->replace && world < gap->replace->min_world) { + while (gap->replace) { + size_t replace_min_world = jl_atomic_load_relaxed(&gap->replace->min_world); + if (world >= replace_min_world) + break; gap->insert = &gap->replace->next; - gap->max_world = gap->replace->min_world - 1; + gap->max_world = replace_min_world - 1; gap->parent = (jl_value_t*)gap->replace; gap->replace = jl_atomic_load_relaxed(gap->insert); } @@ -126,13 +129,73 @@ static void update_implicit_resolution(struct implicit_search_resolution *to_upd static jl_binding_partition_t *jl_implicit_import_resolved(jl_binding_t *b, struct implicit_search_gap gap, struct implicit_search_resolution resolution) { + size_t new_kind = resolution.ultimate_kind | gap.inherited_flags; + size_t new_max_world = gap.max_world < resolution.max_world ? gap.max_world : resolution.max_world; + size_t new_min_world = gap.min_world > resolution.min_world ? gap.min_world : resolution.min_world; + jl_binding_partition_t *next = gap.replace; + if (jl_is_binding_partition(gap.parent)) { + // Check if we can merge this into the previous binding partition + jl_binding_partition_t *prev = (jl_binding_partition_t *)gap.parent; + assert(new_max_world != ~(size_t)0); // It is inconsistent to have a gap with `gap.parent` set, but max_world == ~(size_t)0 + size_t expected_prev_min_world = new_max_world + 1; + if (prev->restriction == resolution.binding_or_const && prev->kind == new_kind) { +retry: + if (!jl_atomic_cmpswap(&prev->min_world, &expected_prev_min_world, new_min_world)) { + if (expected_prev_min_world <= new_min_world) { + return prev; + } + else if (expected_prev_min_world <= new_max_world) { + // Concurrent modification of the partition. However, our lookup is still valid, + // so we should still be able to extend the partition. + goto retry; + } + // There remains a gap - proceed + } else { + if (next) { + size_t next_min_world = jl_atomic_load_relaxed(&next->min_world); + expected_prev_min_world = new_min_world; + for (;;) { + // We've updated the previous partition - check if we've closed a gap + size_t next_max_world = jl_atomic_load_relaxed(&next->max_world); + if (next_max_world >= expected_prev_min_world-1 && next->kind == new_kind && next->restriction == resolution.binding_or_const) { + if (jl_atomic_cmpswap(&prev->min_world, &expected_prev_min_world, next_min_world)) { + jl_binding_partition_t *nextnext = jl_atomic_load_relaxed(&next->next); + if (!jl_atomic_cmpswap(&prev->next, &next, nextnext)) { + // `next` may have been merged into its subsequent partition - we need to retry + assert(next); + continue; + } + // N.B.: This can lose modifications to next->{min_world, next}. + // However, those modifications could only have been for another implicit + // partition, so we are ok to lose them and recompute them later if necessary. + } + assert(expected_prev_min_world <= new_min_world); + } + break; + } + } + return prev; + } + } + } jl_binding_partition_t *new_bpart = new_binding_partition(); - jl_atomic_store_relaxed(&new_bpart->max_world, gap.max_world < resolution.max_world ? gap.max_world : resolution.max_world); - new_bpart->min_world = gap.min_world > resolution.min_world ? gap.min_world : resolution.min_world; - new_bpart->kind = resolution.ultimate_kind | gap.inherited_flags; + jl_atomic_store_relaxed(&new_bpart->max_world, new_max_world); + new_bpart->kind = new_kind; new_bpart->restriction = resolution.binding_or_const; jl_gc_wb_fresh(new_bpart, new_bpart->restriction); - jl_atomic_store_relaxed(&new_bpart->next, gap.replace); + + if (next) { + // See if we can merge the next partition into this one + size_t next_max_world = jl_atomic_load_relaxed(&next->max_world); + if (next_max_world == new_min_world - 1 && next->kind == new_kind && next->restriction == resolution.binding_or_const) { + // See above for potentially losing modifications to next. + new_min_world = jl_atomic_load_acquire(&next->min_world); + next = jl_atomic_load_relaxed(&next->next); + } + } + + jl_atomic_store_relaxed(&new_bpart->min_world, new_min_world); + jl_atomic_store_relaxed(&new_bpart->next, next); if (!jl_atomic_cmpswap(gap.insert, &gap.replace, new_bpart)) return NULL; jl_gc_wb(gap.parent, new_bpart); @@ -170,13 +233,11 @@ struct implicit_search_resolution jl_resolve_implicit_import(jl_binding_t *b, mo struct _jl_module_using data = *module_usings_getidx(m, i); JL_UNLOCK(&m->lock); if (data.min_world > world) { - if (max_world > data.min_world) - max_world = data.min_world - 1; + max_world = WORLDMIN(max_world, data.min_world - 1); continue; } if (data.max_world < world) { - if (min_world < data.max_world) - min_world = data.max_world + 1; + min_world = WORLDMAX(min_world, data.max_world + 1); continue; } @@ -198,7 +259,7 @@ struct implicit_search_resolution jl_resolve_implicit_import(jl_binding_t *b, mo while (tempbpart && jl_bkind_is_some_explicit_import(jl_binding_kind(tempbpart))) { max_world = WORLDMIN(max_world, jl_atomic_load_relaxed(&tempbpart->max_world)); - min_world = WORLDMAX(min_world, tempbpart->min_world); + min_world = WORLDMAX(min_world, jl_atomic_load_relaxed(&tempbpart->min_world)); tempb = (jl_binding_t*)tempbpart->restriction; tempbpart = jl_get_binding_partition_if_present(tempb, world, &gap); @@ -206,7 +267,7 @@ struct implicit_search_resolution jl_resolve_implicit_import(jl_binding_t *b, mo int tempbpart_valid = tempbpart && (trust_cache || !jl_bkind_is_some_implicit(jl_binding_kind(tempbpart))); size_t tembppart_max_world = tempbpart_valid ? jl_atomic_load_relaxed(&tempbpart->max_world) : gap.max_world; - size_t tembppart_min_world = tempbpart ? WORLDMAX(tempbpart->min_world, gap.min_world) : gap.min_world; + size_t tembppart_min_world = tempbpart ? WORLDMAX(jl_atomic_load_relaxed(&tempbpart->min_world), gap.min_world) : gap.min_world; max_world = WORLDMIN(max_world, tembppart_max_world); min_world = WORLDMAX(min_world, tembppart_min_world); @@ -279,8 +340,15 @@ JL_DLLEXPORT jl_binding_partition_t *jl_maybe_reresolve_implicit(jl_binding_t *b assert(bpart == jl_atomic_load_relaxed(&b->partitions)); assert(bpart); struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, new_max_world+1, 0); - if (resolution.min_world == bpart->min_world) { - assert(bpart->restriction == resolution.binding_or_const && jl_binding_kind(bpart) == resolution.ultimate_kind); + int resolution_unchanged = bpart->restriction == resolution.binding_or_const && jl_binding_kind(bpart) == resolution.ultimate_kind; + size_t bpart_min_world = jl_atomic_load_relaxed(&bpart->min_world); + if (resolution.min_world == bpart_min_world) { + // The resolution has the same world bounds - it must be unchanged + assert(resolution_unchanged); + return bpart; + } else if (resolution_unchanged) { + // If the resolution is unchanged, we can still keep the bpart + assert(resolution.min_world > bpart_min_world); return bpart; } assert(resolution.min_world == new_max_world+1 && "Missed an invalidation or bad resolution bounds"); @@ -298,14 +366,14 @@ JL_DLLEXPORT jl_binding_partition_t *jl_maybe_reresolve_implicit(jl_binding_t *b JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t *bpart) { - struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_atomic_load_relaxed(&jl_world_counter), 0); - bpart->min_world = resolution.min_world; + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_atomic_load_acquire(&jl_world_counter), 0); + jl_atomic_store_relaxed(&bpart->min_world, resolution.min_world); jl_atomic_store_relaxed(&bpart->max_world, resolution.max_world); bpart->restriction = resolution.binding_or_const; bpart->kind = resolution.ultimate_kind; } -STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, jl_value_t *parent, _Atomic(jl_binding_partition_t *)*insert, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED +STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, jl_value_t *parent, _Atomic(jl_binding_partition_t *)*insert, size_t world, size_t max_world, modstack_t *st) JL_GLOBALLY_ROOTED { assert(jl_is_binding(b)); struct implicit_search_gap gap; @@ -313,7 +381,7 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b gap.insert = insert; gap.inherited_flags = 0; gap.min_world = 0; - gap.max_world = ~(size_t)0; + gap.max_world = max_world; while (1) { gap.replace = jl_atomic_load_relaxed(gap.insert); jl_binding_partition_t *bpart = jl_get_binding_partition__(b, world, &gap); @@ -330,14 +398,14 @@ jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) if (!b) return NULL; // Duplicate the code for the entry frame for branch prediction - return jl_get_binding_partition_(b, (jl_value_t*)b, &b->partitions, world, NULL); + return jl_get_binding_partition_(b, (jl_value_t*)b, &b->partitions, world, ~(size_t)0, NULL); } jl_binding_partition_t *jl_get_binding_partition_with_hint(jl_binding_t *b, jl_binding_partition_t *prev, size_t world) JL_GLOBALLY_ROOTED { // Helper for getting a binding partition for an older world after we've already looked up the partition for a newer world assert(b); - assert(prev->min_world > world); - return jl_get_binding_partition_(b, (jl_value_t*)prev, &prev->next, world, NULL); + size_t prev_min_world = jl_atomic_load_relaxed(&prev->min_world); + return jl_get_binding_partition_(b, (jl_value_t*)prev, &prev->next, world, prev_min_world-1, NULL); } jl_binding_partition_t *jl_get_binding_partition_all(jl_binding_t *b, size_t min_world, size_t max_world) { @@ -362,10 +430,11 @@ JL_DLLEXPORT int jl_get_binding_leaf_partitions_restriction_kind(jl_binding_t *b while (validated_min_world > min_world) { bpart = bpart ? jl_get_binding_partition_with_hint(b, bpart, validated_min_world - 1) : jl_get_binding_partition(b, validated_min_world - 1); - while (validated_min_world > min_world && validated_min_world > bpart->min_world) { + size_t bpart_min_world = jl_atomic_load_relaxed(&bpart->min_world); + while (validated_min_world > min_world && validated_min_world > bpart_min_world) { jl_binding_t *curb = b; jl_binding_partition_t *curbpart = bpart; - size_t cur_min_world = bpart->min_world; + size_t cur_min_world = bpart_min_world; size_t cur_max_world = validated_min_world - 1; jl_walk_binding_inplace_worlds(&curb, &curbpart, &cur_min_world, &cur_max_world, &maybe_depwarn, cur_max_world); enum jl_partition_kind kind = jl_binding_kind(curbpart); @@ -394,14 +463,33 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_leaf_partitions_value_if_const(jl_bindin struct restriction_kind_pair rkp = { NULL, NULL, PARTITION_KIND_GUARD, 0 }; if (!jl_get_binding_leaf_partitions_restriction_kind(b, &rkp, min_world, max_world)) return NULL; - if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != PARTITION_KIND_BACKDATED_CONST) { + if (jl_bkind_is_real_constant(rkp.kind)) { *maybe_depwarn = rkp.maybe_depwarn; return rkp.restriction; } return NULL; } -JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) +JL_DLLEXPORT size_t jl_binding_backedges_length(jl_binding_t *b) +{ + JL_LOCK(&b->globalref->mod->lock); + size_t len = 0; + if (b->backedges) + len = jl_array_len(b->backedges); + JL_UNLOCK(&b->globalref->mod->lock); + return len; +} + +JL_DLLEXPORT jl_value_t *jl_binding_backedges_getindex(jl_binding_t *b, size_t i) +{ + JL_LOCK(&b->globalref->mod->lock); + assert(b->backedges); + jl_value_t *ret = jl_array_ptr_ref(b->backedges, i-1); + JL_UNLOCK(&b->globalref->mod->lock); + return ret; +} + +static jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) { jl_task_t *ct = jl_current_task; const jl_uuid_t uuid_zero = {0, 0}; @@ -410,11 +498,13 @@ JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) jl_set_typetagof(m, jl_module_tag, 0); assert(jl_is_symbol(name)); m->name = name; - m->parent = parent; + m->parent = parent ? parent : m; m->istopmod = 0; m->uuid = uuid_zero; - static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing - m->build_id.lo = jl_hrtime() + (++mcounter); + static _Atomic(unsigned int) mcounter; // simple counter backup, in case hrtime is not incrementing + unsigned int count = jl_atomic_fetch_add_relaxed(&mcounter, 1); + // TODO: this is used for ir decompression and is liable to hash collisions so use more of the bits + m->build_id.lo = bitmix(jl_hrtime() + count, jl_rand()); if (!m->build_id.lo) m->build_id.lo++; // build id 0 is invalid m->build_id.hi = ~(uint64_t)0; @@ -437,27 +527,28 @@ JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) return m; } -JL_DLLEXPORT void jl_add_default_names(jl_module_t *m, uint8_t default_using_core, uint8_t self_name) +static void jl_add_default_names(jl_module_t *m, uint8_t default_using_core, uint8_t self_name) { if (jl_core_module) { // Bootstrap: Before jl_core_module is defined, we don't have enough infrastructure // for bindings, so Core itself gets special handling in jltypes.c if (default_using_core) { - jl_module_using(m, jl_core_module); + jl_module_initial_using(m, jl_core_module); } if (self_name) { // export own name, so "using Foo" makes "Foo" itself visible - jl_set_const(m, m->name, (jl_value_t*)m); - jl_module_public(m, m->name, 1); + jl_set_initial_const(m, m->name, (jl_value_t*)m, 1); } } } -JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name) +jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, uint8_t default_using_core, uint8_t self_name) { jl_module_t *m = jl_new_module__(name, parent); JL_GC_PUSH1(&m); + JL_LOCK(&world_counter_lock); jl_add_default_names(m, default_using_core, self_name); + JL_UNLOCK(&world_counter_lock); JL_GC_POP(); return m; } @@ -495,7 +586,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( jl_errorf("cannot declare %s.%s constant; it was already declared global", jl_symbol_name(mod->name), jl_symbol_name(var)); } - if (bpart->min_world == new_world) { + if (jl_atomic_load_relaxed(&bpart->min_world) == new_world) { bpart->kind = constant_kind | (bpart->kind & PARTITION_MASK_FLAG); bpart->restriction = val; if (val) @@ -512,13 +603,14 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( for (;;) { enum jl_partition_kind prev_kind = jl_binding_kind(prev_bpart); if (jl_bkind_is_some_constant(prev_kind) || prev_kind == PARTITION_KIND_GLOBAL || - (jl_bkind_is_some_import(prev_kind))) { + jl_bkind_is_some_import(prev_kind)) { need_backdate = 0; break; } - if (prev_bpart->min_world == 0) + size_t prev_bpart_min_world = jl_atomic_load_relaxed(&prev_bpart->min_world); + if (prev_bpart_min_world == 0) break; - prev_bpart = jl_get_binding_partition(b, prev_bpart->min_world - 1); + prev_bpart = jl_get_binding_partition(b, prev_bpart_min_world - 1); } } // If backdate is required, replace each existing partition by a new one. @@ -531,7 +623,8 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( while (1) { backdate_bpart->kind = (size_t)PARTITION_KIND_BACKDATED_CONST | (prev_bpart->kind & 0xf0); backdate_bpart->restriction = val; - backdate_bpart->min_world = prev_bpart->min_world; + jl_atomic_store_relaxed(&backdate_bpart->min_world, + jl_atomic_load_relaxed(&prev_bpart->min_world)); jl_gc_wb_fresh(backdate_bpart, val); jl_atomic_store_relaxed(&backdate_bpart->max_world, jl_atomic_load_relaxed(&prev_bpart->max_world)); @@ -760,11 +853,12 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, // return module of binding JL_DLLEXPORT jl_module_t *jl_get_module_of_binding(jl_module_t *m, jl_sym_t *var) { + size_t world = jl_current_task->world_age; jl_binding_t *b = jl_get_module_binding(m, var, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); + jl_walk_binding_inplace(&b, &bpart, world); if (jl_binding_kind(bpart) == PARTITION_KIND_IMPLICIT_CONST) { - struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_current_task->world_age, 0); + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, world, 0); if (!resolution.debug_only_ultimate_binding) jl_error("Constant binding was imported from multiple modules"); b = resolution.debug_only_ultimate_binding; @@ -779,15 +873,20 @@ static NOINLINE void print_backdate_admonition(jl_binding_t *b) JL_NOTSAFEPOINT " Julia 1.12 has introduced more strict world age semantics for global bindings.\n" " !!! This code may malfunction under Revise.\n" " !!! This code will error in future versions of Julia.\n" - "Hint: Add an appropriate `invokelatest` around the access to this binding.\n", + "Hint: Add an appropriate `invokelatest` around the access to this binding.\n" + "To make this warning an error, and hence obtain a stack trace, use `julia --depwarn=error`.\n", jl_symbol_name(b->globalref->mod->name), jl_symbol_name(b->globalref->name)); } static inline void check_backdated_binding(jl_binding_t *b, enum jl_partition_kind kind) JL_NOTSAFEPOINT { - if (__unlikely(kind == PARTITION_KIND_BACKDATED_CONST) && - !(jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION) & BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION)) { - print_backdate_admonition(b); + if (__unlikely(kind == PARTITION_KIND_BACKDATED_CONST)) { + // We don't want functions that inference executes speculatively to print this warning, so turn those into + // an error for inference purposes. + if (jl_current_task->ptls->in_pure_callback || jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) + jl_undefined_var_error(b->globalref->name, (jl_value_t*)b->globalref->mod); + if (!(jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION) & BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION)) + print_backdate_admonition(b); } } @@ -811,16 +910,17 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_in_world(jl_binding_t *b, size_t w return jl_atomic_load_relaxed(&b->value); } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_depwarn(jl_binding_t *b) +static jl_value_t *jl_get_binding_value_depwarn(jl_binding_t *b, size_t world) { - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); if (jl_options.depwarn) { int needs_depwarn = 0; - jl_walk_binding_inplace_depwarn(&b, &bpart, jl_current_task->world_age, &needs_depwarn); + jl_walk_binding_inplace_depwarn(&b, &bpart, world, &needs_depwarn); if (needs_depwarn) jl_binding_deprecation_warning(b); - } else { - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + } + else { + jl_walk_binding_inplace(&b, &bpart, world); } enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) @@ -833,11 +933,11 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_depwarn(jl_binding_t *b) return jl_atomic_load_relaxed(&b->value); } - JL_DLLEXPORT jl_value_t *jl_get_binding_value_seqcst(jl_binding_t *b) { - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + size_t world = jl_current_task->world_age; + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); + jl_walk_binding_inplace(&b, &bpart, world); enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) return NULL; @@ -849,51 +949,52 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_seqcst(jl_binding_t *b) return jl_atomic_load(&b->value); } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_const(jl_binding_t *b) +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_const(jl_binding_t *b) { - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + // See note below. Note that this is for some deprecated uses, and should not be added to new code. + size_t world = jl_atomic_load_acquire(&jl_world_counter); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); + jl_walk_binding_inplace(&b, &bpart, world); enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) return NULL; - if (!jl_bkind_is_some_constant(kind)) + if (!jl_bkind_is_real_constant(kind)) return NULL; - check_backdated_binding(b, kind); return bpart->restriction; } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_and_const(jl_binding_t *b) +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_resolved_and_const_debug_only(jl_binding_t *b) { - // Unlike jl_get_binding_value_if_const this doesn't try to allocate new binding partitions if they - // don't already exist, making this JL_NOTSAFEPOINT. + // Unlike jl_get_latest_binding_value_if_const this doesn't try to allocate new binding partitions if they + // don't already exist, making this JL_NOTSAFEPOINT. However, as a result, this may fail to return + // a value - even if one does exist. It should only be used for reflection/debugging when the integrity + // of the runtime is not guaranteed. if (!b) return NULL; jl_binding_partition_t *bpart = jl_atomic_load_relaxed(&b->partitions); if (!bpart) return NULL; size_t max_world = jl_atomic_load_relaxed(&bpart->max_world); - if (bpart->min_world > jl_current_task->world_age || jl_current_task->world_age > max_world) + if (max_world != ~(size_t)0) return NULL; enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) return NULL; - if (!jl_bkind_is_some_constant(kind)) + if (!jl_bkind_is_real_constant(kind)) return NULL; - check_backdated_binding(b, kind); return bpart->restriction; } -JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved(jl_binding_t *b) +JL_DLLEXPORT jl_value_t *jl_get_latest_binding_value_if_resolved_debug_only(jl_binding_t *b) { - // Unlike jl_get_binding_value this doesn't try to allocate new binding partitions if they - // don't already exist, making this JL_NOTSAFEPOINT. + // See note above. Use for debug/reflection purposes only. if (!b) return NULL; jl_binding_partition_t *bpart = jl_atomic_load_relaxed(&b->partitions); if (!bpart) return NULL; size_t max_world = jl_atomic_load_relaxed(&bpart->max_world); - if (bpart->min_world > jl_current_task->world_age || jl_current_task->world_age > max_world) + if (max_world != ~(size_t)0) return NULL; enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) @@ -901,7 +1002,6 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved(jl_binding_t *b) if (jl_bkind_is_some_import(kind)) return NULL; if (jl_bkind_is_some_constant(kind)) { - check_backdated_binding(b, kind); return bpart->restriction; } return jl_atomic_load_relaxed(&b->value); @@ -936,6 +1036,7 @@ static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b) // along the way. JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b, size_t new_world) { + assert(new_world > jl_atomic_load_relaxed(&jl_world_counter)); jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_constant(kind) && kind != PARTITION_KIND_IMPLICIT_CONST) @@ -957,7 +1058,7 @@ JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b, size_t new_w check_safe_newbinding(b->globalref->mod, b->globalref->name); return NULL; } - jl_module_t *from = jl_binding_dbgmodule(b);\ + jl_module_t *from = jl_binding_dbgmodule(b); assert(from); // Can only be NULL if implicit, which we excluded above jl_errorf("invalid method definition in %s: exported function %s.%s does not exist", jl_module_debug_name(b->globalref->mod), jl_module_debug_name(from), jl_symbol_name(b->globalref->name)); @@ -1044,8 +1145,8 @@ JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *var) return b && jl_binding_kind(bpart) == PARTITION_KIND_IMPORTED; } -extern const char *jl_filename; -extern int jl_lineno; +extern _Atomic(const char *) jl_filename; +extern _Atomic(int) jl_lineno; static char const dep_message_prefix[] = "_dep_message_"; @@ -1082,14 +1183,14 @@ static void jl_binding_dep_message(jl_binding_t *b) jl_printf(JL_STDERR, " instead."); } else { - jl_methtable_t *mt = jl_gf_mtable(v); - if (mt != NULL) { + jl_typename_t *tn = ((jl_datatype_t*)jl_typeof(v))->name; + if (tn != NULL) { jl_printf(JL_STDERR, ", use "); - if (mt->module != jl_core_module) { - jl_static_show(JL_STDERR, (jl_value_t*)mt->module); + if (tn->module != jl_core_module) { + jl_static_show(JL_STDERR, (jl_value_t*)tn->module); jl_printf(JL_STDERR, "."); } - jl_printf(JL_STDERR, "%s", jl_symbol_name(mt->name)); + jl_printf(JL_STDERR, "%s", jl_symbol_name(tn->singletonname)); jl_printf(JL_STDERR, " instead."); } } @@ -1123,7 +1224,7 @@ static int eq_bindings(jl_binding_partition_t *owner, jl_binding_t *alias, size_ } // NOTE: we use explici since explicit is a C++ keyword -static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici) +JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici) { check_safe_import_from(from); jl_binding_t *b = jl_get_binding(from, s); @@ -1200,24 +1301,27 @@ static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl JL_UNLOCK(&world_counter_lock); } -JL_DLLEXPORT void jl_module_import(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s) -{ - module_import_(ct, to, from, s, s, 1); -} - -JL_DLLEXPORT void jl_module_import_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname) -{ - module_import_(ct, to, from, asname, s, 1); -} - -JL_DLLEXPORT void jl_module_use(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s) +JL_DLLEXPORT void jl_import_module(jl_task_t *ct, jl_module_t *JL_NONNULL m, jl_module_t *import, jl_sym_t *asname) { - module_import_(ct, to, from, s, s, 0); -} - -JL_DLLEXPORT void jl_module_use_as(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *s, jl_sym_t *asname) -{ - module_import_(ct, to, from, asname, s, 0); + assert(m); + jl_sym_t *name = asname ? asname : import->name; + // TODO: this is a bit race-y with what error message we might print + jl_binding_t *b = jl_get_module_binding(m, name, 1); + size_t world = jl_atomic_load_acquire(&jl_world_counter); + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); + enum jl_partition_kind kind = jl_binding_kind(bpart); + if (!jl_bkind_is_some_implicit(kind) && kind != PARTITION_KIND_DECLARED) { + // Unlike regular constant declaration, we allow this as long as we eventually end up at a constant. + jl_walk_binding_inplace(&b, &bpart, world); + if (jl_bkind_is_some_constant(jl_binding_kind(bpart))) { + // Already declared (e.g. on another thread) or imported. + if (bpart->restriction == (jl_value_t*)import) + return; + } + jl_errorf("importing %s into %s conflicts with an existing global", + jl_symbol_name(name), jl_symbol_name(m->name)); + } + jl_declare_constant_val2(b, m, name, (jl_value_t*)import, PARTITION_KIND_CONST_IMPORT); } void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to) @@ -1231,6 +1335,19 @@ void jl_add_usings_backedge(jl_module_t *from, jl_module_t *to) JL_UNLOCK(&from->lock); } +void jl_module_initial_using(jl_module_t *to, jl_module_t *from) +{ + struct _jl_module_using new_item = { + .mod = from, + .min_world = 0, + .max_world = ~(size_t)0 + }; + arraylist_grow(&to->usings, sizeof(struct _jl_module_using)/sizeof(void*)); + memcpy(&to->usings.items[to->usings.len-3], &new_item, sizeof(struct _jl_module_using)); + jl_gc_wb(to, from); + jl_add_usings_backedge(from, to); +} + JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from) { if (to == from) @@ -1323,11 +1440,10 @@ JL_DLLEXPORT jl_value_t *jl_get_module_binding_or_nothing(jl_module_t *m, jl_sym return (jl_value_t*)b; } -JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported) +int jl_module_public_(jl_module_t *from, jl_sym_t *s, int exported, size_t new_world) { + // caller must hold world_counter_lock jl_binding_t *b = jl_get_module_binding(from, s, 1); - JL_LOCK(&world_counter_lock); - size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); int was_exported = (bpart->kind & PARTITION_FLAG_EXPORTED) != 0; if (jl_atomic_load_relaxed(&b->flags) & BINDING_FLAG_PUBLICP) { @@ -1342,9 +1458,9 @@ JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported) jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_PUBLICP); if (was_exported != exported) { jl_replace_binding_locked2(b, bpart, bpart->restriction, bpart->kind | PARTITION_FLAG_EXPORTED, new_world); - jl_atomic_store_release(&jl_world_counter, new_world); + return 1; } - JL_UNLOCK(&world_counter_lock); + return 0; } JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import) // unlike most queries here, this is currently seq_cst @@ -1361,22 +1477,19 @@ JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import) // u } else { jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); } - if (jl_bkind_is_some_guard(jl_binding_kind(bpart))) + enum jl_partition_kind kind = jl_binding_kind(bpart); + if (jl_bkind_is_some_guard(kind)) return 0; - if (jl_bkind_is_defined_constant(jl_binding_kind(bpart))) { - // N.B.: No backdated check for isdefined + if (jl_bkind_is_defined_constant(kind)) { + if (__unlikely(kind == PARTITION_KIND_BACKDATED_CONST)) { + return !(jl_current_task->ptls->in_pure_callback || jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR); + } + // N.B.: No backdated admonition for isdefined return 1; } return jl_atomic_load(&b->value) != NULL; } -JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var) -{ - jl_binding_t *b = jl_get_module_binding(m, var, 0); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - return b && ((bpart->kind & PARTITION_FLAG_EXPORTED) || jl_binding_kind(bpart) == PARTITION_KIND_GLOBAL); -} - JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_module_binding(m, var, 0); @@ -1455,18 +1568,27 @@ JL_DLLEXPORT jl_binding_t *jl_get_module_binding(jl_module_t *m, jl_sym_t *var, } -JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr) +// get the value (or null) in the world +jl_value_t *jl_get_globalref_value(jl_globalref_t *gr, size_t world) { jl_binding_t *b = gr->binding; if (!b) b = jl_get_module_binding(gr->mod, gr->name, 1); - return jl_get_binding_value_depwarn(b); + return jl_get_binding_value_depwarn(b, world); } +// get the value (or null) in the world +jl_value_t *jl_get_global_value(jl_module_t *m, jl_sym_t *var, size_t world) +{ + jl_binding_t *b = jl_get_module_binding(m, var, 1); + return jl_get_binding_value_depwarn(b, world); +} + +// get the global (or null) in the latest world JL_DLLEXPORT jl_value_t *jl_get_global(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_module_binding(m, var, 1); - return jl_get_binding_value_depwarn(b); + return jl_get_binding_value_depwarn(b, jl_atomic_load_acquire(&jl_world_counter)); } JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT) @@ -1475,12 +1597,28 @@ JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *va jl_checked_assignment(bp, m, var, val); } +void jl_set_initial_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT, int exported) +{ + // this function is only valid during initialization, so there is no risk of data races her are not too important to use + int kind = PARTITION_KIND_CONST | (exported ? PARTITION_FLAG_EXPORTED : 0); + // jl_declare_constant_val3(NULL, m, var, (jl_value_t*)jl_any_type, kind, 0); + jl_binding_t *bp = jl_get_module_binding(m, var, 1); + jl_binding_partition_t *bpart = jl_get_binding_partition(bp, 0); + assert(jl_atomic_load_relaxed(&bpart->min_world) == 0); + jl_atomic_store_relaxed(&bpart->max_world, ~(size_t)0); // jl_check_new_binding_implicit likely incorrectly truncated it + if (exported) + jl_atomic_fetch_or_relaxed(&bp->flags, BINDING_FLAG_PUBLICP); + bpart->kind = kind | (bpart->kind & PARTITION_MASK_FLAG); + bpart->restriction = val; + jl_gc_wb(bpart, val); +} + JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT) { - // this function is mostly only used during initialization, so the data races here are not too important to us + // this function is dangerous and unsound. do not use. jl_binding_t *bp = jl_get_module_binding(m, var, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(bp, jl_current_task->world_age); - bpart->min_world = 0; + jl_atomic_store_relaxed(&bpart->min_world, 0); jl_atomic_store_release(&bpart->max_world, ~(size_t)0); bpart->kind = PARTITION_KIND_CONST | (bpart->kind & PARTITION_MASK_FLAG); bpart->restriction = val; @@ -1507,6 +1645,7 @@ void jl_invalidate_binding_refs(jl_globalref_t *ref, jl_binding_partition_t *inv JL_DLLEXPORT void jl_add_binding_backedge(jl_binding_t *b, jl_value_t *edge) { + JL_LOCK(&b->globalref->mod->lock); if (!b->backedges) { b->backedges = jl_alloc_vec_any(0); jl_gc_wb(b, b->backedges); @@ -1514,9 +1653,11 @@ JL_DLLEXPORT void jl_add_binding_backedge(jl_binding_t *b, jl_value_t *edge) jl_array_ptr_ref(b->backedges, jl_array_len(b->backedges)-1) == edge) { // Optimization: Deduplicate repeated insertion of the same edge (e.g. during // definition of a method that contains many references to the same global) + JL_UNLOCK(&b->globalref->mod->lock); return; } jl_array_ptr_1d_push(b->backedges, edge); + JL_UNLOCK(&b->globalref->mod->lock); } // Called for all GlobalRefs found in lowered code. Adds backedges for cross-module @@ -1557,13 +1698,19 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, // Until the first such replacement, we can fast-path validation. // For these purposes, we consider the `Main` module to be a non-sysimg module. // This is legal, because we special case the `Main` in check_safe_import_from. - if (jl_object_in_image((jl_value_t*)b) && b->globalref->mod != jl_main_module && jl_atomic_load_relaxed(&jl_first_image_replacement_world) == ~(size_t)0) + if (jl_object_in_image((jl_value_t*)b) && b->globalref->mod != jl_main_module && jl_atomic_load_relaxed(&jl_first_image_replacement_world) == ~(size_t)0) { + // During incremental compilation replacement of image bindings is forbidden; + // We use this to avoid inserting backedges while loading pkgimages. + // `check_safe_newbinding` checks an equivalent condition on `b->globalref->mod`, + // but doesn't quite query `jl_object_in_image`, so assert here to be extra sure. + assert(!(jl_options.incremental && jl_generating_output())); jl_atomic_store_relaxed(&jl_first_image_replacement_world, new_world); + } assert(jl_atomic_load_relaxed(&b->partitions) == old_bpart); jl_binding_partition_t *new_bpart = new_binding_partition(); JL_GC_PUSH1(&new_bpart); - new_bpart->min_world = new_world; + jl_atomic_store_relaxed(&new_bpart->min_world, new_world); if ((kind & PARTITION_MASK_KIND) == PARTITION_FAKE_KIND_IMPLICIT_RECOMPUTE) { assert(!restriction_val); struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, new_world, 0); @@ -1615,7 +1762,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding(jl_binding_t *b, size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; jl_binding_partition_t *bpart = jl_replace_binding_locked(b, old_bpart, restriction_val, kind, new_world); - if (bpart && bpart->min_world == new_world) + if (bpart && jl_atomic_load_relaxed(&bpart->min_world) == new_world) jl_atomic_store_release(&jl_world_counter, new_world); JL_UNLOCK(&world_counter_lock); @@ -1629,7 +1776,7 @@ JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr) b = jl_get_module_binding(gr->mod, gr->name, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); - return jl_bkind_is_some_constant(jl_binding_kind(bpart)); + return jl_bkind_is_real_constant(jl_binding_kind(bpart)); } JL_DLLEXPORT void jl_disable_binding(jl_globalref_t *gr) @@ -1637,16 +1784,20 @@ JL_DLLEXPORT void jl_disable_binding(jl_globalref_t *gr) jl_binding_t *b = gr->binding; if (!b) b = jl_get_module_binding(gr->mod, gr->name, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (jl_binding_kind(bpart) == PARTITION_KIND_GUARD) { - // Already guard + for (;;) { + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_atomic_load_acquire(&jl_world_counter)); + + if (jl_binding_kind(bpart) == PARTITION_KIND_GUARD) { + // Already guard + return; + } + + if (!jl_replace_binding(b, bpart, NULL, PARTITION_KIND_GUARD)) + continue; + return; } - - for (;;) - if (jl_replace_binding(b, bpart, NULL, PARTITION_KIND_GUARD)) - break; } JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var) @@ -1654,7 +1805,7 @@ JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var) jl_binding_t *b = jl_get_binding(m, var); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); - return b && jl_bkind_is_some_constant(jl_binding_kind(bpart)); + return b && jl_bkind_is_real_constant(jl_binding_kind(bpart)); } // set the deprecated flag for a binding: @@ -1721,8 +1872,8 @@ void jl_binding_deprecation_warning(jl_binding_t *b) jl_binding_dep_message(b); if (jl_options.depwarn != JL_OPTIONS_DEPWARN_ERROR) { - if (jl_lineno != 0) { - jl_printf(JL_STDERR, " likely near %s:%d\n", jl_filename, jl_lineno); + if (jl_atomic_load_relaxed(&jl_lineno) != 0) { + jl_printf(JL_STDERR, " likely near %s:%d\n", jl_atomic_load_relaxed(&jl_filename), jl_atomic_load_relaxed(&jl_lineno)); } } diff --git a/src/opaque_closure.c b/src/opaque_closure.c index a10b5c617753c..2e39d5965b45a 100644 --- a/src/opaque_closure.c +++ b/src/opaque_closure.c @@ -159,7 +159,6 @@ JL_DLLEXPORT jl_opaque_closure_t *jl_new_opaque_closure_from_code_info(jl_tuplet size_t world = jl_current_task->world_age; // these are only legal in the current world since they are not in any tables jl_atomic_store_release(&meth->primary_world, world); - jl_atomic_store_release(&meth->deleted_world, world); if (isinferred) { jl_value_t *argslotty = jl_array_ptr_ref(ci->slottypes, 0); diff --git a/src/passes.h b/src/passes.h index 83721525d6f7e..0c5a124ade952 100644 --- a/src/passes.h +++ b/src/passes.h @@ -43,6 +43,11 @@ struct FinalLowerGCPass : PassInfoMixin { static bool isRequired() { return true; } }; +struct ExpandAtomicModifyPass : PassInfoMixin { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) JL_NOTSAFEPOINT; +}; + + // Module Passes struct CPUFeaturesPass : PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) JL_NOTSAFEPOINT; diff --git a/src/pipeline.cpp b/src/pipeline.cpp index eb93943653b34..f91db6fc037d7 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -574,6 +574,7 @@ static void buildIntrinsicLoweringPipeline(ModulePassManager &MPM, PassBuilder * FunctionPassManager FPM; JULIA_PASS(FPM.addPass(LateLowerGCPass())); JULIA_PASS(FPM.addPass(FinalLowerGCPass())); + JULIA_PASS(FPM.addPass(ExpandAtomicModifyPass())); // after LateLowerGCPass so that all IPO is valid MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } JULIA_PASS(MPM.addPass(LowerPTLSPass(options.dump_native))); @@ -590,7 +591,8 @@ static void buildIntrinsicLoweringPipeline(ModulePassManager &MPM, PassBuilder * FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions())); MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } - } else if (!options.remove_ni) { + } + else if (!options.remove_ni) { JULIA_PASS(MPM.addPass(RemoveNIPass())); } MPM.addPass(AfterIntrinsicLoweringMarkerPass()); diff --git a/src/precompile.c b/src/precompile.c index c21cf5367fba6..8d4e8495fc4bd 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -36,27 +36,30 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { // char*: src text // At the end we write int32(0) as a terminal sentinel. size_t len = jl_array_nrows(udeps); - static jl_value_t *replace_depot_func = NULL; - if (!replace_depot_func) - replace_depot_func = jl_get_global(jl_base_module, jl_symbol("replace_depot_path")); - static jl_value_t *normalize_depots_func = NULL; - if (!normalize_depots_func) - normalize_depots_func = jl_get_global(jl_base_module, jl_symbol("normalize_depots_for_relocation")); ios_t srctext; - jl_value_t *deptuple = NULL, *depots = NULL; - JL_GC_PUSH3(&deptuple, &udeps, &depots); + jl_value_t *replace_depot_func = NULL; + jl_value_t *normalize_depots_func = NULL; + jl_value_t *deptuple = NULL; + jl_value_t *depots = NULL; jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + JL_GC_PUSH4(&deptuple, &depots, &replace_depot_func, &normalize_depots_func); + replace_depot_func = jl_eval_global_var(jl_base_module, jl_symbol("replace_depot_path"), jl_current_task->world_age); + normalize_depots_func = jl_eval_global_var(jl_base_module, jl_symbol("normalize_depots_for_relocation"), jl_current_task->world_age); depots = jl_apply(&normalize_depots_func, 1); - ct->world_age = last_age; + jl_datatype_t *deptuple_p[5] = {jl_module_type, jl_string_type, jl_uint64_type, jl_uint32_type, jl_float64_type}; + jl_value_t *jl_deptuple_type = jl_apply_tuple_type_v((jl_value_t**)deptuple_p, 5); + JL_GC_PROMISE_ROOTED(jl_deptuple_type); +#define jl_is_deptuple(v) (jl_typeis((v), jl_deptuple_type)) for (size_t i = 0; i < len; i++) { deptuple = jl_array_ptr_ref(udeps, i); - jl_value_t *depmod = jl_fieldref(deptuple, 0); // module + jl_value_t *depmod = jl_fieldref_noalloc(deptuple, 0); // module // Dependencies declared with `include_dependency` are excluded // because these may not be Julia code (and could be huge) + JL_TYPECHK(write_srctext, deptuple, deptuple); if (depmod != (jl_value_t*)jl_main_module) { - jl_value_t *abspath = jl_fieldref(deptuple, 1); // file abspath + jl_value_t *abspath = jl_fieldref_noalloc(deptuple, 1); // file abspath const char *abspathstr = jl_string_data(abspath); if (!abspathstr[0]) continue; @@ -67,17 +70,11 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { continue; } - jl_value_t **replace_depot_args; - JL_GC_PUSHARGS(replace_depot_args, 3); + jl_value_t *replace_depot_args[3]; replace_depot_args[0] = replace_depot_func; replace_depot_args[1] = abspath; replace_depot_args[2] = depots; - jl_task_t *ct = jl_current_task; - size_t last_age = ct->world_age; - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); jl_value_t *depalias = (jl_value_t*)jl_apply(replace_depot_args, 3); - ct->world_age = last_age; - JL_GC_POP(); size_t slen = jl_string_len(depalias); write_int32(f, slen); @@ -91,6 +88,8 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { ios_seek_end(f); } } + ct->world_age = last_age; +#undef jl_is_deptuple JL_GC_POP(); } write_int32(f, 0); // mark the end of the source text @@ -140,7 +139,6 @@ JL_DLLEXPORT void jl_write_compiler_output(void) } } - assert(jl_precompile_toplevel_module == NULL); void *native_code = NULL; bool_t emit_native = jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm; diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 8d441d6c7792a..a34a3838c6ddc 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -159,12 +159,12 @@ static int compile_all_collect_(jl_methtable_t *mt, void *env) return 1; } -static void jl_compile_all_defs(jl_array_t *mis, int all) +static void jl_compile_all_defs(jl_array_t *mis, int all, jl_array_t *mod_array) { jl_array_t *allmeths = jl_alloc_vec_any(0); JL_GC_PUSH1(&allmeths); - jl_foreach_reachable_mtable(compile_all_collect_, allmeths); + jl_foreach_reachable_mtable(compile_all_collect_, mod_array, allmeths); size_t world = jl_atomic_load_acquire(&jl_world_counter); size_t i, l = jl_array_nrows(allmeths); @@ -208,7 +208,7 @@ static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closur jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); if (inferred && (jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL || inferred == jl_nothing || - ((jl_is_string(inferred) || jl_is_code_info(inferred)) && jl_ir_inlining_cost(inferred) == UINT16_MAX))) { + ((jl_is_string(inferred) || jl_is_code_info(inferred) || jl_is_uint8(inferred)) && jl_ir_inlining_cost(inferred) == UINT16_MAX))) { do_compile = 1; } else if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL || jl_atomic_load_relaxed(&codeinst->precompile)) { @@ -224,38 +224,53 @@ static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closur return 1; } -static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *closure) +struct precompile_enq_all_specializations_env { + jl_array_t *worklist; + jl_array_t *m; +}; + +static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *env) { jl_method_t *m = def->func.method; - if (m->external_mt) - return 1; + assert(!m->external_mt); + struct precompile_enq_all_specializations_env *closure = (struct precompile_enq_all_specializations_env*)env; + if (closure->worklist) { + size_t i, l = jl_array_nrows(closure->worklist); + for (i = 0; i < l; i++) { + if (m->module == (jl_module_t*)jl_array_ptr_ref(closure->worklist, i)) + break; + } + if (i == l) + return 1; + } if ((m->name == jl_symbol("__init__") || m->ccallable) && jl_is_dispatch_tupletype(m->sig)) { // ensure `__init__()` and @ccallables get strongly-hinted, specialized, and compiled jl_method_instance_t *mi = jl_specializations_get_linfo(m, m->sig, jl_emptysvec); - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); + jl_array_ptr_1d_push(closure->m, (jl_value_t*)mi); } else { jl_value_t *specializations = jl_atomic_load_relaxed(&def->func.method->specializations); if (!jl_is_svec(specializations)) { - precompile_enq_specialization_((jl_method_instance_t*)specializations, closure); + precompile_enq_specialization_((jl_method_instance_t*)specializations, closure->m); } else { size_t i, l = jl_svec_len(specializations); for (i = 0; i < l; i++) { jl_value_t *mi = jl_svecref(specializations, i); if (mi != jl_nothing) - precompile_enq_specialization_((jl_method_instance_t*)mi, closure); + precompile_enq_specialization_((jl_method_instance_t*)mi, closure->m); } } } if (m->ccallable) - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)m->ccallable); + jl_array_ptr_1d_push(closure->m, (jl_value_t*)m->ccallable); return 1; } -static int precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) +static int precompile_enq_all_specializations_(jl_array_t *worklist, jl_array_t *env) { - return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), precompile_enq_all_specializations__, env); + struct precompile_enq_all_specializations_env closure = {worklist, env}; + return jl_typemap_visitor(jl_atomic_load_relaxed(&jl_method_table->defs), precompile_enq_all_specializations__, &closure); } static void *jl_precompile_(jl_array_t *m, int external_linkage) @@ -284,13 +299,13 @@ static void *jl_precompile_(jl_array_t *m, int external_linkage) return native_code; } -static void *jl_precompile(int all) +static void *jl_precompile(int all, jl_array_t *mod_array) { // array of MethodInstances and ccallable aliases to include in the output jl_array_t *m = jl_alloc_vec_any(0); JL_GC_PUSH1(&m); - jl_compile_all_defs(m, all); - jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); + jl_compile_all_defs(m, all, mod_array); + precompile_enq_all_specializations_(NULL, m); void *native_code = jl_precompile_(m, 0); JL_GC_POP(); return native_code; @@ -311,13 +326,8 @@ static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_met jl_array_t *m = jl_alloc_vec_any(0); JL_GC_PUSH1(&m); if (!suppress_precompile) { - size_t i, n = jl_array_nrows(worklist); - for (i = 0; i < n; i++) { - jl_module_t *mod = (jl_module_t*)jl_array_ptr_ref(worklist, i); - assert(jl_is_module(mod)); - foreach_mtable_in_module(mod, precompile_enq_all_specializations_, m); - } - n = jl_array_nrows(extext_methods); + precompile_enq_all_specializations_(worklist, m); + size_t i, n = jl_array_nrows(extext_methods); for (i = 0; i < n; i++) { jl_method_t *method = (jl_method_t*)jl_array_ptr_ref(extext_methods, i); assert(jl_is_method(method)); @@ -350,21 +360,15 @@ static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_met static int enq_ccallable_entrypoints_(jl_typemap_entry_t *def, void *closure) { jl_method_t *m = def->func.method; - if (m->external_mt) - return 1; + assert(!m->external_mt); if (m->ccallable) jl_add_entrypoint((jl_tupletype_t*)jl_svecref(m->ccallable, 1)); return 1; } -static int enq_ccallable_entrypoints(jl_methtable_t *mt, void *env) -{ - return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), enq_ccallable_entrypoints_, env); -} - JL_DLLEXPORT void jl_add_ccallable_entrypoints(void) { - jl_foreach_reachable_mtable(enq_ccallable_entrypoints, NULL); + jl_typemap_visitor(jl_atomic_load_relaxed(&jl_method_table->defs), enq_ccallable_entrypoints_, NULL); } static void *jl_precompile_trimmed(size_t world) @@ -402,38 +406,43 @@ static void *jl_precompile_trimmed(size_t world) return native_code; } -static void jl_rebuild_methtables(arraylist_t* MIs, htable_t* mtables) +static void jl_rebuild_methtables(arraylist_t *MIs, htable_t *mtables) JL_GC_DISABLED { - size_t i; - for (i = 0; i < MIs->len; i++) { + // Rebuild MethodTable to contain only those methods for which we compiled code. + // This can have significant soundness problems if there previously existed + // any ambiguous methods, but it would probably be pretty hard to do this + // fully correctly (with the necessary inserted guard entries). + htable_t ms; + htable_new(&ms, 0); + for (size_t i = 0; i < MIs->len; i++) { jl_method_instance_t *mi = (jl_method_instance_t*)MIs->items[i]; jl_method_t *m = mi->def.method; + // Check if the method is already in the new table, if not then insert it there + void **inserted = ptrhash_bp(&ms, m); + if (*inserted != HT_NOTFOUND) + continue; + *inserted = (void*)m; jl_methtable_t *old_mt = jl_method_get_table(m); if ((jl_value_t *)old_mt == jl_nothing) continue; - jl_sym_t *name = old_mt->name; - if (!ptrhash_has(mtables, old_mt)) { - jl_methtable_t *new_mt = jl_new_method_table(name, m->module); + if (!ptrhash_has(mtables, old_mt)){ + jl_methtable_t *new_mt = jl_new_method_table(old_mt->name, old_mt->module); OBJHASH_PIN(old_mt) OBJHASH_PIN(new_mt) ptrhash_put(mtables, old_mt, new_mt); } jl_methtable_t *mt = (jl_methtable_t*)ptrhash_get(mtables, old_mt); - size_t world = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t * lookup = jl_methtable_lookup(mt, m->sig, world); - // Check if the method is already in the new table, if not then insert it there - if (lookup == jl_nothing || (jl_method_t*)lookup != m) { - //TODO: should this be a function like unsafe_insert_method? - size_t min_world = jl_atomic_load_relaxed(&m->primary_world); - size_t max_world = jl_atomic_load_relaxed(&m->deleted_world); - jl_atomic_store_relaxed(&m->primary_world, ~(size_t)0); - jl_atomic_store_relaxed(&m->deleted_world, 1); - jl_typemap_entry_t *newentry = jl_method_table_add(mt, m, NULL); - jl_atomic_store_relaxed(&m->primary_world, min_world); - jl_atomic_store_relaxed(&m->deleted_world, max_world); - jl_atomic_store_relaxed(&newentry->min_world, min_world); - jl_atomic_store_relaxed(&newentry->max_world, max_world); - } + //TODO: should this be a function like unsafe_insert_method, since all that is wanted is the jl_typemap_insert on a copy of the existing entry + size_t min_world = jl_atomic_load_relaxed(&m->primary_world); + size_t max_world = ~(size_t)0; + int dispatch_status = jl_atomic_load_relaxed(&m->dispatch_status); + jl_atomic_store_relaxed(&m->primary_world, ~(size_t)0); + jl_atomic_store_relaxed(&m->dispatch_status, 0); + jl_typemap_entry_t *newentry = jl_method_table_add(mt, m, NULL); + jl_atomic_store_relaxed(&m->primary_world, min_world); + jl_atomic_store_relaxed(&m->dispatch_status, dispatch_status); + jl_atomic_store_relaxed(&newentry->min_world, min_world); + jl_atomic_store_relaxed(&newentry->max_world, max_world); // short-circuit jl_method_table_insert } - + htable_free(&ms); } diff --git a/src/processor.cpp b/src/processor.cpp index 6f95ee7f3790a..443b86531b3a1 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -506,6 +506,20 @@ parse_cmdline(const char *option, F &&feature_cb) if (!option) abort(); + // Preprocess the option string to expand "sysimage" keyword + std::string processed_option; + if (strncmp(option, "sysimage", 8) == 0 && (option[8] == '\0' || option[8] == ';')) { + // Replace "sysimage" with the actual sysimage CPU target + jl_value_t *target_str = jl_get_sysimage_cpu_target(); + if (target_str != nullptr) { + processed_option = std::string(jl_string_data(target_str), jl_string_len(target_str)); + if (option[8] == ';') { + processed_option += option + 8; // append the rest after "sysimage" + } + option = processed_option.c_str(); + } + } + llvm::SmallVector, 0> res; TargetData arg{}; auto reset_arg = [&] { @@ -633,6 +647,12 @@ static inline jl_image_t parse_sysimg(jl_image_buf_t image, F &&callback, void * const jl_image_pointers_t *pointers = (const jl_image_pointers_t *)image.pointers; const void *ids = pointers->target_data; + + // Set the sysimage CPU target from the stored string + if (pointers->cpu_target_string) { + jl_set_sysimage_cpu_target(pointers->cpu_target_string); + } + jl_value_t* rejection_reason = nullptr; JL_GC_PUSH1(&rejection_reason); uint32_t target_idx = callback(ctx, ids, &rejection_reason); @@ -972,19 +992,6 @@ static std::string jl_get_cpu_features_llvm(void) attr.append(ele.getKey().str()); } } - // Explicitly disabled features need to be added at the end so that - // they are not re-enabled by other features that implies them by default. - for (auto &ele: HostFeatures) { - if (!ele.getValue()) { - if (!attr.empty()) { - attr.append(",-"); - } - else { - attr.append("-"); - } - attr.append(ele.getKey().str()); - } - } return attr; } @@ -1002,6 +1009,9 @@ static std::string jl_get_cpu_features_llvm(void) #endif +// Global variable to store the CPU target string used for the sysimage +static std::string sysimage_cpu_target; + JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void) { return jl_cstr_to_string(host_cpu_name().c_str()); @@ -1038,3 +1048,17 @@ extern "C" JL_DLLEXPORT void jl_reflect_feature_names(const FeatureName **fnames *fnames = feature_names; *nf = nfeature_names; } + +extern "C" JL_DLLEXPORT jl_value_t *jl_get_sysimage_cpu_target(void) { + if (sysimage_cpu_target.empty()) { + return jl_cstr_to_string("native"); + } + return jl_cstr_to_string(sysimage_cpu_target.c_str()); +} + +// Function to set the sysimage CPU target (called during initialization) +void jl_set_sysimage_cpu_target(const char *cpu_target) { + if (cpu_target) { + sysimage_cpu_target = cpu_target; + } +} diff --git a/src/processor.h b/src/processor.h index 65b634fd0ba26..b22e12d0aa4c0 100644 --- a/src/processor.h +++ b/src/processor.h @@ -195,6 +195,8 @@ typedef struct { // This contains the number of targets // in addition to the name and feature set of each target. const void *target_data; + // Original CPU target string used to build this sysimage + const char *cpu_target_string; } jl_image_pointers_t; /** @@ -210,10 +212,15 @@ typedef struct { jl_image_t jl_init_processor_sysimg(jl_image_buf_t image, const char *cpu_target); jl_image_t jl_init_processor_pkgimg(jl_image_buf_t image); +// Internal function to set the sysimage CPU target during initialization +void jl_set_sysimage_cpu_target(const char *cpu_target); + // Return the name of the host CPU as a julia string. JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void); // Return the features of the host CPU as a julia string. JL_DLLEXPORT jl_value_t *jl_get_cpu_features(void); +// Return the CPU target string used to build the current sysimage +JL_DLLEXPORT jl_value_t *jl_get_sysimage_cpu_target(void); // Dump the name and feature set of the host CPU JL_DLLEXPORT jl_value_t *jl_cpu_has_fma(int bits); // Check if the CPU has native FMA instructions; diff --git a/src/processor_arm.cpp b/src/processor_arm.cpp index 66704a718a14d..b73f5c3e4bbac 100644 --- a/src/processor_arm.cpp +++ b/src/processor_arm.cpp @@ -166,9 +166,11 @@ enum class CPU : uint32_t { apple_a14, apple_a15, apple_a16, + apple_a17, apple_m1, apple_m2, apple_m3, + apple_m4, apple_s4, apple_s5, @@ -355,9 +357,11 @@ constexpr auto apple_a13 = armv8_4a_crypto | get_feature_masks(fp16fml, fullfp16 constexpr auto apple_a14 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3); constexpr auto apple_a15 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); constexpr auto apple_a16 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); +constexpr auto apple_a17 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); constexpr auto apple_m1 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3); constexpr auto apple_m2 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); constexpr auto apple_m3 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); +constexpr auto apple_m4 = armv8_5a_crypto | get_feature_masks(dotprod,fp16fml, fullfp16, sha3, i8mm, bf16); // Features based on https://github.com/llvm/llvm-project/blob/82507f1798768280cf5d5aab95caaafbc7fe6f47/llvm/include/llvm/Support/AArch64TargetParser.def // and sysctl -a hw.optional constexpr auto apple_s4 = apple_a12; @@ -441,9 +445,11 @@ static constexpr CPUSpec cpus[] = { {"apple-a14", CPU::apple_a14, CPU::apple_a13, 120000, Feature::apple_a14}, {"apple-a15", CPU::apple_a15, CPU::apple_a14, 160000, Feature::apple_a15}, {"apple-a16", CPU::apple_a16, CPU::apple_a14, 160000, Feature::apple_a16}, + {"apple-a17", CPU::apple_a17, CPU::apple_a16, 190000, Feature::apple_a17}, {"apple-m1", CPU::apple_m1, CPU::apple_a14, 130000, Feature::apple_m1}, {"apple-m2", CPU::apple_m2, CPU::apple_m1, 160000, Feature::apple_m2}, {"apple-m3", CPU::apple_m3, CPU::apple_m2, 180000, Feature::apple_m3}, + {"apple-m4", CPU::apple_m4, CPU::apple_m3, 190000, Feature::apple_m4}, {"apple-s4", CPU::apple_s4, CPU::generic, 100000, Feature::apple_s4}, {"apple-s5", CPU::apple_s5, CPU::generic, 100000, Feature::apple_s5}, {"thunderx3t110", CPU::marvell_thunderx3t110, CPU::cavium_thunderx2t99, 110000, @@ -722,6 +728,8 @@ static NOINLINE std::pair> _get_host_cpu() return std::make_pair((uint32_t)CPU::apple_m2, Feature::apple_m2); else if (cpu_name.find("M3") != StringRef ::npos) return std::make_pair((uint32_t)CPU::apple_m3, Feature::apple_m3); + else if (cpu_name.find("M4") != StringRef ::npos) + return std::make_pair((uint32_t)CPU::apple_m4, Feature::apple_m4); else return std::make_pair((uint32_t)CPU::apple_m1, Feature::apple_m1); } @@ -1042,7 +1050,10 @@ static CPU get_cpu_name(CPUID cpuid) default: return CPU::generic; } case 0x61: // 'a': Apple - // https://opensource.apple.com/source/xnu/xnu-7195.141.2/osfmk/arm/cpuid.h.auto.html + // Data here is partially based on these sources: + // https://github.com/apple-oss-distributions/xnu/blob/main/osfmk/arm/cpuid.h + // https://asahilinux.org/docs/hw/soc/soc-codenames/#socs + // https://github.com/llvm/llvm-project/blob/main/llvm/lib/Target/AArch64/AArch64Processors.td switch (cpuid.part) { case 0x0: // Swift return CPU::apple_swift; @@ -1067,31 +1078,57 @@ static CPU get_cpu_name(CPUID cpuid) return CPU::apple_a12; case 0xF: // Tempest M9 return CPU::apple_s4; - case 0x12: // Lightning - case 0x13: // Thunder + case 0x12: // H12 Cebu p-Core "Lightning" + case 0x13: // H12 Cebu e-Core "Thunder" return CPU::apple_a13; - case 0x20: // Icestorm - case 0x21: // Firestorm + case 0x20: // H13 Sicily e-Core "Icestorm" + case 0x21: // H13 Sicily p-Core "Firestorm" return CPU::apple_a14; - case 0x22: // Icestorm m1 - case 0x23: // Firestorm m1 - case 0x24: - case 0x25: // From https://github.com/AsahiLinux/m1n1/blob/3b9a71422e45209ef57c563e418f877bf54358be/src/chickens.c#L9 - case 0x28: - case 0x29: + case 0x22: // H13G Tonga e-Core "Icestorm" used in Apple M1 + case 0x23: // H13G Tonga p-Core "Firestorm" used in Apple M1 + case 0x24: // H13J Jade Chop e-Core "Icestorm" used in Apple M1 Pro + case 0x25: // H13J Jade Chop p-Core "Firestorm" used in Apple M1 Pro + case 0x28: // H13J Jade Die e-Core "Icestorm" used in Apple M1 Max / Ultra + case 0x29: // H13J Jade Die p-Core "Firestorm" used in Apple M1 Max / Ultra return CPU::apple_m1; - case 0x30: // Blizzard m2 - case 0x31: // Avalanche m2 - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x38: - case 0x39: + case 0x30: // H14 Ellis e-Core "Blizzard" used in Apple A15 + case 0x31: // H14 Ellis p-Core "Avalanche" used in Apple A15 + return CPU::apple_a15; + case 0x32: // H14G Staten e-Core "Blizzard" used in Apple M2 + case 0x33: // H14G Staten p-Core "Avalanche" used in Apple M2 + case 0x34: // H14S Rhodes Chop e-Core "Blizzard" used in Apple M2 Pro + case 0x35: // H14S Rhodes Chop p-Core "Avalanche" used in Apple M2 Pro + case 0x38: // H14C Rhodes Die e-Core "Blizzard" used in Apple M2 Max / Ultra + case 0x39: // H14C Rhodes Die p-Core "Avalanche" used in Apple M2 Max / Ultra return CPU::apple_m2; - case 0x49: // Everest m3 - case 0x48: // Sawtooth m3 + case 0x40: // H15 Crete e-Core "Sawtooth" used in Apple A16 + case 0x41: // H15 Crete p-Core "Everest" used in Apple A16 + return CPU::apple_a16; + case 0x42: // H15 Ibiza e-Core "Sawtooth" used in Apple M3 + case 0x43: // H15 Ibiza p-Core "Everest" used in Apple M3 + case 0x44: // H15 Lobos e-Core "Sawtooth" used in Apple M3 Pro + case 0x45: // H15 Lobos p-Core "Everest" used in Apple M3 Pro + case 0x49: // H15 Palma e-Core "Sawtooth" used in Apple M3 Max + case 0x48: // H15 Palma p-Core "Everest" used in Apple M3 Max return CPU::apple_m3; + //case 0x46: // M11 e-Core "Sawtooth" used in Apple S9 + //case 0x47: does not exist + //return CPU::apple_s9; + case 0x50: // H15 Coll e-Core "Sawtooth" used in Apple A17 Pro + case 0x51: // H15 Coll p-Core "Everest" used in Apple A17 Pro + return CPU::apple_a17; + case 0x52: // H16G Donan e-Core used in Apple M4 + case 0x53: // H16H Donan p-Core used in Apple M4 + case 0x54: // H16S Brava S e-Core used in Apple M4 Pro + case 0x55: // H16S Brava S p-Core used in Apple M4 Pro + case 0x58: // H16C Brava C e-Core used in Apple M4 Max + case 0x59: // H16C Brava C p-Core used in Apple M4 Max + return CPU::apple_m4; + //case 0x60: // H17P Tahiti e-Core used in Apple A18 Pro + //case 0x61: // H17P Tahiti p-Core used in Apple A18 Pro + //case 0x6a: // H17A Tupai e-Core used in Apple A18 + //case 0x6b: // H17A Tupai p-Core used in Apple A18 + //return CPU::apple_a18; default: return CPU::generic; } case 0x68: // 'h': Huaxintong Semiconductor diff --git a/src/rtutils.c b/src/rtutils.c index 587a82b97a2f5..98cea3b6305cd 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -5,6 +5,8 @@ */ #include "platform.h" +#include +#include #include #include #include @@ -579,7 +581,7 @@ JL_DLLEXPORT jl_value_t *jl_stderr_obj(void) JL_NOTSAFEPOINT if (jl_base_module == NULL) return NULL; jl_binding_t *stderr_obj = jl_get_module_binding(jl_base_module, jl_symbol("stderr"), 0); - return stderr_obj ? jl_get_binding_value_if_resolved(stderr_obj) : NULL; + return stderr_obj ? jl_get_latest_binding_value_if_resolved_debug_only(stderr_obj) : NULL; } // toys for debugging --------------------------------------------------------- @@ -671,12 +673,11 @@ JL_DLLEXPORT jl_value_t *jl_argument_datatype(jl_value_t *argt JL_PROPAGATES_ROO static int is_globname_binding(jl_value_t *v, jl_datatype_t *dv) JL_NOTSAFEPOINT { - jl_sym_t *globname = dv->name->mt != NULL ? dv->name->mt->name : NULL; + jl_sym_t *globname = dv->name->singletonname; if (globname && dv->name->module) { jl_binding_t *b = jl_get_module_binding(dv->name->module, globname, 0); - jl_value_t *bv = jl_get_binding_value_if_resolved_and_const(b); - // The `||` makes this function work for both function instances and function types. - if (bv && (bv == v || jl_typeof(bv) == v)) + jl_value_t *bv = jl_get_latest_binding_value_if_resolved_and_const_debug_only(b); + if (bv && ((jl_value_t*)dv == v ? jl_typeof(bv) == v : bv == v)) return 1; } return 0; @@ -684,7 +685,7 @@ static int is_globname_binding(jl_value_t *v, jl_datatype_t *dv) JL_NOTSAFEPOINT static int is_globfunction(jl_value_t *v, jl_datatype_t *dv, jl_sym_t **globname_out) JL_NOTSAFEPOINT { - jl_sym_t *globname = dv->name->mt != NULL ? dv->name->mt->name : NULL; + jl_sym_t *globname = dv->name->singletonname; *globname_out = globname; if (globname && !strchr(jl_symbol_name(globname), '#') && !strchr(jl_symbol_name(globname), '@')) { return 1; @@ -692,12 +693,12 @@ static int is_globfunction(jl_value_t *v, jl_datatype_t *dv, jl_sym_t **globname return 0; } -static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, int wrap) JL_NOTSAFEPOINT +static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, int wrap, int raw) JL_NOTSAFEPOINT { size_t n = 0; if (wrap) n += jl_printf(out, "\""); - if (!u8_isvalid(str, len)) { + if (!raw && !u8_isvalid(str, len)) { // alternate print algorithm that preserves data if it's not UTF-8 static const char hexdig[] = "0123456789abcdef"; for (size_t i = 0; i < len; i++) { @@ -714,7 +715,11 @@ static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, int special = 0; for (size_t i = 0; i < len; i++) { uint8_t c = str[i]; - if (c < 32 || c == 0x7f || c == '\\' || c == '"' || c == '$') { + if (raw && ((c == '\\' && i == len-1) || c == '"')) { + special = 1; + break; + } + else if (!raw && (c < 32 || c == 0x7f || c == '\\' || c == '"' || c == '$')) { special = 1; break; } @@ -723,6 +728,25 @@ static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, jl_uv_puts(out, str, len); n += len; } + else if (raw) { + // REF: Base.escape_raw_string + int escapes = 0; + for (size_t i = 0; i < len; i++) { + uint8_t c = str[i]; + if (c == '\\') { + escapes++; + } + else { + if (c == '"') + for (escapes++; escapes > 0; escapes--) + n += jl_printf(out, "\\"); + escapes = 0; + } + n += jl_printf(out, "%c", str[i]); + } + for (; escapes > 0; escapes--) + n += jl_printf(out, "\\"); + } else { char buf[512]; size_t i = 0; @@ -738,18 +762,28 @@ static size_t jl_static_show_string(JL_STREAM *out, const char *str, size_t len, return n; } +static int jl_is_quoted_sym(const char *sn) +{ + static const char *const quoted_syms[] = {":", "::", ":=", "=", "==", "===", "=>", "`"}; + for (int i = 0; i < sizeof quoted_syms / sizeof *quoted_syms; i++) + if (!strcmp(sn, quoted_syms[i])) + return 1; + return 0; +} + +// TODO: in theory, we need a separate function for showing symbols in an +// expression context (where `Symbol("foo\x01bar")` is ok) and a syntactic +// context (where var"" must be used). static size_t jl_static_show_symbol(JL_STREAM *out, jl_sym_t *name) JL_NOTSAFEPOINT { size_t n = 0; const char *sn = jl_symbol_name(name); - int quoted = !jl_is_identifier(sn) && !jl_is_operator(sn); - if (quoted) { - n += jl_printf(out, "var"); - // TODO: this is not quite right, since repr uses String escaping rules, and Symbol uses raw string rules - n += jl_static_show_string(out, sn, strlen(sn), 1); + if (jl_is_identifier(sn) || (jl_is_operator(sn) && !jl_is_quoted_sym(sn))) { + n += jl_printf(out, "%s", sn); } else { - n += jl_printf(out, "%s", sn); + n += jl_printf(out, "var"); + n += jl_static_show_string(out, sn, strlen(sn), 1, 1); } return n; } @@ -778,6 +812,51 @@ static int jl_static_is_function_(jl_datatype_t *vt) JL_NOTSAFEPOINT { return 0; } +static size_t jl_static_show_float(JL_STREAM *out, double v, + jl_datatype_t *vt) JL_NOTSAFEPOINT +{ + size_t n = 0; + // TODO: non-canonical NaNs do not round-trip + // TOOD: BFloat16 + const char *size_suffix = vt == jl_float16_type ? "16" : + vt == jl_float32_type ? "32" : + ""; + // Requires minimum 1 (sign) + 17 (sig) + 1 (dot) + 5 ("e-123") + 1 (null) + char buf[32]; + // Base B significand digits required to print n base-b significand bits + // (including leading 1): N = 2 + floor(n/log(b, B)) + // Float16 5 + // Float32 9 + // Float64 17 + // REF: https://dl.acm.org/doi/pdf/10.1145/93542.93559 + if (isnan(v)) { + n += jl_printf(out, "NaN%s", size_suffix); + } + else if (isinf(v)) { + n += jl_printf(out, "%sInf%s", v < 0 ? "-" : "", size_suffix); + } + else if (vt == jl_float64_type) { + n += jl_printf(out, "%#.17g", v); + } + else if (vt == jl_float32_type) { + size_t m = snprintf(buf, sizeof buf, "%.9g", v); + // If the exponent was printed, replace it with 'f' + char *p = (char *)memchr(buf, 'e', m); + if (p) + *p = 'f'; + jl_uv_puts(out, buf, m); + n += m; + // If no exponent was printed, we must add one + if (!p) + n += jl_printf(out, "f0"); + } + else { + assert(vt == jl_float16_type); + n += jl_printf(out, "Float16(%#.5g)", v); + } + return n; +} + // `v` might be pointing to a field inlined in a structure therefore // `jl_typeof(v)` may not be the same with `vt` and only `vt` should be // used to determine the type of the value. @@ -815,6 +894,9 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt else if (v == (jl_value_t*)jl_methtable_type) { n += jl_printf(out, "Core.MethodTable"); } + else if (v == (jl_value_t*)jl_methcache_type) { + n += jl_printf(out, "Core.MethodCache"); + } else if (v == (jl_value_t*)jl_any_type) { n += jl_printf(out, "Any"); } @@ -955,17 +1037,21 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt int f = *(uint32_t*)jl_data_ptr(v); n += jl_printf(out, "#", f, jl_intrinsic_name(f)); } + else if (vt == jl_long_type) { + // Avoid unnecessary Int64(x)/Int32(x) + n += jl_printf(out, "%" PRIdPTR, *(intptr_t*)v); + } else if (vt == jl_int64_type) { - n += jl_printf(out, "%" PRId64, *(int64_t*)v); + n += jl_printf(out, "Int64(%" PRId64 ")", *(int64_t*)v); } else if (vt == jl_int32_type) { - n += jl_printf(out, "%" PRId32, *(int32_t*)v); + n += jl_printf(out, "Int32(%" PRId32 ")", *(int32_t*)v); } else if (vt == jl_int16_type) { - n += jl_printf(out, "%" PRId16, *(int16_t*)v); + n += jl_printf(out, "Int16(%" PRId16 ")", *(int16_t*)v); } else if (vt == jl_int8_type) { - n += jl_printf(out, "%" PRId8, *(int8_t*)v); + n += jl_printf(out, "Int8(%" PRId8 ")", *(int8_t*)v); } else if (vt == jl_uint64_type) { n += jl_printf(out, "0x%016" PRIx64, *(uint64_t*)v); @@ -979,18 +1065,14 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt else if (vt == jl_uint8_type) { n += jl_printf(out, "0x%02" PRIx8, *(uint8_t*)v); } - else if (jl_pointer_type && jl_is_cpointer_type((jl_value_t*)vt)) { -#ifdef _P64 - n += jl_printf(out, "0x%016" PRIx64, *(uint64_t*)v); -#else - n += jl_printf(out, "0x%08" PRIx32, *(uint32_t*)v); -#endif + else if (vt == jl_float16_type) { + n += jl_static_show_float(out, julia_half_to_float(*(uint16_t *)v), vt); } else if (vt == jl_float32_type) { - n += jl_printf(out, "%gf", *(float*)v); + n += jl_static_show_float(out, *(float *)v, vt); } else if (vt == jl_float64_type) { - n += jl_printf(out, "%g", *(double*)v); + n += jl_static_show_float(out, *(double *)v, vt); } else if (vt == jl_bool_type) { n += jl_printf(out, "%s", *(uint8_t*)v ? "true" : "false"); @@ -998,8 +1080,11 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt else if (v == jl_nothing || (jl_nothing && (jl_value_t*)vt == jl_typeof(jl_nothing))) { n += jl_printf(out, "nothing"); } + else if (v == (jl_value_t*)jl_method_table) { + n += jl_printf(out, "Core.methodtable"); + } else if (vt == jl_string_type) { - n += jl_static_show_string(out, jl_string_data(v), jl_string_len(v), 1); + n += jl_static_show_string(out, jl_string_data(v), jl_string_len(v), 1, 0); } else if (v == jl_bottom_type) { n += jl_printf(out, "Union{}"); @@ -1209,6 +1294,11 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt } else { char *ptr = ((char*)m->ptr) + j * layout->size; + if (layout->flags.arrayelem_islocked) { + // Skip the lock at the beginning for locked arrays + size_t lock_size = sizeof(jl_mutex_t); + ptr += lock_size; + } n += jl_static_show_x_(out, (jl_value_t*)ptr, (jl_datatype_t*)(typetagdata ? jl_nth_union_component(el_type, typetagdata[j]) : el_type), depth, ctx); @@ -1427,10 +1517,8 @@ size_t jl_static_show_func_sig_(JL_STREAM *s, jl_value_t *type, jl_static_show_c return n; } if ((jl_nparams(ftype) == 0 || ftype == ((jl_datatype_t*)ftype)->name->wrapper) && - ((jl_datatype_t*)ftype)->name->mt && - ((jl_datatype_t*)ftype)->name->mt != jl_type_type_mt && - ((jl_datatype_t*)ftype)->name->mt != jl_nonfunction_mt) { - n += jl_static_show_symbol(s, ((jl_datatype_t*)ftype)->name->mt->name); + !jl_is_type_type(ftype) && !jl_is_type_type((jl_value_t*)((jl_datatype_t*)ftype)->super)) { // aka !iskind + n += jl_static_show_symbol(s, ((jl_datatype_t*)ftype)->name->singletonname); } else { n += jl_printf(s, "(::"); @@ -1506,15 +1594,18 @@ JL_DLLEXPORT void jl_test_failure_breakpoint(jl_value_t *v) // logging tools -------------------------------------------------------------- +// DO NOT USE THIS FUNCTION FOR NEW CODE +// The internal should not be doing anything that requires logging, which means most functions would trigger UB if calling this void jl_log(int level, jl_value_t *module, jl_value_t *group, jl_value_t *id, jl_value_t *file, jl_value_t *line, jl_value_t *kwargs, jl_value_t *msg) { - static jl_value_t *logmsg_func = NULL; - if (!logmsg_func && jl_base_module) { - jl_value_t *corelogging = jl_get_global(jl_base_module, jl_symbol("CoreLogging")); + jl_value_t *logmsg_func = NULL; + jl_task_t *ct = jl_current_task; + if (jl_base_module) { + jl_value_t *corelogging = jl_get_global_value(jl_base_module, jl_symbol("CoreLogging"), ct->world_age); if (corelogging && jl_is_module(corelogging)) { - logmsg_func = jl_get_global((jl_module_t*)corelogging, jl_symbol("logmsg_shim")); + logmsg_func = jl_get_global_value((jl_module_t*)corelogging, jl_symbol("logmsg_shim"), ct->world_age); } } if (!logmsg_func) { @@ -1529,10 +1620,10 @@ void jl_log(int level, jl_value_t *module, jl_value_t *group, jl_value_t *id, } jl_printf(str, "\n@ "); if (jl_is_string(file)) { - jl_static_show_string(str, jl_string_data(file), jl_string_len(file), 0); + jl_static_show_string(str, jl_string_data(file), jl_string_len(file), 0, 0); } else if (jl_is_symbol(file)) { - jl_static_show_string(str, jl_symbol_name((jl_sym_t*)file), strlen(jl_symbol_name((jl_sym_t*)file)), 0); + jl_static_show_string(str, jl_symbol_name((jl_sym_t*)file), strlen(jl_symbol_name((jl_sym_t*)file)), 0, 0); } jl_printf(str, ":"); jl_static_show(str, line); diff --git a/src/runtime_ccall.cpp b/src/runtime_ccall.cpp index 97b14c06699a8..7d69b8a2f260f 100644 --- a/src/runtime_ccall.cpp +++ b/src/runtime_ccall.cpp @@ -329,6 +329,8 @@ jl_value_t *jl_get_cfunction_trampoline( JL_GCC_IGNORE_STOP struct cfuncdata_t { + _Atomic(void *) fptr; + _Atomic(size_t) last_world; jl_code_instance_t** plast_codeinst; jl_code_instance_t* last_codeinst; void *unspecialized; @@ -360,7 +362,7 @@ static jl_mutex_t cfun_lock; // read theFptr // acquire jl_world_counter extern "C" JL_DLLEXPORT -void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) *last_world, void *data) +void *jl_get_abi_converter(jl_task_t *ct, void *data) { cfuncdata_t *cfuncdata = (cfuncdata_t*)data; jl_value_t *sigt = *cfuncdata->sigt; @@ -375,8 +377,8 @@ void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) // check first, while behind this lock, of the validity of the current contents of this cfunc thunk JL_LOCK(&cfun_lock); do { - size_t last_world_v = jl_atomic_load_relaxed(last_world); - void *f = jl_atomic_load_relaxed(fptr); + size_t last_world_v = jl_atomic_load_relaxed(&cfuncdata->last_world); + void *f = jl_atomic_load_relaxed(&cfuncdata->fptr); jl_code_instance_t *last_ci = cfuncdata->plast_codeinst ? *cfuncdata->plast_codeinst : nullptr; world = jl_atomic_load_acquire(&jl_world_counter); ct->world_age = world; @@ -388,14 +390,14 @@ void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) if (f != nullptr) { if (last_ci == nullptr) { if (mi == nullptr) { - jl_atomic_store_release(last_world, world); + jl_atomic_store_release(&cfuncdata->last_world, world); JL_UNLOCK(&cfun_lock); return f; } } else { if (jl_get_ci_mi(last_ci) == mi && jl_atomic_load_relaxed(&last_ci->max_world) >= world) { // same dispatch and source - jl_atomic_store_release(last_world, world); + jl_atomic_store_release(&cfuncdata->last_world, world); JL_UNLOCK(&cfun_lock); return f; } @@ -403,59 +405,44 @@ void *jl_get_abi_converter(jl_task_t *ct, _Atomic(void*) *fptr, _Atomic(size_t) } JL_UNLOCK(&cfun_lock); // next, try to figure out what the target should look like (outside of the lock since this is very slow) - codeinst = mi ? jl_type_infer(mi, world, SOURCE_MODE_ABI) : nullptr; + codeinst = mi ? jl_type_infer(mi, world, SOURCE_MODE_ABI, jl_options.trim) : nullptr; // relock for the remainder of the function JL_LOCK(&cfun_lock); } while (jl_atomic_load_acquire(&jl_world_counter) != world); // restart entirely, since jl_world_counter changed thus jl_get_specialization1 might have changed // double-check if the values were set on another thread - size_t last_world_v = jl_atomic_load_relaxed(last_world); - void *f = jl_atomic_load_relaxed(fptr); + size_t last_world_v = jl_atomic_load_relaxed(&cfuncdata->last_world); + void *f = jl_atomic_load_relaxed(&cfuncdata->fptr); if (world == last_world_v) { JL_UNLOCK(&cfun_lock); return f; // another thread fixed this up while we were away } - auto assign_fptr = [fptr, last_world, cfuncdata, world, codeinst](void *f) { + auto assign_fptr = [cfuncdata, world, codeinst](void *f) { cfuncdata->plast_codeinst = &cfuncdata->last_codeinst; cfuncdata->last_codeinst = codeinst; - jl_atomic_store_relaxed(fptr, f); - jl_atomic_store_release(last_world, world); + jl_atomic_store_relaxed(&cfuncdata->fptr, f); + jl_atomic_store_release(&cfuncdata->last_world, world); JL_UNLOCK(&cfun_lock); return f; }; - jl_callptr_t invoke = nullptr; - if (codeinst != NULL) { - jl_value_t *astrt = codeinst->rettype; - if (astrt != (jl_value_t*)jl_bottom_type && - jl_type_intersection(astrt, declrt) == jl_bottom_type) { - // Do not warn if the function never returns since it is - // occasionally required by the C API (typically error callbacks) - // even though we're likely to encounter memory errors in that case - jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); - } - uint8_t specsigflags; - jl_read_codeinst_invoke(codeinst, &specsigflags, &invoke, &f, 1); - if (invoke != nullptr) { - if (invoke == jl_fptr_const_return_addr) { - return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, nullptr, false)); - } - else if (invoke == jl_fptr_args_addr) { - assert(f); - if (!specsig && jl_subtype(astrt, declrt)) - return assign_fptr(f); - return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, f, false)); - } - else if (specsigflags & 0b1) { - assert(f); - if (specsig && jl_egal(mi->specTypes, sigt) && jl_egal(declrt, astrt)) - return assign_fptr(f); - return assign_fptr(jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, f, true)); - } - } + bool is_opaque_closure = false; + jl_abi_t from_abi = { jl_pinned_ref_create(jl_value_t, sigt), jl_pinned_ref_create(jl_value_t, declrt), nargs, specsig, is_opaque_closure }; + if (codeinst == nullptr) { + // Generate an adapter to a dynamic dispatch + if (cfuncdata->unspecialized == nullptr) + cfuncdata->unspecialized = jl_jit_abi_converter(ct, from_abi, nullptr); + + return assign_fptr(cfuncdata->unspecialized); + } + + jl_value_t *astrt = codeinst->rettype; + if (astrt != (jl_value_t*)jl_bottom_type && + jl_type_intersection(astrt, declrt) == jl_bottom_type) { + // Do not warn if the function never returns since it is + // occasionally required by the C API (typically error callbacks) + // even though we're likely to encounter memory errors in that case + jl_printf(JL_STDERR, "WARNING: cfunction: return type of %s does not match\n", name_from_method_instance(mi)); } - f = jl_jit_abi_converter(ct, cfuncdata->unspecialized, declrt, sigt, nargs, specsig, codeinst, invoke, nullptr, false); - if (codeinst == nullptr) - cfuncdata->unspecialized = f; - return assign_fptr(f); + return assign_fptr(jl_jit_abi_converter(ct, from_abi, codeinst)); } void jl_init_runtime_ccall(void) diff --git a/src/scheduler.c b/src/scheduler.c index 731a0c5146605..b13e4072c7d73 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -329,12 +329,15 @@ void jl_task_wait_empty(void) jl_task_t *ct = jl_current_task; if (jl_atomic_load_relaxed(&ct->tid) == 0 && jl_base_module) { jl_wait_empty_begin(); - jl_value_t *f = jl_get_global(jl_base_module, jl_symbol("wait")); - wait_empty = ct; size_t lastage = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - if (f) + jl_value_t *f = jl_get_global_value(jl_base_module, jl_symbol("wait"), ct->world_age); + wait_empty = ct; + if (f) { + JL_GC_PUSH1(&f); jl_apply_generic(f, NULL, 0); + JL_GC_POP(); + } // we are back from jl_task_get_next now ct->world_age = lastage; wait_empty = NULL; diff --git a/src/signal-handling.c b/src/signal-handling.c index 7a5e5d1bedc41..f65be903c2d5b 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -102,7 +102,7 @@ void jl_init_profile_lock(void) #endif } -uintptr_t jl_lock_profile_rd_held(void) +static uintptr_t jl_lock_profile_rd_held(void) JL_NOTSAFEPOINT { #ifndef _OS_WINDOWS_ return (uintptr_t)pthread_getspecific(debuginfo_asyncsafe_held); @@ -111,38 +111,69 @@ uintptr_t jl_lock_profile_rd_held(void) #endif } -void jl_lock_profile(void) +int jl_lock_profile(void) { uintptr_t held = jl_lock_profile_rd_held(); - if (held++ == 0) + if (held == -1) + return 0; + if (held == 0) { + held = -1; +#ifndef _OS_WINDOWS_ + pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); +#else + TlsSetValue(debuginfo_asyncsafe_held, (void*)held); +#endif uv_rwlock_rdlock(&debuginfo_asyncsafe); + held = 0; + } + held++; #ifndef _OS_WINDOWS_ pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); #else TlsSetValue(debuginfo_asyncsafe_held, (void*)held); #endif + return 1; } JL_DLLEXPORT void jl_unlock_profile(void) { uintptr_t held = jl_lock_profile_rd_held(); - assert(held); - if (--held == 0) - uv_rwlock_rdunlock(&debuginfo_asyncsafe); + assert(held && held != -1); + held--; #ifndef _OS_WINDOWS_ pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); #else TlsSetValue(debuginfo_asyncsafe_held, (void*)held); #endif + if (held == 0) + uv_rwlock_rdunlock(&debuginfo_asyncsafe); } -void jl_lock_profile_wr(void) +int jl_lock_profile_wr(void) { + uintptr_t held = jl_lock_profile_rd_held(); + if (held) + return 0; + held = -1; +#ifndef _OS_WINDOWS_ + pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); +#else + TlsSetValue(debuginfo_asyncsafe_held, (void*)held); +#endif uv_rwlock_wrlock(&debuginfo_asyncsafe); + return 1; } void jl_unlock_profile_wr(void) { + uintptr_t held = jl_lock_profile_rd_held(); + assert(held == -1); + held = 0; +#ifndef _OS_WINDOWS_ + pthread_setspecific(debuginfo_asyncsafe_held, (void*)held); +#else + TlsSetValue(debuginfo_asyncsafe_held, (void*)held); +#endif uv_rwlock_wrunlock(&debuginfo_asyncsafe); } @@ -606,7 +637,7 @@ void jl_critical_error(int sig, int si_code, bt_context_t *context, jl_task_t *c else jl_safe_printf("\n[%d] signal %d: %s\n", getpid(), sig, strsignal(sig)); } - jl_safe_printf("in expression starting at %s:%d\n", jl_filename, jl_lineno); + jl_safe_printf("in expression starting at %s:%d\n", jl_atomic_load_relaxed(&jl_filename), jl_atomic_load_relaxed(&jl_lineno)); if (context && ct) { // Must avoid extended backtrace frames here unless we're sure bt_data // is properly rooted. diff --git a/src/signals-mach.c b/src/signals-mach.c index bca3404fd101a..39a46729deb22 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -146,8 +146,6 @@ static void jl_mach_gc_wait(jl_ptls_t ptls2, mach_port_t thread, int16_t tid) static mach_port_t segv_port = 0; -#define STR(x) #x -#define XSTR(x) STR(x) #define HANDLE_MACH_ERROR(msg, retval) \ if (retval != KERN_SUCCESS) { mach_error(msg XSTR(: __FILE__:__LINE__:), (retval)); abort(); } diff --git a/src/stackwalk.c b/src/stackwalk.c index 1f3c8d690c8ce..6f8f9c8b89db1 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -326,7 +326,11 @@ static void decode_backtrace(jl_bt_element_t *bt_data, size_t bt_size, bt = *btout = jl_alloc_array_1d(array_ptr_void_type, bt_size); static_assert(sizeof(jl_bt_element_t) == sizeof(void*), "jl_bt_element_t is presented as Ptr{Cvoid} on julia side"); - memcpy(jl_array_data(bt, jl_bt_element_t), bt_data, bt_size * sizeof(jl_bt_element_t)); + if (bt_data != NULL) { + memcpy(jl_array_data(bt, jl_bt_element_t), bt_data, bt_size * sizeof(jl_bt_element_t)); + } else { + assert(bt_size == 0); + } bt2 = *bt2out = jl_alloc_array_1d(jl_array_any_type, 0); // Scan the backtrace buffer for any gc-managed values for (size_t i = 0; i < bt_size; i += jl_bt_entry_size(bt_data + i)) { diff --git a/src/staticdata.c b/src/staticdata.c index 139205d28ad13..eb4c17504ff57 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -90,6 +90,22 @@ static const size_t WORLD_AGE_REVALIDATION_SENTINEL = 0x1; JL_DLLEXPORT size_t jl_require_world = ~(size_t)0; JL_DLLEXPORT _Atomic(size_t) jl_first_image_replacement_world = ~(size_t)0; +// This structure is used to store hash tables for the memoization +// of queries in staticdata.c (currently only `type_in_worklist`). +typedef struct { + htable_t type_in_worklist; +} jl_query_cache; + +static void init_query_cache(jl_query_cache *cache) +{ + htable_new(&cache->type_in_worklist, 0); +} + +static void destroy_query_cache(jl_query_cache *cache) +{ + htable_free(&cache->type_in_worklist); +} + #include "staticdata_utils.c" #include "precompile_utils.c" @@ -100,228 +116,176 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 197 +#define NUM_TAGS 151 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. -jl_value_t **const*const get_tags(void) { +static void get_tags(jl_value_t **tags[NUM_TAGS]) +{ // Make sure to keep an extra slot at the end to sentinel length - static void * _tags[NUM_TAGS] = {NULL}; - - // Lazyily-initialize this list - if (_tags[0] == NULL) { - unsigned int i = 0; -#define INSERT_TAG(sym) _tags[i++] = &(sym) - // builtin types - INSERT_TAG(jl_any_type); - INSERT_TAG(jl_symbol_type); - INSERT_TAG(jl_ssavalue_type); - INSERT_TAG(jl_datatype_type); - INSERT_TAG(jl_slotnumber_type); - INSERT_TAG(jl_simplevector_type); - INSERT_TAG(jl_array_type); - INSERT_TAG(jl_expr_type); - INSERT_TAG(jl_binding_type); - INSERT_TAG(jl_binding_partition_type); - INSERT_TAG(jl_globalref_type); - INSERT_TAG(jl_string_type); - INSERT_TAG(jl_module_type); - INSERT_TAG(jl_tvar_type); - INSERT_TAG(jl_method_instance_type); - INSERT_TAG(jl_method_type); - INSERT_TAG(jl_code_instance_type); - INSERT_TAG(jl_linenumbernode_type); - INSERT_TAG(jl_lineinfonode_type); - INSERT_TAG(jl_gotonode_type); - INSERT_TAG(jl_quotenode_type); - INSERT_TAG(jl_gotoifnot_type); - INSERT_TAG(jl_enternode_type); - INSERT_TAG(jl_argument_type); - INSERT_TAG(jl_returnnode_type); - INSERT_TAG(jl_const_type); - INSERT_TAG(jl_partial_struct_type); - INSERT_TAG(jl_partial_opaque_type); - INSERT_TAG(jl_interconditional_type); - INSERT_TAG(jl_method_match_type); - INSERT_TAG(jl_pinode_type); - INSERT_TAG(jl_phinode_type); - INSERT_TAG(jl_phicnode_type); - INSERT_TAG(jl_upsilonnode_type); - INSERT_TAG(jl_type_type); - INSERT_TAG(jl_bottom_type); - INSERT_TAG(jl_ref_type); - INSERT_TAG(jl_pointer_type); - INSERT_TAG(jl_llvmpointer_type); - INSERT_TAG(jl_vararg_type); - INSERT_TAG(jl_abstractarray_type); - INSERT_TAG(jl_densearray_type); - INSERT_TAG(jl_nothing_type); - INSERT_TAG(jl_function_type); - INSERT_TAG(jl_typeofbottom_type); - INSERT_TAG(jl_unionall_type); - INSERT_TAG(jl_typename_type); - INSERT_TAG(jl_builtin_type); - INSERT_TAG(jl_code_info_type); - INSERT_TAG(jl_opaque_closure_type); - INSERT_TAG(jl_task_type); - INSERT_TAG(jl_uniontype_type); - INSERT_TAG(jl_abstractstring_type); - INSERT_TAG(jl_array_any_type); - INSERT_TAG(jl_intrinsic_type); - INSERT_TAG(jl_methtable_type); - INSERT_TAG(jl_typemap_level_type); - INSERT_TAG(jl_typemap_entry_type); - INSERT_TAG(jl_voidpointer_type); - INSERT_TAG(jl_uint8pointer_type); - INSERT_TAG(jl_newvarnode_type); - INSERT_TAG(jl_anytuple_type_type); - INSERT_TAG(jl_anytuple_type); - INSERT_TAG(jl_namedtuple_type); - INSERT_TAG(jl_emptytuple_type); - INSERT_TAG(jl_array_symbol_type); - INSERT_TAG(jl_array_uint8_type); - INSERT_TAG(jl_array_uint32_type); - INSERT_TAG(jl_array_int32_type); - INSERT_TAG(jl_array_uint64_type); - INSERT_TAG(jl_int32_type); - INSERT_TAG(jl_int64_type); - INSERT_TAG(jl_bool_type); - INSERT_TAG(jl_uint8_type); - INSERT_TAG(jl_uint16_type); - INSERT_TAG(jl_uint32_type); - INSERT_TAG(jl_uint64_type); - INSERT_TAG(jl_char_type); - INSERT_TAG(jl_weakref_type); - INSERT_TAG(jl_int8_type); - INSERT_TAG(jl_int16_type); - INSERT_TAG(jl_float16_type); - INSERT_TAG(jl_float32_type); - INSERT_TAG(jl_float64_type); - INSERT_TAG(jl_bfloat16_type); - INSERT_TAG(jl_floatingpoint_type); - INSERT_TAG(jl_number_type); - INSERT_TAG(jl_signed_type); - INSERT_TAG(jl_pair_type); - INSERT_TAG(jl_genericmemory_type); - INSERT_TAG(jl_memory_any_type); - INSERT_TAG(jl_memory_uint8_type); - INSERT_TAG(jl_memory_uint16_type); - INSERT_TAG(jl_memory_uint32_type); - INSERT_TAG(jl_memory_uint64_type); - INSERT_TAG(jl_genericmemoryref_type); - INSERT_TAG(jl_memoryref_any_type); - INSERT_TAG(jl_memoryref_uint8_type); - INSERT_TAG(jl_addrspace_type); - INSERT_TAG(jl_addrspace_typename); - INSERT_TAG(jl_addrspacecore_type); - INSERT_TAG(jl_debuginfo_type); - INSERT_TAG(jl_abioverride_type); - - // special typenames - INSERT_TAG(jl_tuple_typename); - INSERT_TAG(jl_pointer_typename); - INSERT_TAG(jl_llvmpointer_typename); - INSERT_TAG(jl_array_typename); - INSERT_TAG(jl_type_typename); - INSERT_TAG(jl_namedtuple_typename); - INSERT_TAG(jl_vecelement_typename); - INSERT_TAG(jl_opaque_closure_typename); - INSERT_TAG(jl_genericmemory_typename); - INSERT_TAG(jl_genericmemoryref_typename); - - // special exceptions - INSERT_TAG(jl_errorexception_type); - INSERT_TAG(jl_argumenterror_type); - INSERT_TAG(jl_typeerror_type); - INSERT_TAG(jl_methoderror_type); - INSERT_TAG(jl_loaderror_type); - INSERT_TAG(jl_initerror_type); - INSERT_TAG(jl_undefvarerror_type); - INSERT_TAG(jl_fielderror_type); - INSERT_TAG(jl_stackovf_exception); - INSERT_TAG(jl_diverror_exception); - INSERT_TAG(jl_interrupt_exception); - INSERT_TAG(jl_boundserror_type); - INSERT_TAG(jl_memory_exception); - INSERT_TAG(jl_undefref_exception); - INSERT_TAG(jl_readonlymemory_exception); - INSERT_TAG(jl_atomicerror_type); - INSERT_TAG(jl_missingcodeerror_type); - INSERT_TAG(jl_precompilable_error); - INSERT_TAG(jl_trimfailure_type); - - // other special values - INSERT_TAG(jl_emptysvec); - INSERT_TAG(jl_emptytuple); - INSERT_TAG(jl_false); - INSERT_TAG(jl_true); - INSERT_TAG(jl_an_empty_string); - INSERT_TAG(jl_an_empty_vec_any); - INSERT_TAG(jl_an_empty_memory_any); - INSERT_TAG(jl_module_init_order); - INSERT_TAG(jl_core_module); - INSERT_TAG(jl_base_module); - INSERT_TAG(jl_main_module); - INSERT_TAG(jl_top_module); - INSERT_TAG(jl_typeinf_func); - INSERT_TAG(jl_type_type_mt); - INSERT_TAG(jl_nonfunction_mt); - INSERT_TAG(jl_kwcall_mt); - INSERT_TAG(jl_kwcall_func); - INSERT_TAG(jl_opaque_closure_method); - INSERT_TAG(jl_nulldebuginfo); - - // some Core.Builtin Functions that we want to be able to reference: - INSERT_TAG(jl_builtin_throw); - INSERT_TAG(jl_builtin_is); - INSERT_TAG(jl_builtin_typeof); - INSERT_TAG(jl_builtin_sizeof); - INSERT_TAG(jl_builtin_issubtype); - INSERT_TAG(jl_builtin_isa); - INSERT_TAG(jl_builtin_typeassert); - INSERT_TAG(jl_builtin__apply_iterate); - INSERT_TAG(jl_builtin_isdefined); - INSERT_TAG(jl_builtin_nfields); - INSERT_TAG(jl_builtin_tuple); - INSERT_TAG(jl_builtin_svec); - INSERT_TAG(jl_builtin_getfield); - INSERT_TAG(jl_builtin_setfield); - INSERT_TAG(jl_builtin_swapfield); - INSERT_TAG(jl_builtin_modifyfield); - INSERT_TAG(jl_builtin_replacefield); - INSERT_TAG(jl_builtin_setfieldonce); - INSERT_TAG(jl_builtin_fieldtype); - INSERT_TAG(jl_builtin_memorynew); - INSERT_TAG(jl_builtin_memoryref); - INSERT_TAG(jl_builtin_memoryrefoffset); - INSERT_TAG(jl_builtin_memoryrefget); - INSERT_TAG(jl_builtin_memoryrefset); - INSERT_TAG(jl_builtin_memoryref_isassigned); - INSERT_TAG(jl_builtin_memoryrefswap); - INSERT_TAG(jl_builtin_memoryrefmodify); - INSERT_TAG(jl_builtin_memoryrefreplace); - INSERT_TAG(jl_builtin_memoryrefsetonce); - INSERT_TAG(jl_builtin_apply_type); - INSERT_TAG(jl_builtin_applicable); - INSERT_TAG(jl_builtin_invoke); - INSERT_TAG(jl_builtin__expr); - INSERT_TAG(jl_builtin_ifelse); - INSERT_TAG(jl_builtin__typebody); - INSERT_TAG(jl_builtin_donotdelete); - INSERT_TAG(jl_builtin_compilerbarrier); - INSERT_TAG(jl_builtin_getglobal); - INSERT_TAG(jl_builtin_setglobal); - INSERT_TAG(jl_builtin_isdefinedglobal); - INSERT_TAG(jl_builtin_swapglobal); - INSERT_TAG(jl_builtin_modifyglobal); - INSERT_TAG(jl_builtin_replaceglobal); - INSERT_TAG(jl_builtin_setglobalonce); - INSERT_TAG(jl_builtin_current_scope); - // n.b. must update NUM_TAGS when you add something here + unsigned int i = 0; +#define INSERT_TAG(sym) tags[i++] = (jl_value_t**)&(sym) + // builtin types + INSERT_TAG(jl_any_type); + INSERT_TAG(jl_symbol_type); + INSERT_TAG(jl_ssavalue_type); + INSERT_TAG(jl_datatype_type); + INSERT_TAG(jl_slotnumber_type); + INSERT_TAG(jl_simplevector_type); + INSERT_TAG(jl_array_type); + INSERT_TAG(jl_expr_type); + INSERT_TAG(jl_binding_type); + INSERT_TAG(jl_binding_partition_type); + INSERT_TAG(jl_globalref_type); + INSERT_TAG(jl_string_type); + INSERT_TAG(jl_module_type); + INSERT_TAG(jl_tvar_type); + INSERT_TAG(jl_method_instance_type); + INSERT_TAG(jl_method_type); + INSERT_TAG(jl_code_instance_type); + INSERT_TAG(jl_linenumbernode_type); + INSERT_TAG(jl_lineinfonode_type); + INSERT_TAG(jl_gotonode_type); + INSERT_TAG(jl_quotenode_type); + INSERT_TAG(jl_gotoifnot_type); + INSERT_TAG(jl_enternode_type); + INSERT_TAG(jl_argument_type); + INSERT_TAG(jl_returnnode_type); + INSERT_TAG(jl_const_type); + INSERT_TAG(jl_partial_struct_type); + INSERT_TAG(jl_partial_opaque_type); + INSERT_TAG(jl_interconditional_type); + INSERT_TAG(jl_method_match_type); + INSERT_TAG(jl_pinode_type); + INSERT_TAG(jl_phinode_type); + INSERT_TAG(jl_phicnode_type); + INSERT_TAG(jl_upsilonnode_type); + INSERT_TAG(jl_type_type); + INSERT_TAG(jl_bottom_type); + INSERT_TAG(jl_ref_type); + INSERT_TAG(jl_pointer_type); + INSERT_TAG(jl_llvmpointer_type); + INSERT_TAG(jl_vararg_type); + INSERT_TAG(jl_abstractarray_type); + INSERT_TAG(jl_densearray_type); + INSERT_TAG(jl_nothing_type); + INSERT_TAG(jl_function_type); + INSERT_TAG(jl_typeofbottom_type); + INSERT_TAG(jl_unionall_type); + INSERT_TAG(jl_typename_type); + INSERT_TAG(jl_builtin_type); + INSERT_TAG(jl_code_info_type); + INSERT_TAG(jl_opaque_closure_type); + INSERT_TAG(jl_task_type); + INSERT_TAG(jl_uniontype_type); + INSERT_TAG(jl_abstractstring_type); + INSERT_TAG(jl_array_any_type); + INSERT_TAG(jl_intrinsic_type); + INSERT_TAG(jl_methtable_type); + INSERT_TAG(jl_methcache_type); + INSERT_TAG(jl_typemap_level_type); + INSERT_TAG(jl_typemap_entry_type); + INSERT_TAG(jl_voidpointer_type); + INSERT_TAG(jl_uint8pointer_type); + INSERT_TAG(jl_newvarnode_type); + INSERT_TAG(jl_anytuple_type_type); + INSERT_TAG(jl_anytuple_type); + INSERT_TAG(jl_namedtuple_type); + INSERT_TAG(jl_emptytuple_type); + INSERT_TAG(jl_array_symbol_type); + INSERT_TAG(jl_array_uint8_type); + INSERT_TAG(jl_array_uint32_type); + INSERT_TAG(jl_array_int32_type); + INSERT_TAG(jl_array_uint64_type); + INSERT_TAG(jl_int32_type); + INSERT_TAG(jl_int64_type); + INSERT_TAG(jl_bool_type); + INSERT_TAG(jl_uint8_type); + INSERT_TAG(jl_uint16_type); + INSERT_TAG(jl_uint32_type); + INSERT_TAG(jl_uint64_type); + INSERT_TAG(jl_char_type); + INSERT_TAG(jl_weakref_type); + INSERT_TAG(jl_int8_type); + INSERT_TAG(jl_int16_type); + INSERT_TAG(jl_float16_type); + INSERT_TAG(jl_float32_type); + INSERT_TAG(jl_float64_type); + INSERT_TAG(jl_bfloat16_type); + INSERT_TAG(jl_floatingpoint_type); + INSERT_TAG(jl_number_type); + INSERT_TAG(jl_signed_type); + INSERT_TAG(jl_pair_type); + INSERT_TAG(jl_genericmemory_type); + INSERT_TAG(jl_memory_any_type); + INSERT_TAG(jl_memory_uint8_type); + INSERT_TAG(jl_memory_uint16_type); + INSERT_TAG(jl_memory_uint32_type); + INSERT_TAG(jl_memory_uint64_type); + INSERT_TAG(jl_genericmemoryref_type); + INSERT_TAG(jl_memoryref_any_type); + INSERT_TAG(jl_memoryref_uint8_type); + INSERT_TAG(jl_addrspace_type); + INSERT_TAG(jl_addrspace_typename); + INSERT_TAG(jl_addrspacecore_type); + INSERT_TAG(jl_debuginfo_type); + INSERT_TAG(jl_abioverride_type); + INSERT_TAG(jl_kwcall_type); + + // special typenames + INSERT_TAG(jl_tuple_typename); + INSERT_TAG(jl_pointer_typename); + INSERT_TAG(jl_llvmpointer_typename); + INSERT_TAG(jl_array_typename); + INSERT_TAG(jl_type_typename); + INSERT_TAG(jl_namedtuple_typename); + INSERT_TAG(jl_vecelement_typename); + INSERT_TAG(jl_opaque_closure_typename); + INSERT_TAG(jl_genericmemory_typename); + INSERT_TAG(jl_genericmemoryref_typename); + + // special exceptions + INSERT_TAG(jl_errorexception_type); + INSERT_TAG(jl_argumenterror_type); + INSERT_TAG(jl_typeerror_type); + INSERT_TAG(jl_methoderror_type); + INSERT_TAG(jl_loaderror_type); + INSERT_TAG(jl_initerror_type); + INSERT_TAG(jl_undefvarerror_type); + INSERT_TAG(jl_fielderror_type); + INSERT_TAG(jl_stackovf_exception); + INSERT_TAG(jl_diverror_exception); + INSERT_TAG(jl_interrupt_exception); + INSERT_TAG(jl_boundserror_type); + INSERT_TAG(jl_memory_exception); + INSERT_TAG(jl_undefref_exception); + INSERT_TAG(jl_readonlymemory_exception); + INSERT_TAG(jl_atomicerror_type); + INSERT_TAG(jl_missingcodeerror_type); + INSERT_TAG(jl_precompilable_error); + INSERT_TAG(jl_trimfailure_type); + + // other special values + INSERT_TAG(jl_emptysvec); + INSERT_TAG(jl_emptytuple); + INSERT_TAG(jl_false); + INSERT_TAG(jl_true); + INSERT_TAG(jl_an_empty_string); + INSERT_TAG(jl_an_empty_vec_any); + INSERT_TAG(jl_an_empty_memory_any); + INSERT_TAG(jl_module_init_order); + INSERT_TAG(jl_core_module); + INSERT_TAG(jl_base_module); + INSERT_TAG(jl_main_module); + INSERT_TAG(jl_top_module); + INSERT_TAG(jl_typeinf_func); + INSERT_TAG(jl_opaque_closure_method); + INSERT_TAG(jl_nulldebuginfo); + INSERT_TAG(jl_method_table); + // n.b. must update NUM_TAGS when you add something here #undef INSERT_TAG - assert(i == NUM_TAGS - 1); - } - return (jl_value_t**const*const) _tags; + assert(i == NUM_TAGS - 1); + tags[i] = NULL; } // hash of definitions for predefined tagged object @@ -357,12 +321,12 @@ static uintptr_t img_max; // HT_NOTFOUND is a valid integer ID, so we store the integer ids mangled. // This pair of functions mangles/demanges -static size_t from_seroder_entry(void *entry) +static size_t from_seroder_entry(void *entry) JL_NOTSAFEPOINT { return (size_t)((char*)entry - (char*)HT_NOTFOUND - 1); } -static void *to_seroder_entry(size_t idx) +static void *to_seroder_entry(size_t idx) JL_NOTSAFEPOINT { return (void*)((char*)HT_NOTFOUND + 1 + idx); } @@ -370,7 +334,7 @@ static void *to_seroder_entry(size_t idx) static htable_t new_methtables; static size_t precompilation_world; -static int ptr_cmp(const void *l, const void *r) +static int ptr_cmp(const void *l, const void *r) JL_NOTSAFEPOINT { uintptr_t left = *(const uintptr_t*)l; uintptr_t right = *(const uintptr_t*)r; @@ -378,7 +342,7 @@ static int ptr_cmp(const void *l, const void *r) } // Build an eytzinger tree from a sorted array -static int eytzinger(uintptr_t *src, uintptr_t *dest, size_t i, size_t k, size_t n) +static int eytzinger(uintptr_t *src, uintptr_t *dest, size_t i, size_t k, size_t n) JL_NOTSAFEPOINT { if (k <= n) { i = eytzinger(src, dest, i, 2 * k, n); @@ -418,7 +382,7 @@ static size_t eyt_obj_idx(jl_value_t *obj) JL_NOTSAFEPOINT } //used in staticdata.c after we add an image -void rebuild_image_blob_tree(void) +void rebuild_image_blob_tree(void) JL_NOTSAFEPOINT { size_t inc = 1 + jl_linkage_blobs.len - eytzinger_image_tree.len; assert(eytzinger_idxs.len == eytzinger_image_tree.len); @@ -494,33 +458,15 @@ JL_DLLEXPORT jl_value_t *jl_object_top_module(jl_value_t* v) JL_NOTSAFEPOINT } // hash of definitions for predefined function pointers +// (reverse is jl_builtin_f_addrs) static htable_t fptr_to_id; + void *native_functions; // opaque jl_native_code_desc_t blob used for fetching data from LLVM // table of struct field addresses to rewrite during saving static htable_t field_replace; static htable_t bits_replace; -// array of definitions for the predefined function pointers -// (reverse of fptr_to_id) -// This is a manually constructed dual of the fvars array, which would be produced by codegen for Julia code, for C. -static const jl_fptr_args_t id_to_fptrs[] = { - &jl_f_throw, &jl_f_throw_methoderror, &jl_f_is, &jl_f_typeof, &jl_f_issubtype, &jl_f_isa, - &jl_f_typeassert, &jl_f__apply_iterate, - &jl_f_invokelatest, &jl_f_invoke_in_world, &jl_f__call_in_world_total, &jl_f_isdefined, &jl_f_isdefinedglobal, - &jl_f_tuple, &jl_f_svec, &jl_f_intrinsic_call, - &jl_f_getfield, &jl_f_setfield, &jl_f_swapfield, &jl_f_modifyfield, &jl_f_setfieldonce, - &jl_f_replacefield, &jl_f_fieldtype, &jl_f_nfields, &jl_f_apply_type, &jl_f_memorynew, - &jl_f_memoryref, &jl_f_memoryrefoffset, &jl_f_memoryrefget, &jl_f_memoryref_isassigned, - &jl_f_memoryrefset, &jl_f_memoryrefswap, &jl_f_memoryrefmodify, &jl_f_memoryrefreplace, &jl_f_memoryrefsetonce, - &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar, - &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype, - &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef, &jl_f__defaultctors, - &jl_f_opaque_closure_call, &jl_f_donotdelete, &jl_f_compilerbarrier, &jl_f_get_binding_type, - &jl_f_getglobal, &jl_f_setglobal, &jl_f_swapglobal, &jl_f_modifyglobal, &jl_f_replaceglobal, &jl_f_setglobalonce, - &jl_f_finalizer, &jl_f__compute_sparams, &jl_f__svec_ref, - &jl_f_current_scope, - NULL }; typedef struct { ios_t *s; // the main stream @@ -552,6 +498,7 @@ typedef struct { jl_array_t *method_roots_list; htable_t method_roots_index; uint64_t worklist_key; + jl_query_cache *query_cache; jl_ptls_t ptls; jl_image_t *image; int8_t incremental; @@ -559,7 +506,6 @@ typedef struct { static jl_value_t *jl_bigint_type = NULL; static int gmp_limb_size = 0; -static jl_sym_t *jl_docmeta_sym = NULL; #ifdef _P64 #define RELOC_TAG_OFFSET 61 @@ -675,14 +621,13 @@ static int jl_needs_serialization(jl_serializer_state *s, jl_value_t *v) JL_NOTS return 1; } - -static int caching_tag(jl_value_t *v) JL_NOTSAFEPOINT +static int caching_tag(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFEPOINT { if (jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; jl_value_t *m = mi->def.value; if (jl_is_method(m) && jl_object_in_image(m)) - return 1 + type_in_worklist(mi->specTypes); + return 1 + type_in_worklist(mi->specTypes, query_cache); } if (jl_is_binding(v)) { jl_globalref_t *gr = ((jl_binding_t*)v)->globalref; @@ -697,24 +642,24 @@ static int caching_tag(jl_value_t *v) JL_NOTSAFEPOINT if (jl_is_tuple_type(dt) ? !dt->isconcretetype : dt->hasfreetypevars) return 0; // aka !is_cacheable from jltypes.c if (jl_object_in_image((jl_value_t*)dt->name)) - return 1 + type_in_worklist(v); + return 1 + type_in_worklist(v, query_cache); } jl_value_t *dtv = jl_typeof(v); if (jl_is_datatype_singleton((jl_datatype_t*)dtv)) { - return 1 - type_in_worklist(dtv); // these are already recached in the datatype in the image + return 1 - type_in_worklist(dtv, query_cache); // these are already recached in the datatype in the image } return 0; } -static int needs_recaching(jl_value_t *v) JL_NOTSAFEPOINT +static int needs_recaching(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFEPOINT { - return caching_tag(v) == 2; + return caching_tag(v, query_cache) == 2; } -static int needs_uniquing(jl_value_t *v) JL_NOTSAFEPOINT +static int needs_uniquing(jl_value_t *v, jl_query_cache *query_cache) JL_NOTSAFEPOINT { assert(!jl_object_in_image(v)); - return caching_tag(v) == 1; + return caching_tag(v, query_cache) == 1; } static void record_field_change(jl_value_t **addr, jl_value_t *newval) JL_NOTSAFEPOINT @@ -773,40 +718,32 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ jl_queue_for_serialization(s, m->parent); if (!jl_options.strip_metadata) jl_queue_for_serialization(s, m->file); + jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindingkeyset)); if (jl_options.trim) { jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->bindings), 0, 1); - } else { - jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindings)); - } - jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindingkeyset)); - if (jl_options.strip_metadata || jl_options.trim) { jl_svec_t *table = jl_atomic_load_relaxed(&m->bindings); for (size_t i = 0; i < jl_svec_len(table); i++) { jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i); if ((void*)b == jl_nothing) break; - if (jl_options.strip_metadata) { - jl_sym_t *name = b->globalref->name; - if (name == jl_docmeta_sym && jl_get_binding_value(b)) - record_field_change((jl_value_t**)&b->value, jl_nothing); - } - if (jl_options.trim) { - jl_value_t *val = jl_get_binding_value(b); - // keep binding objects that are defined and ... - if (val && - // ... point to modules ... - (jl_is_module(val) || - // ... or point to __init__ methods ... - !strcmp(jl_symbol_name(b->globalref->name), "__init__") || - // ... or point to Base functions accessed by the runtime - (m == jl_base_module && (!strcmp(jl_symbol_name(b->globalref->name), "wait") || - !strcmp(jl_symbol_name(b->globalref->name), "task_done_hook"))))) { - record_field_change((jl_value_t**)&b->backedges, NULL); - jl_queue_for_serialization(s, b); - } + jl_value_t *val = jl_get_binding_value_in_world(b, jl_atomic_load_relaxed(&jl_world_counter)); + // keep binding objects that are defined in the latest world and ... + if (val && + // ... point to modules ... + (jl_is_module(val) || + // ... or point to __init__ methods ... + !strcmp(jl_symbol_name(b->globalref->name), "__init__") || + // ... or point to Base functions accessed by the runtime + (m == jl_base_module && (!strcmp(jl_symbol_name(b->globalref->name), "wait") || + !strcmp(jl_symbol_name(b->globalref->name), "task_done_hook") || + !strcmp(jl_symbol_name(b->globalref->name), "_uv_hook_close"))))) { + jl_queue_for_serialization(s, b); } } } + else { + jl_queue_for_serialization(s, jl_atomic_load_relaxed(&m->bindings)); + } for (size_t i = 0; i < module_usings_length(m); i++) { jl_queue_for_serialization(s, module_usings_getmod(m, i)); @@ -822,6 +759,15 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ } } +static int codeinst_may_be_runnable(jl_code_instance_t *ci, int incremental) { + size_t max_world = jl_atomic_load_relaxed(&ci->max_world); + if (max_world == ~(size_t)0) + return 1; + if (incremental) + return 0; + return jl_atomic_load_relaxed(&ci->min_world) <= jl_typeinf_world && jl_typeinf_world <= max_world; +} + // Anything that requires uniquing or fixing during deserialization needs to be "toplevel" // in serialization (i.e., have its own entry in `serialization_order`). Consequently, // objects that act as containers for other potentially-"problematic" objects must add such "children" @@ -843,7 +789,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ jl_datatype_t *dt = (jl_datatype_t*)v; // ensure all type parameters are recached jl_queue_for_serialization_(s, (jl_value_t*)dt->parameters, 1, 1); - if (jl_is_datatype_singleton(dt) && needs_uniquing(dt->instance)) { + if (jl_is_datatype_singleton(dt) && needs_uniquing(dt->instance, s->query_cache)) { assert(jl_needs_serialization(s, dt->instance)); // should be true, since we visited dt // do not visit dt->instance for our template object as it leads to unwanted cycles here // (it may get serialized from elsewhere though) @@ -851,40 +797,73 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } goto done_fields; // for now } - if (s->incremental && jl_is_method_instance(v)) { + if (jl_is_method_instance(v)) { jl_method_instance_t *mi = (jl_method_instance_t*)v; - jl_value_t *def = mi->def.value; - if (needs_uniquing(v)) { - // we only need 3 specific fields of this (the rest are not used) - jl_queue_for_serialization(s, mi->def.value); - jl_queue_for_serialization(s, mi->specTypes); - jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals); - goto done_fields; - } - else if (jl_is_method(def) && jl_object_in_image(def)) { - // we only need 3 specific fields of this (the rest are restored afterward, if valid) - // in particular, cache is repopulated by jl_mi_cache_insert for all foreign function, - // so must not be present here + if (s->incremental) { + jl_value_t *def = mi->def.value; + if (needs_uniquing(v, s->query_cache)) { + // we only need 3 specific fields of this (the rest are not used) + jl_queue_for_serialization(s, mi->def.value); + jl_queue_for_serialization(s, mi->specTypes); + jl_queue_for_serialization(s, (jl_value_t*)mi->sparam_vals); + goto done_fields; + } + else if (jl_is_method(def) && jl_object_in_image(def)) { + // we only need 3 specific fields of this (the rest are restored afterward, if valid) + // in particular, cache is repopulated by jl_mi_cache_insert for all foreign function, + // so must not be present here + record_field_change((jl_value_t**)&mi->cache, NULL); + } + else { + assert(!needs_recaching(v, s->query_cache)); + } + // Any back-edges will be re-validated and added by staticdata.jl, so + // drop them from the image here record_field_change((jl_value_t**)&mi->backedges, NULL); - record_field_change((jl_value_t**)&mi->cache, NULL); + // n.b. opaque closures cannot be inspected and relied upon like a + // normal method since they can get improperly introduced by generated + // functions, so if they appeared at all, we will probably serialize + // them wrong and segfault. The jl_code_for_staged function should + // prevent this from happening, so we do not need to detect that user + // error now. } - else { - assert(!needs_recaching(v)); + // don't recurse into all backedges memory (yet) + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1); + if (backedges) { + assert(!jl_options.trim && !jl_options.strip_ir); + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + size_t i = 0, n = jl_array_nrows(backedges); + while (i < n) { + jl_value_t *invokeTypes; + jl_code_instance_t *caller; + i = get_next_edge((jl_array_t*)backedges, i, &invokeTypes, &caller); + if (invokeTypes) + jl_queue_for_serialization(s, invokeTypes); + } } - // n.b. opaque closures cannot be inspected and relied upon like a - // normal method since they can get improperly introduced by generated - // functions, so if they appeared at all, we will probably serialize - // them wrong and segfault. The jl_code_for_staged function should - // prevent this from happening, so we do not need to detect that user - // error now. - } - if (s->incremental && jl_is_binding(v)) { - if (needs_uniquing(v)) { - jl_binding_t *b = (jl_binding_t*)v; + } + if (jl_is_binding(v)) { + jl_binding_t *b = (jl_binding_t*)v; + if (s->incremental && needs_uniquing(v, s->query_cache)) { jl_queue_for_serialization(s, b->globalref->mod); jl_queue_for_serialization(s, b->globalref->name); goto done_fields; } + if (jl_options.trim || jl_options.strip_ir) { + record_field_change((jl_value_t**)&b->backedges, NULL); + } + else { + // don't recurse into all backedges memory (yet) + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); + if (backedges) { + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i++) { + jl_value_t *b = jl_array_ptr_ref(backedges, i); + if (!jl_is_code_instance(b) && !jl_is_method_instance(b) && !jl_is_method(b)) // otherwise usually a Binding? + jl_queue_for_serialization(s, b); + } + } + } } if (s->incremental && jl_is_globalref(v)) { jl_globalref_t *gr = (jl_globalref_t*)v; @@ -902,46 +881,81 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ assert(!jl_object_in_image((jl_value_t*)tn->wrapper)); } } - if (s->incremental && jl_is_code_instance(v)) { + if (jl_is_mtable(v)) { + jl_methtable_t *mt = (jl_methtable_t*)v; + // Any back-edges will be re-validated and added by staticdata.jl, so + // drop them from the image here + if (s->incremental || jl_options.trim || jl_options.strip_ir) { + record_field_change((jl_value_t**)&mt->backedges, jl_an_empty_memory_any); + } + else { + // don't recurse into all backedges memory (yet) + jl_value_t *allbackedges = get_replaceable_field((jl_value_t**)&mt->backedges, 1); + jl_queue_for_serialization_(s, allbackedges, 0, 1); + for (size_t i = 0, n = ((jl_genericmemory_t*)allbackedges)->length; i < n; i += 2) { + jl_value_t *tn = jl_genericmemory_ptr_ref(allbackedges, i); + jl_queue_for_serialization(s, tn); + jl_value_t *backedges = jl_genericmemory_ptr_ref(allbackedges, i + 1); + if (backedges && backedges != jl_nothing) { + jl_queue_for_serialization_(s, (jl_value_t*)((jl_array_t*)backedges)->ref.mem, 0, 1); + jl_queue_for_serialization(s, backedges); + for (size_t i = 0, n = jl_array_nrows(backedges); i < n; i += 2) { + jl_value_t *t = jl_array_ptr_ref(backedges, i); + assert(!jl_is_code_instance(t)); + jl_queue_for_serialization(s, t); + } + } + } + } + } + if (jl_is_code_instance(v)) { jl_code_instance_t *ci = (jl_code_instance_t*)v; jl_method_instance_t *mi = jl_get_ci_mi(ci); - // make sure we don't serialize other reachable cache entries of foreign methods - // Should this now be: - // if (ci !in ci->defs->cache) - // record_field_change((jl_value_t**)&ci->next, NULL); - // Why are we checking that the method/module this originates from is in_image? - // and then disconnect this CI? - if (jl_object_in_image((jl_value_t*)mi->def.value)) { - // TODO: if (ci in ci->defs->cache) - record_field_change((jl_value_t**)&ci->next, NULL); + if (s->incremental) { + // make sure we don't serialize other reachable cache entries of foreign methods + // Should this now be: + // if (ci !in ci->defs->cache) + // record_field_change((jl_value_t**)&ci->next, NULL); + // Why are we checking that the method/module this originates from is in_image? + // and then disconnect this CI? + if (jl_object_in_image((jl_value_t*)mi->def.value)) { + // TODO: if (ci in ci->defs->cache) + record_field_change((jl_value_t**)&ci->next, NULL); + } } jl_value_t *inferred = jl_atomic_load_relaxed(&ci->inferred); - if (inferred && inferred != jl_nothing) { // disregard if there is nothing here to delete (e.g. builtins, unspecialized) + if (inferred && inferred != jl_nothing && !jl_is_uint8(inferred)) { // disregard if there is nothing here to delete (e.g. builtins, unspecialized) jl_method_t *def = mi->def.method; if (jl_is_method(def)) { // don't delete toplevel code - int is_relocatable = jl_is_code_info(inferred) || + int is_relocatable = !s->incremental || jl_is_code_info(inferred) || (jl_is_string(inferred) && jl_string_len(inferred) > 0 && jl_string_data(inferred)[jl_string_len(inferred) - 1]); + int discard = 0; if (!is_relocatable) { - inferred = jl_nothing; + discard = 1; } else if (def->source == NULL) { // don't delete code from optimized opaque closures that can't be reconstructed (and builtins) } - else if (jl_atomic_load_relaxed(&ci->max_world) != ~(size_t)0 || // delete all code that cannot run - jl_atomic_load_relaxed(&ci->invoke) == jl_fptr_const_return) { // delete all code that just returns a constant - inferred = jl_nothing; + else if (!codeinst_may_be_runnable(ci, s->incremental) || // delete all code that cannot run + jl_atomic_load_relaxed(&ci->invoke) == jl_fptr_const_return) { // delete all code that just returns a constant + discard = 1; } else if (native_functions && // don't delete any code if making a ji file (ci->owner == jl_nothing) && // don't delete code for external interpreters !effects_foldable(jl_atomic_load_relaxed(&ci->ipo_purity_bits)) && // don't delete code we may want for irinterp jl_ir_inlining_cost(inferred) == UINT16_MAX) { // don't delete inlineable code // delete the code now: if we thought it was worth keeping, it would have been converted to object code - inferred = jl_nothing; + discard = 1; } - if (inferred == jl_nothing) { - record_field_change((jl_value_t**)&ci->inferred, jl_nothing); + if (discard) { + // keep only the inlining cost, so inference can later decide if it is worth getting the source back + if (jl_is_string(inferred) || jl_is_code_info(inferred)) + inferred = jl_box_uint8(jl_encode_inlining_cost(jl_ir_inlining_cost(inferred))); + else + inferred = jl_nothing; + record_field_change((jl_value_t**)&ci->inferred, inferred); } - else if (jl_is_string(inferred)) { + else if (s->incremental && jl_is_string(inferred)) { // New roots for external methods if (jl_object_in_image((jl_value_t*)def)) { void **pfound = ptrhash_bp(&s->method_roots_index, def); @@ -1022,14 +1036,9 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } } } - else if (jl_typetagis(v, jl_module_tag << 4)) { + else if (jl_is_module(v)) { jl_queue_module_for_serialization(s, (jl_module_t*)v); } - else if (jl_is_binding_partition(v)) { - jl_binding_partition_t *bpart = (jl_binding_partition_t*)v; - jl_queue_for_serialization_(s, bpart->restriction, 1, immediate); - jl_queue_for_serialization_(s, get_replaceable_field((jl_value_t**)&bpart->next, 0), 1, immediate); - } else if (layout->nfields > 0) { if (jl_options.trim) { if (jl_is_method(v)) { @@ -1037,22 +1046,39 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ if (jl_is_svec(jl_atomic_load_relaxed(&m->specializations))) jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->specializations), 0, 1); } - else if (jl_typetagis(v, jl_typename_type)) { - jl_typename_t *tn = (jl_typename_t*)v; - if (tn->mt != NULL && !tn->mt->frozen) { - jl_methtable_t * new_methtable = (jl_methtable_t *)ptrhash_get(&new_methtables, tn->mt); - if (new_methtable != HT_NOTFOUND) - record_field_change((jl_value_t **)&tn->mt, (jl_value_t*)new_methtable); - else - record_field_change((jl_value_t **)&tn->mt, NULL); + else if (jl_is_mtable(v)) { + jl_methtable_t *mt = (jl_methtable_t*)v; + jl_methtable_t *newmt = (jl_methtable_t*)ptrhash_get(&new_methtables, mt); + if (newmt != HT_NOTFOUND) + record_field_change((jl_value_t **)&mt->defs, (jl_value_t*)jl_atomic_load_relaxed(&newmt->defs)); + else + record_field_change((jl_value_t **)&mt->defs, jl_nothing); + } + else if (jl_is_mcache(v)) { + jl_methcache_t *mc = (jl_methcache_t*)v; + jl_value_t *cache = jl_atomic_load_relaxed(&mc->cache); + if (!jl_typetagis(cache, jl_typemap_entry_type) || ((jl_typemap_entry_t*)cache)->sig != jl_tuple_type) { // aka Builtins (maybe sometimes OpaqueClosure too) + record_field_change((jl_value_t **)&mc->cache, jl_nothing); } + record_field_change((jl_value_t **)&mc->leafcache, jl_an_empty_memory_any); } + // TODO: prune any partitions and partition data that has been deleted in the current world + //else if (jl_is_binding(v)) { + // jl_binding_t *b = (jl_binding_t*)v; + //} + //else if (jl_is_binding_partition(v)) { + // jl_binding_partition_t *bpart = (jl_binding_partition_t*)v; + //} } char *data = (char*)jl_data_ptr(v); size_t i, np = layout->npointers; + size_t fldidx = 1; for (i = 0; i < np; i++) { uint32_t ptr = jl_ptr_offset(t, i); - int mutabl = t->name->mutabl; + size_t offset = jl_ptr_offset(t, i) * sizeof(jl_value_t*); + while (offset >= (fldidx == layout->nfields ? jl_datatype_size(t) : jl_field_offset(t, fldidx))) + fldidx++; + int mutabl = !jl_field_isconst(t, fldidx - 1); jl_value_t *fld = get_replaceable_field(&((jl_value_t**)data)[ptr], mutabl); jl_queue_for_serialization_(s, fld, 1, immediate); } @@ -1099,13 +1125,26 @@ static void jl_queue_for_serialization_(jl_serializer_state *s, jl_value_t *v, i if (!jl_needs_serialization(s, v)) return; - jl_value_t *t = jl_typeof(v); + jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v); + // check early from errors, so we have a little bit of contextual state for debugging them + if (t == jl_task_type) { + jl_error("Task cannot be serialized"); + } + if (s->incremental && needs_uniquing(v, s->query_cache) && t == jl_binding_type) { + jl_binding_t *b = (jl_binding_t*)v; + if (b->globalref == NULL) + jl_error("Binding cannot be serialized"); // no way (currently) to recover its identity + } + if (jl_is_foreign_type(t) == 1) { + jl_error("Cannot serialize instances of foreign datatypes"); + } + // Items that require postorder traversal must visit their children prior to insertion into // the worklist/serialization_order (and also before their first use) if (s->incremental && !immediate) { - if (jl_is_datatype(t) && needs_uniquing(v)) + if (jl_is_datatype(t) && needs_uniquing(v, s->query_cache)) immediate = 1; - if (jl_is_datatype_singleton((jl_datatype_t*)t) && needs_uniquing(v)) + if (jl_is_datatype_singleton((jl_datatype_t*)t) && needs_uniquing(v, s->query_cache)) immediate = 1; } @@ -1182,7 +1221,8 @@ static void write_pointer(ios_t *s) JL_NOTSAFEPOINT } // Records the buildid holding `v` and returns the tagged offset within the corresponding image -static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) { +static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_GC_DISABLED +{ size_t i = external_blob_index(v); if (i < n_linkage_blobs()) { // We found the sysimg/pkg that this item links against @@ -1211,7 +1251,7 @@ static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_ // but symbols, small integers, and a couple of special items (`nothing` and the root Task) // have special handling. #define backref_id(s, v, link_ids) _backref_id(s, (jl_value_t*)(v), link_ids) -static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_NOTSAFEPOINT +static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_GC_DISABLED { assert(v != NULL && "cannot get backref to NULL object"); if (jl_is_symbol(v)) { @@ -1268,7 +1308,7 @@ static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t * static void record_uniquing(jl_serializer_state *s, jl_value_t *fld, uintptr_t offset) JL_NOTSAFEPOINT { - if (s->incremental && jl_needs_serialization(s, fld) && needs_uniquing(fld)) { + if (s->incremental && jl_needs_serialization(s, fld) && needs_uniquing(fld, s->query_cache)) { if (jl_is_datatype(fld) || jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(fld))) arraylist_push(&s->uniquing_types, (void*)(uintptr_t)offset); else if (jl_is_method_instance(fld) || jl_is_binding(fld)) @@ -1424,7 +1464,7 @@ static void record_memoryrefs_inside(jl_serializer_state *s, jl_datatype_t *t, s } } -static void record_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAFEPOINT +static void record_gvars(jl_serializer_state *s, arraylist_t *globals) JL_GC_DISABLED { for (size_t i = 0; i < globals->len; i++) jl_queue_for_serialization(s, globals->items[i]); @@ -1492,7 +1532,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED // write header if (object_id_expected) write_uint(f, jl_object_id(v)); - if (s->incremental && jl_needs_serialization(s, (jl_value_t*)t) && needs_uniquing((jl_value_t*)t)) + if (s->incremental && jl_needs_serialization(s, (jl_value_t*)t) && needs_uniquing((jl_value_t*)t, s->query_cache)) arraylist_push(&s->uniquing_types, (void*)(uintptr_t)(ios_pos(f)|1)); if (f == s->const_data) write_uint(s->const_data, ((uintptr_t)t->smalltag << 4) | GC_OLD_MARKED | GC_IN_IMAGE); @@ -1503,11 +1543,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED layout_table.items[item] = (void*)(reloc_offset | (f == s->const_data)); // store the inverse mapping of `serialization_order` (`id` => object-as-streampos) if (s->incremental) { - if (needs_uniquing(v)) { - if (jl_typetagis(v, jl_binding_type)) { + if (needs_uniquing(v, s->query_cache)) { + if (jl_is_binding(v)) { jl_binding_t *b = (jl_binding_t*)v; - if (b->globalref == NULL) - jl_error("Binding cannot be serialized"); // no way (currently) to recover its identity write_pointerfield(s, (jl_value_t*)b->globalref->mod); write_pointerfield(s, (jl_value_t*)b->globalref->name); continue; @@ -1532,7 +1570,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED assert(jl_is_datatype_singleton(t) && "unreachable"); } } - else if (needs_recaching(v)) { + else if (needs_recaching(v, s->query_cache)) { arraylist_push(jl_is_datatype(v) ? &s->fixup_types : &s->fixup_objs, (void*)reloc_offset); } } @@ -1672,7 +1710,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED jl_write_module(s, item, (jl_module_t*)v); } else if (jl_typetagis(v, jl_task_tag << 4)) { - jl_error("Task cannot be serialized"); + abort(); // unreachable } else if (jl_is_svec(v)) { assert(f == s->s); @@ -1688,7 +1726,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED write_uint8(f, '\0'); // null-terminated strings for easier C-compatibility } else if (jl_is_foreign_type(t) == 1) { - jl_error("Cannot serialize instances of foreign datatypes"); + abort(); // unreachable } else if (jl_datatype_nfields(t) == 0) { // The object has no fields, so we just snapshot its byte representation @@ -1714,32 +1752,6 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED ios_write(s->const_data, (char*)pdata, nb); write_pointer(f); } - else if (jl_is_binding_partition(v)) { - jl_binding_partition_t *bpart = (jl_binding_partition_t*)v; - write_pointerfield(s, bpart->restriction); - size_t max_world = jl_atomic_load_relaxed(&bpart->max_world); - if (s->incremental) { - if (max_world == ~(size_t)0) { - // Still valid. Will be considered to be defined in jl_require_world - // after reload, which is the first world before new code runs. - // We use this as a quick check to determine whether a binding was - // invalidated. If a binding was first defined in or before - // jl_require_world, then we can assume that all precompile processes - // will have seen it consistently. - write_uint(f, jl_require_world); - write_uint(f, max_world); - } else { - // The world will not be reachable after loading - write_uint(f, 1); - write_uint(f, 0); - } - } else { - write_uint(f, bpart->min_world); - write_uint(f, max_world); - } - write_pointerfield(s, (jl_value_t*)jl_atomic_load_relaxed(&bpart->next)); - write_uint(f, bpart->kind); - } else { // Generic object::DataType serialization by field const char *data = (const char*)v; @@ -1752,7 +1764,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED tot = offset; size_t fsz = jl_field_size(t, i); jl_value_t *replace = (jl_value_t*)ptrhash_get(&bits_replace, (void*)slot); - if (replace != HT_NOTFOUND) { + if (replace != HT_NOTFOUND && fsz > 0) { assert(t->name->mutabl && !jl_field_isptr(t, i)); jl_value_t *rty = jl_typeof(replace); size_t sz = jl_datatype_size(rty); @@ -1779,9 +1791,12 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } size_t np = t->layout->npointers; + size_t fldidx = 1; for (i = 0; i < np; i++) { size_t offset = jl_ptr_offset(t, i) * sizeof(jl_value_t*); - int mutabl = t->name->mutabl; + while (offset >= (fldidx == nf ? jl_datatype_size(t) : jl_field_offset(t, fldidx))) + fldidx++; + int mutabl = !jl_field_isconst(t, fldidx - 1); jl_value_t *fld = get_replaceable_field((jl_value_t**)&data[offset], mutabl); size_t fld_pos = offset + reloc_offset; if (fld != NULL) { @@ -1801,15 +1816,33 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED jl_typemap_entry_t *newentry = (jl_typemap_entry_t*)&s->s->buf[reloc_offset]; if (jl_atomic_load_relaxed(&newentry->max_world) == ~(size_t)0) { if (jl_atomic_load_relaxed(&newentry->min_world) > 1) { - jl_atomic_store_release(&newentry->min_world, ~(size_t)0); - jl_atomic_store_release(&newentry->max_world, WORLD_AGE_REVALIDATION_SENTINEL); + jl_atomic_store_relaxed(&newentry->min_world, ~(size_t)0); + jl_atomic_store_relaxed(&newentry->max_world, WORLD_AGE_REVALIDATION_SENTINEL); arraylist_push(&s->fixup_objs, (void*)reloc_offset); } } else { // garbage newentry - delete it :( - jl_atomic_store_release(&newentry->min_world, 1); - jl_atomic_store_release(&newentry->max_world, 0); + jl_atomic_store_relaxed(&newentry->min_world, 1); + jl_atomic_store_relaxed(&newentry->max_world, 0); + } + } + else if (s->incremental && jl_is_binding_partition(v)) { + jl_binding_partition_t *newbpart = (jl_binding_partition_t*)&s->s->buf[reloc_offset]; + size_t max_world = jl_atomic_load_relaxed(&newbpart->max_world); + if (max_world == ~(size_t)0) { + // Still valid. Will be considered to be defined in jl_require_world + // after reload, which is the first world before new code runs. + // We use this as a quick check to determine whether a binding was + // invalidated. If a binding was first defined in or before + // jl_require_world, then we can assume that all precompile processes + // will have seen it consistently. + jl_atomic_store_relaxed(&newbpart->min_world, jl_require_world); + } + else { + // The world will not be reachable after loading + jl_atomic_store_relaxed(&newbpart->min_world, 1); + jl_atomic_store_relaxed(&newbpart->max_world, 0); } } else if (jl_is_method(v)) { @@ -1818,16 +1851,14 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED jl_method_t *m = (jl_method_t*)v; jl_method_t *newm = (jl_method_t*)&f->buf[reloc_offset]; if (s->incremental) { - if (jl_atomic_load_relaxed(&newm->deleted_world) == ~(size_t)0) { - if (jl_atomic_load_relaxed(&newm->primary_world) > 1) { - jl_atomic_store_relaxed(&newm->primary_world, ~(size_t)0); // min-world - jl_atomic_store_relaxed(&newm->deleted_world, 1); // max_world - arraylist_push(&s->fixup_objs, (void*)reloc_offset); - } - } - else { - jl_atomic_store_relaxed(&newm->primary_world, 1); - jl_atomic_store_relaxed(&newm->deleted_world, 0); + if (jl_atomic_load_relaxed(&newm->primary_world) > 1) { + jl_atomic_store_relaxed(&newm->primary_world, ~(size_t)0); // min-world + int dispatch_status = jl_atomic_load_relaxed(&newm->dispatch_status); + int new_dispatch_status = 0; + if (!(dispatch_status & METHOD_SIG_LATEST_ONLY)) + new_dispatch_status |= METHOD_SIG_PRECOMPILE_MANY; + jl_atomic_store_relaxed(&newm->dispatch_status, new_dispatch_status); + arraylist_push(&s->fixup_objs, (void*)reloc_offset); } } else { @@ -1838,6 +1869,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED assert(f == s->s); jl_method_instance_t *newmi = (jl_method_instance_t*)&f->buf[reloc_offset]; jl_atomic_store_relaxed(&newmi->flags, 0); + if (s->incremental) { + jl_atomic_store_relaxed(&newmi->dispatch_status, 0); + } } else if (jl_is_code_instance(v)) { assert(f == s->s); @@ -1967,7 +2001,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } } void *superidx = ptrhash_get(&serialization_order, dt->super); - if (s->incremental && superidx != HT_NOTFOUND && from_seroder_entry(superidx) > item && needs_uniquing((jl_value_t*)dt->super)) + if (s->incremental && superidx != HT_NOTFOUND && from_seroder_entry(superidx) > item && needs_uniquing((jl_value_t*)dt->super, s->query_cache)) arraylist_push(&s->uniquing_super, dt->super); } else if (jl_is_typename(v)) { @@ -2073,7 +2107,7 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) case FunctionRef: if (offset & BuiltinFunctionTag) { offset &= ~BuiltinFunctionTag; - assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer id"); + assert(offset < jl_n_builtins && "unknown function pointer id"); } else { assert(offset < JL_API_MAX && "unknown function pointer id"); @@ -2128,8 +2162,8 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas case FunctionRef: if (offset & BuiltinFunctionTag) { offset &= ~BuiltinFunctionTag; - assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer ID"); - return (uintptr_t)id_to_fptrs[offset]; + assert(offset < jl_n_builtins && "unknown function pointer ID"); + return (uintptr_t)jl_builtin_f_addrs[offset]; } switch ((jl_callingconv_t)offset) { case JL_API_BOXED: @@ -2342,7 +2376,7 @@ void gc_sweep_sysimg(void) // the image proper. For example, new methods added to external callables require // insertion into the appropriate method table. #define jl_write_value(s, v) _jl_write_value((s), (jl_value_t*)(v)) -static void _jl_write_value(jl_serializer_state *s, jl_value_t *v) +static void _jl_write_value(jl_serializer_state *s, jl_value_t *v) JL_GC_DISABLED { if (v == NULL) { write_reloc_t(s->s, 0); @@ -2444,7 +2478,7 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) jl_register_fptrs(image->base, &fvars, linfos, img_fvars_max); } -static uint32_t write_gvars(jl_serializer_state *s, arraylist_t *globals, arraylist_t *external_fns) JL_NOTSAFEPOINT +static uint32_t write_gvars(jl_serializer_state *s, arraylist_t *globals, arraylist_t *external_fns) JL_GC_DISABLED { size_t len = globals->len + external_fns->len; ios_ensureroom(s->gvar_record, len * sizeof(reloc_t)); @@ -2560,6 +2594,60 @@ static void jl_prune_type_cache_linear(jl_svec_t *cache) jl_svecset(cache, ins++, jl_nothing); } +static void jl_prune_mi_backedges(jl_array_t *backedges) +{ + if (backedges == NULL) + return; + size_t i = 0, ins = 0, n = jl_array_nrows(backedges); + while (i < n) { + jl_value_t *invokeTypes; + jl_code_instance_t *caller; + i = get_next_edge(backedges, i, &invokeTypes, &caller); + if (ptrhash_get(&serialization_order, caller) != HT_NOTFOUND) + ins = set_next_edge(backedges, ins, invokeTypes, caller); + } + jl_array_del_end(backedges, n - ins); +} + +static void jl_prune_tn_backedges(jl_array_t *backedges) +{ + size_t i = 0, ins = 0, n = jl_array_nrows(backedges); + for (i = 1; i < n; i += 2) { + jl_value_t *ci = jl_array_ptr_ref(backedges, i); + if (ptrhash_get(&serialization_order, ci) != HT_NOTFOUND) { + jl_array_ptr_set(backedges, ins++, jl_array_ptr_ref(backedges, i - 1)); + jl_array_ptr_set(backedges, ins++, ci); + } + } + jl_array_del_end(backedges, n - ins); +} + +static void jl_prune_mt_backedges(jl_genericmemory_t *allbackedges) +{ + for (size_t i = 0, n = allbackedges->length; i < n; i += 2) { + jl_value_t *tn = jl_genericmemory_ptr_ref(allbackedges, i); + jl_value_t *backedges = jl_genericmemory_ptr_ref(allbackedges, i + 1); + if (tn && tn != jl_nothing && backedges) + jl_prune_tn_backedges((jl_array_t*)backedges); + } +} + +static void jl_prune_binding_backedges(jl_array_t *backedges) +{ + if (backedges == NULL) + return; + size_t i = 0, ins = 0, n = jl_array_nrows(backedges); + for (i = 0; i < n; i++) { + jl_value_t *b = jl_array_ptr_ref(backedges, i); + if (ptrhash_get(&serialization_order, b) != HT_NOTFOUND) { + jl_array_ptr_set(backedges, ins, b); + ins++; + } + } + jl_array_del_end(backedges, n - ins); +} + + uint_t bindingkey_hash(size_t idx, jl_value_t *data); static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED @@ -2575,15 +2663,12 @@ static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED if (ti == jl_nothing) continue; jl_binding_t *ref = ((jl_binding_t*)ti); - if (!((ptrhash_get(&serialization_order, ref) == HT_NOTFOUND) && - (ptrhash_get(&serialization_order, ref->globalref) == HT_NOTFOUND))) { - jl_svecset(bindings, i, jl_nothing); + if (ptrhash_get(&serialization_order, ref) != HT_NOTFOUND) arraylist_push(&bindings_list, ref); - } } - jl_genericmemory_t* bindingkeyset = jl_atomic_load_relaxed(&m->bindingkeyset); + jl_genericmemory_t *bindingkeyset = jl_atomic_load_relaxed(&m->bindingkeyset); _Atomic(jl_genericmemory_t*)bindingkeyset2; - jl_atomic_store_relaxed(&bindingkeyset2,(jl_genericmemory_t*)jl_an_empty_memory_any); + jl_atomic_store_relaxed(&bindingkeyset2, (jl_genericmemory_t*)jl_an_empty_memory_any); jl_svec_t *bindings2 = jl_alloc_svec_uninit(bindings_list.len); for (i = 0; i < bindings_list.len; i++) { jl_binding_t *ref = (jl_binding_t*)bindings_list.items[i]; @@ -2611,12 +2696,12 @@ static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED jl_gc_wb(m, jl_atomic_load_relaxed(&bindingkeyset2)); } -static void strip_slotnames(jl_array_t *slotnames) +static void strip_slotnames(jl_array_t *slotnames, int n) { // replace slot names with `?`, except unused_sym since the compiler looks at it jl_sym_t *questionsym = jl_symbol("?"); - int i, l = jl_array_len(slotnames); - for (i = 0; i < l; i++) { + int i; + for (i = 0; i < n; i++) { jl_value_t *s = jl_array_ptr_ref(slotnames, i); if (s != (jl_value_t*)jl_unused_sym) jl_array_ptr_set(slotnames, i, questionsym); @@ -2635,7 +2720,7 @@ static jl_value_t *strip_codeinfo_meta(jl_method_t *m, jl_value_t *ci_, jl_code_ else { ci = (jl_code_info_t*)ci_; } - strip_slotnames(ci->slotnames); + strip_slotnames(ci->slotnames, jl_array_len(ci->slotnames)); ci->debuginfo = jl_nulldebuginfo; jl_gc_wb(ci, ci->debuginfo); jl_value_t *ret = (jl_value_t*)ci; @@ -2651,7 +2736,7 @@ static void strip_specializations_(jl_method_instance_t *mi) jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); while (codeinst) { jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); - if (inferred && inferred != jl_nothing) { + if (inferred && inferred != jl_nothing && !jl_is_uint8(inferred)) { if (jl_options.strip_ir) { record_field_change((jl_value_t**)&codeinst->inferred, jl_nothing); } @@ -2668,7 +2753,7 @@ static void strip_specializations_(jl_method_instance_t *mi) record_field_change((jl_value_t**)&codeinst->debuginfo, (jl_value_t*)jl_nulldebuginfo); codeinst = jl_atomic_load_relaxed(&codeinst->next); } - if (jl_options.strip_ir) { + if (jl_options.trim || jl_options.strip_ir) { record_field_change((jl_value_t**)&mi->backedges, NULL); } } @@ -2709,7 +2794,11 @@ static int strip_all_codeinfos__(jl_typemap_entry_t *def, void *_env) } jl_array_t *slotnames = jl_uncompress_argnames(m->slot_syms); JL_GC_PUSH1(&slotnames); - strip_slotnames(slotnames); + int tostrip = jl_array_len(slotnames); + // for keyword methods, strip only nargs to keep the keyword names at the end for reflection + if (jl_tparam0(jl_unwrap_unionall(m->sig)) == (jl_value_t*)jl_kwcall_type) + tostrip = m->nargs; + strip_slotnames(slotnames, tostrip); m->slot_syms = jl_compress_argnames(slotnames); jl_gc_wb(m, m->slot_syms); JL_GC_POP(); @@ -2739,16 +2828,71 @@ static int strip_all_codeinfos__(jl_typemap_entry_t *def, void *_env) return 1; } -static int strip_all_codeinfos_(jl_methtable_t *mt, void *_env) +static int strip_all_codeinfos_mt(jl_methtable_t *mt, void *_env) { - if (jl_options.strip_ir && mt->backedges) - record_field_change((jl_value_t**)&mt->backedges, NULL); return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), strip_all_codeinfos__, NULL); } -static void jl_strip_all_codeinfos(void) +static void jl_strip_all_codeinfos(jl_array_t *mod_array) +{ + jl_foreach_reachable_mtable(strip_all_codeinfos_mt, mod_array, NULL); +} + +static int strip_module(jl_module_t *m, jl_sym_t *docmeta_sym) { - jl_foreach_reachable_mtable(strip_all_codeinfos_, NULL); + size_t world = jl_atomic_load_relaxed(&jl_world_counter); + jl_svec_t *table = jl_atomic_load_relaxed(&m->bindings); + for (size_t i = 0; i < jl_svec_len(table); i++) { + jl_binding_t *b = (jl_binding_t*)jl_svecref(table, i); + if ((void*)b == jl_nothing) + break; + jl_sym_t *name = b->globalref->name; + jl_value_t *v = jl_get_binding_value_in_world(b, world); + if (v) { + if (jl_is_module(v)) { + jl_module_t *child = (jl_module_t*)v; + if (child != m && child->parent == m && child->name == name) { + // this is the original/primary binding for the submodule + if (!strip_module(child, docmeta_sym)) + return 0; + } + } + } + if (name == docmeta_sym) { + if (jl_atomic_load_relaxed(&b->value)) + record_field_change((jl_value_t**)&b->value, jl_nothing); + // TODO: this is a pretty stupidly unsound way to do this, but it is way to late here to do this correctly (by calling delete_binding and getting an updated world age then dropping all partitions from older worlds) + jl_binding_partition_t *bp = jl_atomic_load_relaxed(&b->partitions); + while (bp) { + if (jl_bkind_is_defined_constant(jl_binding_kind(bp))) { + // XXX: bp->kind = PARTITION_KIND_UNDEF_CONST; + record_field_change((jl_value_t**)&bp->restriction, NULL); + } + bp = jl_atomic_load_relaxed(&bp->next); + } + } + } + return 1; +} + + +static void jl_strip_all_docmeta(jl_array_t *mod_array) +{ + jl_sym_t *docmeta_sym = NULL; + if (jl_base_module) { + jl_value_t *docs = jl_get_global(jl_base_module, jl_symbol("Docs")); + if (docs && jl_is_module(docs)) { + docmeta_sym = (jl_sym_t*)jl_get_global((jl_module_t*)docs, jl_symbol("META")); + } + } + if (!docmeta_sym) + return; + for (size_t i = 0; i < jl_array_nrows(mod_array); i++) { + jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); + assert(jl_is_module(m)); + if (m->parent == m) // some toplevel modules (really just Base) aren't actually + strip_module(m, docmeta_sym); + } } // --- entry points --- @@ -2883,27 +3027,20 @@ JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val, int insert) static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *newly_inferred, /* outputs */ jl_array_t **extext_methods JL_REQUIRE_ROOTED_SLOT, jl_array_t **new_ext_cis JL_REQUIRE_ROOTED_SLOT, - jl_array_t **edges JL_REQUIRE_ROOTED_SLOT) + jl_array_t **edges JL_REQUIRE_ROOTED_SLOT, + jl_query_cache *query_cache) { // extext_methods: [method1, ...], worklist-owned "extending external" methods added to functions owned by modules outside the worklist // edges: [caller1, ext_targets, ...] for worklist-owned methods calling external methods // Save the inferred code from newly inferred, external methods - *new_ext_cis = queue_external_cis(newly_inferred); + *new_ext_cis = queue_external_cis(newly_inferred, query_cache); // Collect method extensions and edges data *extext_methods = jl_alloc_vec_any(0); internal_methods = jl_alloc_vec_any(0); JL_GC_PUSH1(&internal_methods); - jl_collect_methtable_from_mod(jl_type_type_mt, *extext_methods); - jl_collect_methtable_from_mod(jl_nonfunction_mt, *extext_methods); - size_t i, len = jl_array_len(mod_array); - for (i = 0; i < len; i++) { - jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); - assert(jl_is_module(m)); - if (m->parent == m) // some toplevel modules (really just Base) aren't actually - jl_collect_extext_methods_from_mod(*extext_methods, m); - } + jl_collect_extext_methods(*extext_methods, mod_array); if (edges) { // Extract `edges` now (from info prepared by jl_collect_methcache_from_mod) @@ -2919,13 +3056,16 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new // In addition to the system image (where `worklist = NULL`), this can also save incremental images with external linkage static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_array_t *worklist, jl_array_t *extext_methods, - jl_array_t *new_ext_cis, jl_array_t *edges) + jl_array_t *new_ext_cis, jl_array_t *edges, + jl_query_cache *query_cache) { htable_new(&field_replace, 0); htable_new(&bits_replace, 0); // strip metadata and IR when requested - if (jl_options.strip_metadata || jl_options.strip_ir) - jl_strip_all_codeinfos(); + if (jl_options.strip_metadata || jl_options.strip_ir) { + jl_strip_all_codeinfos(mod_array); + jl_strip_all_docmeta(mod_array); + } // collect needed methods and replace method tables that are in the tags array htable_new(&new_methtables, 0); arraylist_t MIs; @@ -2951,7 +3091,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, if (jl_field_isptr(st, field)) { record_field_change((jl_value_t**)fldaddr, newval); } - else { + else if (jl_field_size(st, field) > 0) { // replace the bits OBJHASH_PIN(fldaddr) OBJHASH_PIN(newval) @@ -2971,9 +3111,9 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, int en = jl_gc_enable(0); if (native_functions) { size_t num_gvars, num_external_fns; - jl_get_llvm_gvs(native_functions, &num_gvars, NULL); + jl_get_llvm_gv_inits(native_functions, &num_gvars, NULL); arraylist_grow(&gvars, num_gvars); - jl_get_llvm_gvs(native_functions, &num_gvars, gvars.items); + jl_get_llvm_gv_inits(native_functions, &num_gvars, gvars.items); jl_get_llvm_external_fns(native_functions, &num_external_fns, NULL); arraylist_grow(&external_fns, num_external_fns); jl_get_llvm_external_fns(native_functions, &num_external_fns, @@ -2982,40 +3122,20 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, size_t num_mis; jl_get_llvm_mis(native_functions, &num_mis, NULL); arraylist_grow(&MIs, num_mis); - jl_get_llvm_mis(native_functions, &num_mis, (jl_method_instance_t *)MIs.items); + jl_get_llvm_mis(native_functions, &num_mis, (jl_method_instance_t*)MIs.items); } } if (jl_options.trim) { jl_rebuild_methtables(&MIs, &new_methtables); - jl_methtable_t *mt = (jl_methtable_t *)ptrhash_get(&new_methtables, jl_type_type_mt); - JL_GC_PROMISE_ROOTED(mt); - if (mt != HT_NOTFOUND) - jl_type_type_mt = mt; - else - jl_type_type_mt = jl_new_method_table(jl_type_type_mt->name, jl_type_type_mt->module); - - mt = (jl_methtable_t *)ptrhash_get(&new_methtables, jl_kwcall_mt); - JL_GC_PROMISE_ROOTED(mt); - if (mt != HT_NOTFOUND) - jl_kwcall_mt = mt; - else - jl_kwcall_mt = jl_new_method_table(jl_kwcall_mt->name, jl_kwcall_mt->module); - - mt = (jl_methtable_t *)ptrhash_get(&new_methtables, jl_nonfunction_mt); - JL_GC_PROMISE_ROOTED(mt); - if (mt != HT_NOTFOUND) - jl_nonfunction_mt = mt; - else - jl_nonfunction_mt = jl_new_method_table(jl_nonfunction_mt->name, jl_nonfunction_mt->module); } nsym_tag = 0; htable_new(&symbol_table, 0); - htable_new(&fptr_to_id, sizeof(id_to_fptrs) / sizeof(*id_to_fptrs)); + htable_new(&fptr_to_id, jl_n_builtins); uintptr_t i; - for (i = 0; id_to_fptrs[i] != NULL; i++) { - PTRHASH_PIN(id_to_fptrs[i]) - ptrhash_put(&fptr_to_id, (void*)(uintptr_t)id_to_fptrs[i], (void*)(i + 2)); + for (i = 0; i < jl_n_builtins; i++) { + PTRHASH_PIN(jl_builtin_f_addrs[i]) + ptrhash_put(&fptr_to_id, (void*)(uintptr_t)jl_builtin_f_addrs[i], (void*)(i + 2)); } htable_new(&serialization_order, 25000); htable_new(&nullptrs, 0); @@ -3029,6 +3149,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, ios_mem(&gvar_record, 0); ios_mem(&fptr_record, 0); jl_serializer_state s = {0}; + s.query_cache = query_cache; s.incremental = !(worklist == NULL); s.s = &sysimg; s.const_data = &const_data; @@ -3053,11 +3174,15 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_int32_type, 0); s.method_roots_list = NULL; htable_new(&s.method_roots_index, 0); + jl_value_t **_tags[NUM_TAGS]; + jl_value_t ***tags = s.incremental ? NULL : _tags; if (worklist) { s.method_roots_list = jl_alloc_vec_any(0); s.worklist_key = jl_worklist_key(worklist); } - jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL; + else { + get_tags(_tags); + } if (worklist == NULL) { // empty!(Core.ARGS) @@ -3073,12 +3198,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, gmp_limb_size = jl_unbox_long(jl_get_global((jl_module_t*)jl_get_global(jl_base_module, jl_symbol("GMP")), jl_symbol("BITS_PER_LIMB"))) / 8; } - if (jl_base_module) { - jl_value_t *docs = jl_get_global(jl_base_module, jl_symbol("Docs")); - if (docs && jl_is_module(docs)) { - jl_docmeta_sym = (jl_sym_t*)jl_get_global((jl_module_t*)docs, jl_symbol("META")); - } - } jl_genericmemory_t *global_roots_list = NULL; jl_genericmemory_t *global_roots_keyset = NULL; @@ -3089,6 +3208,8 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_value_t *tag = *tags[i]; jl_queue_for_serialization(&s, tag); } + for (i = 0; i < jl_n_builtins; i++) + jl_queue_for_serialization(&s, jl_builtin_instances[i]); jl_queue_for_serialization(&s, s.ptls->root_task->tls); } else { @@ -3137,16 +3258,18 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_queue_for_serialization(&s, global_roots_keyset); jl_serialize_reachable(&s); } - // step 1.5: prune (garbage collect) some special weak references from - // built-in type caches too + // step 1.5: prune (garbage collect) some special weak references known caches for (i = 0; i < serialization_queue.len; i++) { jl_value_t *v = (jl_value_t*)serialization_queue.items[i]; if (jl_options.trim) { - if (jl_is_method(v)){ + if (jl_is_method(v)) { jl_method_t *m = (jl_method_t*)v; jl_value_t *specializations_ = jl_atomic_load_relaxed(&m->specializations); - if (!jl_is_svec(specializations_)) + if (!jl_is_svec(specializations_)) { + if (ptrhash_get(&serialization_order, specializations_) == HT_NOTFOUND) + record_field_change((jl_value_t **)&m->specializations, (jl_value_t*)jl_emptysvec); continue; + } jl_svec_t *specializations = (jl_svec_t *)specializations_; size_t l = jl_svec_len(specializations), i; @@ -3170,6 +3293,21 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_gc_wb(tn, jl_atomic_load_relaxed(&tn->cache)); jl_prune_type_cache_linear(jl_atomic_load_relaxed(&tn->linearcache)); } + else if (jl_is_method_instance(v)) { + jl_method_instance_t *mi = (jl_method_instance_t*)v; + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mi->backedges, 1); + jl_prune_mi_backedges((jl_array_t*)backedges); + } + else if (jl_is_binding(v)) { + jl_binding_t *b = (jl_binding_t*)v; + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&b->backedges, 1); + jl_prune_binding_backedges((jl_array_t*)backedges); + } + else if (jl_is_mtable(v)) { + jl_methtable_t *mt = (jl_methtable_t*)v; + jl_value_t *backedges = get_replaceable_field((jl_value_t**)&mt->backedges, 1); + jl_prune_mt_backedges((jl_genericmemory_t*)backedges); + } } } @@ -3268,6 +3406,8 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_value_t *tag = *tags[i]; jl_write_value(&s, tag); } + for (i = 0; i < jl_n_builtins; i++) + jl_write_value(&s, jl_builtin_instances[i]); jl_write_value(&s, global_roots_list); jl_write_value(&s, global_roots_keyset); jl_write_value(&s, s.ptls->root_task->tls); @@ -3334,9 +3474,6 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, static void jl_write_header_for_incremental(ios_t *f, jl_array_t *worklist, jl_array_t *mod_array, jl_array_t **udeps, int64_t *srctextpos, int64_t *checksumpos) { - assert(jl_precompile_toplevel_module == NULL); - jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); - *checksumpos = write_header(f, 0); write_uint8(f, jl_cache_flags()); // write description of contents (name, uuid, buildid) @@ -3386,14 +3523,15 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli int64_t datastartpos = 0; JL_GC_PUSH4(&mod_array, &extext_methods, &new_ext_cis, &edges); + jl_query_cache query_cache; + init_query_cache(&query_cache); + + mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) if (worklist) { - mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) // Generate _native_data` if (_native_data != NULL) { - jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, NULL); - jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); + jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, NULL, &query_cache); *_native_data = jl_precompile_worklist(worklist, extext_methods, new_ext_cis); - jl_precompile_toplevel_module = NULL; extext_methods = NULL; new_ext_cis = NULL; } @@ -3412,7 +3550,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli if (jl_options.trim) *_native_data = jl_precompile_trimmed(precompilation_world); else - *_native_data = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); + *_native_data = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL, mod_array); } // Make sure we don't run any Julia code concurrently after this point @@ -3421,7 +3559,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli assert((ct->reentrant_timing & 0b1110) == 0); ct->reentrant_timing |= 0b1000; if (worklist) { - jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, &edges); + jl_prepare_serialization_data(mod_array, newly_inferred, &extext_methods, &new_ext_cis, &edges, &query_cache); if (!emit_split) { write_int32(f, 0); // No clone_targets write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f)); @@ -3433,14 +3571,13 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } if (_native_data != NULL) native_functions = *_native_data; - jl_save_system_image_to_stream(ff, mod_array, worklist, extext_methods, new_ext_cis, edges); + jl_save_system_image_to_stream(ff, mod_array, worklist, extext_methods, new_ext_cis, edges, &query_cache); if (_native_data != NULL) native_functions = NULL; // make sure we don't run any Julia code concurrently before this point // Re-enable running julia code for postoutput hooks, atexit, etc. jl_gc_enable_finalizers(ct, 1); ct->reentrant_timing &= ~0b1000u; - jl_precompile_toplevel_module = NULL; if (worklist) { // Go back and update the checksum in the header @@ -3462,6 +3599,8 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } } + destroy_query_cache(&query_cache); + JL_GC_POP(); *s = f; if (emit_split) @@ -3488,8 +3627,6 @@ JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname) jl_errorf("System image file \"%s\" not found.", fname); ios_bufmode(&f, bm_none); - JL_SIGATOMIC_BEGIN(); - ios_seek_end(&f); size_t len = ios_pos(&f); char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); @@ -3501,8 +3638,6 @@ JL_DLLEXPORT jl_image_buf_t jl_preload_sysimg(const char *fname) ios_close(&f); - JL_SIGATOMIC_END(); - jl_sysimage_buf = (jl_image_buf_t) { .kind = JL_IMAGE_KIND_JI, .pointers = NULL, @@ -3606,7 +3741,7 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t // and allocate a fresh bpart? jl_update_loaded_bpart(b, bpart); bpart->kind |= (raw_kind & PARTITION_MASK_FLAG); - if (bpart->min_world > jl_require_world) + if (jl_atomic_load_relaxed(&bpart->min_world) > jl_require_world) goto invalidated; } if (!jl_bkind_is_some_explicit_import(kind) && kind != PARTITION_KIND_IMPLICIT_GLOBAL) @@ -3617,7 +3752,8 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t jl_binding_partition_t *latest_imported_bpart = jl_atomic_load_relaxed(&imported_binding->partitions); if (!latest_imported_bpart) return 1; - if (latest_imported_bpart->min_world <= bpart->min_world) { + if (jl_atomic_load_relaxed(&latest_imported_bpart->min_world) <= + jl_atomic_load_relaxed(&bpart->min_world)) { add_backedge: // Imported binding is still valid if ((kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) && @@ -3628,13 +3764,15 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t } else { // Binding partition was invalidated - assert(bpart->min_world == jl_require_world); - bpart->min_world = latest_imported_bpart->min_world; + assert(jl_atomic_load_relaxed(&bpart->min_world) == jl_require_world); + jl_atomic_store_relaxed(&bpart->min_world, + jl_atomic_load_relaxed(&latest_imported_bpart->min_world)); } invalidated: // We need to go through and re-validate any bindings in the same image that // may have imported us. if (b->backedges) { + JL_LOCK(&b->globalref->mod->lock); for (size_t i = 0; i < jl_array_len(b->backedges); i++) { jl_value_t *edge = jl_array_ptr_ref(b->backedges, i); if (!jl_is_binding(edge)) @@ -3642,8 +3780,11 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t jl_binding_t *bedge = (jl_binding_t*)edge; if (!jl_atomic_load_relaxed(&bedge->partitions)) continue; + JL_UNLOCK(&b->globalref->mod->lock); jl_validate_binding_partition(bedge, jl_atomic_load_relaxed(&bedge->partitions), mod_idx, 0, 0); + JL_LOCK(&b->globalref->mod->lock); } + JL_UNLOCK(&b->globalref->mod->lock); } if (bpart->kind & PARTITION_FLAG_EXPORTED) { jl_module_t *mod = b->globalref->mod; @@ -3700,7 +3841,11 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, s.gvar_record = &gvar_record; s.fptr_record = &fptr_record; s.ptls = ct->ptls; - jl_value_t **const*const tags = get_tags(); + jl_value_t **_tags[NUM_TAGS]; + jl_value_t ***tags = s.incremental ? NULL : _tags; + if (!s.incremental) + get_tags(_tags); + htable_t new_dt_objs; htable_new(&new_dt_objs, 0); arraylist_new(&deser_sym, 0); @@ -3760,6 +3905,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_value_t **tag = tags[i]; *tag = jl_read_value(&s); } + for (i = 0; i < jl_n_builtins; i++) + jl_builtin_instances[i] = jl_read_value(&s); #define XX(name) \ ijl_small_typeof[(jl_##name##_tag << 4) / sizeof(*ijl_small_typeof)] = jl_##name##_type; JL_SMALL_TYPEOF(XX) @@ -4302,7 +4449,11 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im JL_SIGATOMIC_END(); // Add roots to methods - jl_copy_roots(method_roots_list, jl_worklist_key((jl_array_t*)restored)); + int failed = jl_copy_roots(method_roots_list, jl_worklist_key((jl_array_t*)restored)); + if (failed != 0) { + jl_printf(JL_STDERR, "Error copying roots to methods from Module: %s\n", pkgname); + abort(); + } // Insert method extensions and handle edges int new_methods = jl_array_nrows(extext_methods) > 0; if (!new_methods) { @@ -4434,49 +4585,6 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j return mod; } -JL_DLLEXPORT void _jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) JL_NOTSAFEPOINT -{ - if (jl_atomic_load_relaxed(&ci->max_world) != validated_world) - return; - jl_atomic_store_relaxed(&ci->max_world, ~(size_t)0); - jl_svec_t *edges = jl_atomic_load_relaxed(&ci->edges); - for (size_t i = 0; i < jl_svec_len(edges); i++) { - jl_value_t *edge = jl_svecref(edges, i); - if (!jl_is_code_instance(edge)) - continue; - _jl_promote_ci_to_current((jl_code_instance_t *)edge, validated_world); - } -} - -JL_DLLEXPORT void jl_promote_ci_to_current(jl_code_instance_t *ci, size_t validated_world) -{ - size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); - // No need to acquire the lock if we've been invalidated anyway - if (current_world > validated_world) - return; - JL_LOCK(&world_counter_lock); - current_world = jl_atomic_load_relaxed(&jl_world_counter); - if (current_world == validated_world) { - _jl_promote_ci_to_current(ci, validated_world); - } - JL_UNLOCK(&world_counter_lock); -} - -JL_DLLEXPORT void jl_promote_cis_to_current(jl_code_instance_t **cis, size_t n, size_t validated_world) -{ - size_t current_world = jl_atomic_load_relaxed(&jl_world_counter); - // No need to acquire the lock if we've been invalidated anyway - if (current_world > validated_world) - return; - JL_LOCK(&world_counter_lock); - current_world = jl_atomic_load_relaxed(&jl_world_counter); - if (current_world == validated_world) { - for (size_t i = 0; i < n; i++) { - _jl_promote_ci_to_current(cis[i], validated_world); - } - } - JL_UNLOCK(&world_counter_lock); -} #ifdef __cplusplus } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 606d998865e7c..c3d4ace606f9b 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -131,111 +131,304 @@ JL_DLLEXPORT void jl_push_newly_inferred(jl_value_t* ci) JL_UNLOCK(&newly_inferred_mutex); } - // compute whether a type references something internal to worklist // and thus could not have existed before deserialize // and thus does not need delayed unique-ing -static int type_in_worklist(jl_value_t *v) JL_NOTSAFEPOINT +static int type_in_worklist(jl_value_t *v, jl_query_cache *cache) JL_NOTSAFEPOINT { if (jl_object_in_image(v)) return 0; // fast-path for rejection + + void *cached = HT_NOTFOUND; + if (cache != NULL) + cached = ptrhash_get(&cache->type_in_worklist, v); + + // fast-path for memoized results + if (cached != HT_NOTFOUND) + return cached == v; + + int result = 0; if (jl_is_uniontype(v)) { jl_uniontype_t *u = (jl_uniontype_t*)v; - return type_in_worklist(u->a) || - type_in_worklist(u->b); + result = type_in_worklist(u->a, cache) || + type_in_worklist(u->b, cache); } else if (jl_is_unionall(v)) { jl_unionall_t *ua = (jl_unionall_t*)v; - return type_in_worklist((jl_value_t*)ua->var) || - type_in_worklist(ua->body); + result = type_in_worklist((jl_value_t*)ua->var, cache) || + type_in_worklist(ua->body, cache); } else if (jl_is_typevar(v)) { jl_tvar_t *tv = (jl_tvar_t*)v; - return type_in_worklist(tv->lb) || - type_in_worklist(tv->ub); + result = type_in_worklist(tv->lb, cache) || + type_in_worklist(tv->ub, cache); } else if (jl_is_vararg(v)) { jl_vararg_t *tv = (jl_vararg_t*)v; - if (tv->T && type_in_worklist(tv->T)) - return 1; - if (tv->N && type_in_worklist(tv->N)) - return 1; + result = ((tv->T && type_in_worklist(tv->T, cache)) || + (tv->N && type_in_worklist(tv->N, cache))); } else if (jl_is_datatype(v)) { jl_datatype_t *dt = (jl_datatype_t*)v; - if (!jl_object_in_image((jl_value_t*)dt->name)) - return 1; - jl_svec_t *tt = dt->parameters; - size_t i, l = jl_svec_len(tt); - for (i = 0; i < l; i++) - if (type_in_worklist(jl_tparam(dt, i))) - return 1; + if (!jl_object_in_image((jl_value_t*)dt->name)) { + result = 1; + } + else { + jl_svec_t *tt = dt->parameters; + size_t i, l = jl_svec_len(tt); + for (i = 0; i < l; i++) { + if (type_in_worklist(jl_tparam(dt, i), cache)) { + result = 1; + break; + } + } + } } else { - return type_in_worklist(jl_typeof(v)); + return type_in_worklist(jl_typeof(v), cache); } - return 0; + + // Memoize result + if (cache != NULL) + ptrhash_put(&cache->type_in_worklist, (void*)v, result ? (void*)v : NULL); + + return result; } +// Stack frame for iterative has_backedge_to_worklist implementation +enum backedge_state { + STATE_VISITING, // Initial visit, setup phase + STATE_PROCESSING_EDGES, // Processing backedges loop + STATE_FINISHING // Cleanup and result propagation +}; + +typedef struct { + jl_method_instance_t *mi; // Current method instance + size_t edge_index; // Current position in backedges array + size_t backedges_len; // Total backedges count + jl_array_t *backedges; // Backedges array + int depth; // Stack depth when this frame was created + int cycle; // Cycle depth tracking + int found; // Result found flag + int child_result; // Result from child recursive call + enum backedge_state state; +} backedge_stack_frame_t; + // When we infer external method instances, ensure they link back to the // package. Otherwise they might be, e.g., for external macros. // Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable -static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, arraylist_t *stack) -{ - jl_module_t *mod = mi->def.module; - if (jl_is_method(mod)) - mod = ((jl_method_t*)mod)->module; - assert(jl_is_module(mod)); - uint8_t is_precompiled = jl_atomic_load_relaxed(&mi->flags) & JL_MI_FLAGS_MASK_PRECOMPILED; - if (is_precompiled || !jl_object_in_image((jl_value_t*)mod) || type_in_worklist(mi->specTypes)) { - return 1; - } - if (!mi->backedges) { - return 0; - } - void **bp = ptrhash_bp(visited, mi); - // HT_NOTFOUND: not yet analyzed - // HT_NOTFOUND + 1: no link back - // HT_NOTFOUND + 2: does link back - // HT_NOTFOUND + 3: does link back, and included in new_ext_cis already - // HT_NOTFOUND + 4 + depth: in-progress - int found = (char*)*bp - (char*)HT_NOTFOUND; - if (found) - return found - 1; - arraylist_push(stack, (void*)mi); - int depth = stack->len; - *bp = (void*)((char*)HT_NOTFOUND + 4 + depth); // preliminarily mark as in-progress - size_t i = 0, n = jl_array_nrows(mi->backedges); - int cycle = depth; - while (i < n) { - jl_code_instance_t *be; - i = get_next_edge(mi->backedges, i, NULL, &be); - JL_GC_PROMISE_ROOTED(be); // get_next_edge propagates the edge for us here - int child_found = has_backedge_to_worklist(jl_get_ci_mi(be), visited, stack); - if (child_found == 1 || child_found == 2) { - // found what we were looking for, so terminate early - found = 1; - break; - } - else if (child_found >= 3 && child_found - 3 < cycle) { - // record the cycle will resolve at depth "cycle" - cycle = child_found - 3; - assert(cycle); +static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, arraylist_t *stack, jl_query_cache *query_cache) +{ + // Use arraylist_t for explicit stack of processing frames + arraylist_t frame_stack; + arraylist_new(&frame_stack, 0); + + // Push initial frame + backedge_stack_frame_t initial_frame = { + .mi = mi, + .edge_index = 0, + .backedges_len = 0, + .backedges = NULL, + .depth = 0, + .cycle = 0, + .found = 0, + .child_result = 0, + .state = STATE_VISITING + }; + arraylist_push(&frame_stack, memcpy(malloc(sizeof(backedge_stack_frame_t)), &initial_frame, sizeof(backedge_stack_frame_t))); + + int final_result = 0; + while (1) { + backedge_stack_frame_t *current = (backedge_stack_frame_t*)frame_stack.items[frame_stack.len - 1]; + JL_GC_PROMISE_ROOTED(current->mi); + JL_GC_PROMISE_ROOTED(current->backedges); + + switch (current->state) { + case STATE_VISITING: { + jl_module_t *mod = current->mi->def.module; + if (jl_is_method(mod)) + mod = ((jl_method_t*)mod)->module; + assert(jl_is_module(mod)); + uint8_t is_precompiled = jl_atomic_load_relaxed(¤t->mi->flags) & JL_MI_FLAGS_MASK_PRECOMPILED; + + if (is_precompiled || !jl_object_in_image((jl_value_t*)mod) || type_in_worklist(current->mi->specTypes, query_cache)) { + if (frame_stack.len > 1) { + final_result = 1; + goto propagate_to_parent; + } + current->found = 1; + // Continue to setup below, then go to finishing + } + else if (!current->mi->backedges) { + if (frame_stack.len > 1) { + final_result = 0; + goto propagate_to_parent; + } + current->found = 0; + // Setup minimal state for cleanup, skip backedges processing + arraylist_push(stack, (void*)current->mi); + current->depth = stack->len; + void **bp = ptrhash_bp(visited, current->mi); + *bp = (void*)((char*)HT_NOTFOUND + 4 + current->depth); + current->cycle = current->depth; + current->state = STATE_FINISHING; + break; + } + + void **bp = ptrhash_bp(visited, current->mi); + // HT_NOTFOUND: not yet analyzed + // HT_NOTFOUND + 1: no link back + // HT_NOTFOUND + 2: does link back + // HT_NOTFOUND + 3: does link back, and included in new_ext_cis already + // HT_NOTFOUND + 4 + depth: in-progress + int found = (char*)*bp - (char*)HT_NOTFOUND; + if (found) { + if (frame_stack.len > 1) { + final_result = found - 1; + goto propagate_to_parent; + } + current->found = found - 1; + } + + // Setup for processing + arraylist_push(stack, (void*)current->mi); + current->depth = stack->len; + *bp = (void*)((char*)HT_NOTFOUND + 4 + current->depth); // preliminarily mark as in-progress + current->backedges = jl_mi_get_backedges(current->mi); + current->backedges_len = current->backedges ? jl_array_nrows(current->backedges) : 0; + current->cycle = current->depth; + current->edge_index = 0; + // Don't reset current->found if it was already set by early termination logic above + if (current->found == 0) { + current->state = STATE_PROCESSING_EDGES; + } + else { + // Early termination case - skip processing and go straight to finishing + current->state = STATE_FINISHING; + } + break; + } + + case STATE_PROCESSING_EDGES: { + // If we have a child result to process, handle it first + if (current->child_result != 0) { + if (current->child_result == 1 || current->child_result == 2) { + // found what we were looking for, so terminate early + current->found = 1; + current->state = STATE_FINISHING; + break; + } + else if (current->child_result >= 3 && current->child_result - 3 < current->cycle) { + // record the cycle will resolve at depth "cycle" + current->cycle = current->child_result - 3; + assert(current->cycle); + } + current->child_result = 0; // Clear after processing + } + + // Process backedges iteratively + while (current->edge_index < current->backedges_len && current->backedges) { + jl_code_instance_t *be; + current->edge_index = get_next_edge(current->backedges, current->edge_index, NULL, &be); + if (!be) + continue; + JL_GC_PROMISE_ROOTED(be); // get_next_edge propagates the edge for us here + + jl_method_instance_t *child_mi = jl_get_ci_mi(be); + + // Check if we need to recurse (push new frame) or handle result + jl_module_t *child_mod = child_mi->def.module; + if (jl_is_method(child_mod)) + child_mod = ((jl_method_t*)child_mod)->module; + assert(jl_is_module(child_mod)); + uint8_t child_is_precompiled = jl_atomic_load_relaxed(&child_mi->flags) & JL_MI_FLAGS_MASK_PRECOMPILED; + + // Early termination check for child + if (child_is_precompiled || !jl_object_in_image((jl_value_t*)child_mod) || type_in_worklist(child_mi->specTypes, query_cache)) { + // found what we were looking for, so terminate early + current->found = 1; + break; + } + + if (!child_mi->backedges) { + // This child returns 0, continue with next edge + continue; + } + + void **child_bp = ptrhash_bp(visited, child_mi); + int child_found = (char*)*child_bp - (char*)HT_NOTFOUND; + if (child_found) { + int child_result = child_found - 1; + if (child_result == 1 || child_result == 2) { + // found what we were looking for, so terminate early + current->found = 1; + break; + } + else if (child_result >= 3 && child_result - 3 < current->cycle) { + // record the cycle will resolve at depth "cycle" + current->cycle = child_result - 3; + assert(current->cycle); + } + } + else { + // Need to process child - push new frame and pause current processing + backedge_stack_frame_t child_frame = { + .mi = child_mi, + .edge_index = 0, + .backedges_len = 0, + .backedges = NULL, + .depth = 0, + .cycle = 0, + .found = 0, + .child_result = 0, + .state = STATE_VISITING + }; + arraylist_push(&frame_stack, memcpy(malloc(sizeof(backedge_stack_frame_t)), &child_frame, sizeof(backedge_stack_frame_t))); + goto continue_main_loop; // Resume processing after child completes + } + } + + current->state = STATE_FINISHING; + break; + } + + case STATE_FINISHING: { + if (!current->found && current->cycle != current->depth) { + final_result = current->cycle + 3; + goto propagate_to_parent; + } + + // If we are the top of the current cycle, now mark all other parts of + // our cycle with what we found. + // Or if we found a backedge, also mark all of the other parts of the + // cycle as also having an backedge. + while (stack->len >= current->depth) { + void *mi_ptr = arraylist_pop(stack); + void **bp = ptrhash_bp(visited, mi_ptr); + assert((char*)*bp - (char*)HT_NOTFOUND == 5 + stack->len); + *bp = (void*)((char*)HT_NOTFOUND + 1 + current->found); + } + + final_result = current->found; + goto propagate_to_parent; + } } + + continue_main_loop: + continue; + + propagate_to_parent: + // Propagate result to parent + free(arraylist_pop(&frame_stack)); + if (frame_stack.len == 0) + break; + backedge_stack_frame_t *parent = (backedge_stack_frame_t*)frame_stack.items[frame_stack.len - 1]; + parent->child_result = final_result; } - if (!found && cycle != depth) - return cycle + 3; - // If we are the top of the current cycle, now mark all other parts of - // our cycle with what we found. - // Or if we found a backedge, also mark all of the other parts of the - // cycle as also having an backedge. - while (stack->len >= depth) { - void *mi = arraylist_pop(stack); - bp = ptrhash_bp(visited, mi); - assert((char*)*bp - (char*)HT_NOTFOUND == 5 + stack->len); - *bp = (void*)((char*)HT_NOTFOUND + 1 + found); - } - return found; + // Cleanup remaining frames + assert(frame_stack.len == 0); + arraylist_free(&frame_stack); + return final_result; } // Given the list of CodeInstances that were inferred during the build, select @@ -243,7 +436,7 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, // from the worklist or explicitly added by a `precompile` statement, and // (4) are the most recently computed result for that method. // These will be preserved in the image. -static jl_array_t *queue_external_cis(jl_array_t *list) +static jl_array_t *queue_external_cis(jl_array_t *list, jl_query_cache *query_cache) { if (list == NULL) return NULL; @@ -262,7 +455,7 @@ static jl_array_t *queue_external_cis(jl_array_t *list) jl_method_instance_t *mi = jl_get_ci_mi(ci); jl_method_t *m = mi->def.method; if (ci->owner == jl_nothing && jl_atomic_load_relaxed(&ci->inferred) && jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) { - int found = has_backedge_to_worklist(mi, &visited, &stack); + int found = has_backedge_to_worklist(mi, &visited, &stack, query_cache); assert(found == 0 || found == 1 || found == 2); assert(stack.len == 0); if (found == 1 && jl_atomic_load_relaxed(&ci->max_world) == ~(size_t)0) { @@ -311,9 +504,9 @@ static int jl_collect_methtable_from_mod(jl_methtable_t *mt, void *env) // Collect methods of external functions defined by modules in the worklist // "extext" = "extending external" // Also collect relevant backedges -static void jl_collect_extext_methods_from_mod(jl_array_t *s, jl_module_t *m) +static void jl_collect_extext_methods(jl_array_t *s, jl_array_t *mod_array) { - foreach_mtable_in_module(m, jl_collect_methtable_from_mod, s); + jl_foreach_reachable_mtable(jl_collect_methtable_from_mod, mod_array, s); } static void jl_record_edges(jl_method_instance_t *caller, jl_array_t *edges) @@ -511,35 +704,40 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t { int64_t initial_pos = 0; int64_t pos = 0; - static jl_array_t *deps = NULL; - if (!deps) - deps = (jl_array_t*)jl_get_global(jl_base_module, jl_symbol("_require_dependencies")); - - // unique(deps) to eliminate duplicates while preserving order: - // we preserve order so that the topmost included .jl file comes first - static jl_value_t *unique_func = NULL; - if (!unique_func) - unique_func = jl_get_global(jl_base_module, jl_symbol("unique")); - jl_value_t *uniqargs[2] = {unique_func, (jl_value_t*)deps}; jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_array_t *udeps = (*udepsp = deps && unique_func ? (jl_array_t*)jl_apply(uniqargs, 2) : NULL); - ct->world_age = last_age; + jl_value_t *depots = NULL, *prefs_hash = NULL, *prefs_list = NULL; + jl_value_t *unique_func = NULL; + jl_value_t *replace_depot_func = NULL; + jl_value_t *normalize_depots_func = NULL; + jl_value_t *toplevel = NULL; + jl_value_t *prefs_hash_func = NULL; + jl_value_t *get_compiletime_prefs_func = NULL; + JL_GC_PUSH8(&depots, &prefs_list, &unique_func, &replace_depot_func, &normalize_depots_func, &toplevel, &prefs_hash_func, &get_compiletime_prefs_func); - static jl_value_t *replace_depot_func = NULL; - if (!replace_depot_func) - replace_depot_func = jl_get_global(jl_base_module, jl_symbol("replace_depot_path")); - static jl_value_t *normalize_depots_func = NULL; - if (!normalize_depots_func) - normalize_depots_func = jl_get_global(jl_base_module, jl_symbol("normalize_depots_for_relocation")); + jl_array_t *udeps = (jl_array_t*)jl_get_global_value(jl_base_module, jl_symbol("_require_dependencies"), ct->world_age); + *udepsp = udeps; + + // unique(udeps) to eliminate duplicates while preserving order: + // we preserve order so that the topmost included .jl file comes first + if (udeps) { + unique_func = jl_eval_global_var(jl_base_module, jl_symbol("unique"), ct->world_age); + jl_value_t *uniqargs[2] = {unique_func, (jl_value_t*)udeps}; + udeps = (jl_array_t*)jl_apply(uniqargs, 2); + *udepsp = udeps; + JL_TYPECHK(write_dependency_list, array_any, (jl_value_t*)udeps); + } + + replace_depot_func = jl_get_global_value(jl_base_module, jl_symbol("replace_depot_path"), ct->world_age); + normalize_depots_func = jl_eval_global_var(jl_base_module, jl_symbol("normalize_depots_for_relocation"), ct->world_age); - jl_value_t *depots = NULL, *prefs_hash = NULL, *prefs_list = NULL; - JL_GC_PUSH2(&depots, &prefs_list); - last_age = ct->world_age; - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); depots = jl_apply(&normalize_depots_func, 1); - ct->world_age = last_age; + + jl_datatype_t *deptuple_p[5] = {jl_module_type, jl_string_type, jl_uint64_type, jl_uint32_type, jl_float64_type}; + jl_value_t *jl_deptuple_type = jl_apply_tuple_type_v((jl_value_t**)deptuple_p, 5); + JL_GC_PROMISE_ROOTED(jl_deptuple_type); +#define jl_is_deptuple(v) (jl_typeis((v), jl_deptuple_type)) // write a placeholder for total size so that we can quickly seek past all of the // dependencies if we don't need them @@ -548,20 +746,16 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t size_t i, l = udeps ? jl_array_nrows(udeps) : 0; for (i = 0; i < l; i++) { jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); - jl_value_t *deppath = jl_fieldref(deptuple, 1); + JL_TYPECHK(write_dependency_list, deptuple, deptuple); + jl_value_t *deppath = jl_fieldref_noalloc(deptuple, 1); if (replace_depot_func) { - jl_value_t **replace_depot_args; - JL_GC_PUSHARGS(replace_depot_args, 3); + jl_value_t *replace_depot_args[3]; replace_depot_args[0] = replace_depot_func; replace_depot_args[1] = deppath; replace_depot_args[2] = depots; - ct = jl_current_task; - size_t last_age = ct->world_age; - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); deppath = (jl_value_t*)jl_apply(replace_depot_args, 3); - ct->world_age = last_age; - JL_GC_POP(); + JL_TYPECHK(write_dependency_list, string, deppath); } size_t slen = jl_string_len(deppath); @@ -570,7 +764,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t write_uint64(s, jl_unbox_uint64(jl_fieldref(deptuple, 2))); // fsize write_uint32(s, jl_unbox_uint32(jl_fieldref(deptuple, 3))); // hash write_float64(s, jl_unbox_float64(jl_fieldref(deptuple, 4))); // mtime - jl_module_t *depmod = (jl_module_t*)jl_fieldref(deptuple, 0); // evaluating module + jl_module_t *depmod = (jl_module_t*)jl_fieldref_noalloc(deptuple, 0); // evaluating module jl_module_t *depmod_top = depmod; while (!is_serialization_root_module(depmod_top)) depmod_top = depmod_top->parent; @@ -594,34 +788,31 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t // Calculate Preferences hash for current package. if (jl_base_module) { // Toplevel module is the module we're currently compiling, use it to get our preferences hash - jl_value_t * toplevel = (jl_value_t*)jl_get_global(jl_base_module, jl_symbol("__toplevel__")); - jl_value_t * prefs_hash_func = jl_get_global(jl_base_module, jl_symbol("get_preferences_hash")); - jl_value_t * get_compiletime_prefs_func = jl_get_global(jl_base_module, jl_symbol("get_compiletime_preferences")); - - if (toplevel && prefs_hash_func && get_compiletime_prefs_func) { - // Temporary invoke in newest world age - size_t last_age = ct->world_age; - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); + toplevel = jl_get_global_value(jl_base_module, jl_symbol("__toplevel__"), ct->world_age); + prefs_hash_func = jl_eval_global_var(jl_base_module, jl_symbol("get_preferences_hash"), ct->world_age); + get_compiletime_prefs_func = jl_eval_global_var(jl_base_module, jl_symbol("get_compiletime_preferences"), ct->world_age); + if (toplevel) { // call get_compiletime_prefs(__toplevel__) jl_value_t *args[3] = {get_compiletime_prefs_func, (jl_value_t*)toplevel, NULL}; prefs_list = (jl_value_t*)jl_apply(args, 2); + JL_TYPECHK(write_dependency_list, array, prefs_list); // Call get_preferences_hash(__toplevel__, prefs_list) args[0] = prefs_hash_func; args[2] = prefs_list; prefs_hash = (jl_value_t*)jl_apply(args, 3); - - // Reset world age to normal - ct->world_age = last_age; + JL_TYPECHK(write_dependency_list, uint64, prefs_hash); } } + ct->world_age = last_age; // If we successfully got the preferences, write it out, otherwise write `0` for this `.ji` file. if (prefs_hash != NULL && prefs_list != NULL) { size_t i, l = jl_array_nrows(prefs_list); for (i = 0; i < l; i++) { jl_value_t *pref_name = jl_array_ptr_ref(prefs_list, i); + JL_TYPECHK(write_dependency_list, string, pref_name); size_t slen = jl_string_len(pref_name); write_int32(s, slen); ios_write(s, jl_string_data(pref_name), slen); @@ -639,6 +830,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t write_uint64(s, 0); } JL_GC_POP(); // for depots, prefs_list +#undef jl_is_deptuple // write a dummy file position to indicate the beginning of the source-text pos = ios_pos(s); @@ -685,9 +877,7 @@ static void jl_activate_methods(jl_array_t *external, jl_array_t *internal, size else if (jl_is_method(obj)) { jl_method_t *m = (jl_method_t*)obj; assert(jl_atomic_load_relaxed(&m->primary_world) == ~(size_t)0); - assert(jl_atomic_load_relaxed(&m->deleted_world) == WORLD_AGE_REVALIDATION_SENTINEL); jl_atomic_store_release(&m->primary_world, world); - jl_atomic_store_release(&m->deleted_world, ~(size_t)0); } else if (jl_is_code_instance(obj)) { jl_code_instance_t *ci = (jl_code_instance_t*)obj; @@ -708,24 +898,40 @@ static void jl_activate_methods(jl_array_t *external, jl_array_t *internal, size } for (i = 0; i < l; i++) { jl_typemap_entry_t *entry = (jl_typemap_entry_t*)jl_array_ptr_ref(external, i); - jl_methtable_t *mt = jl_method_get_table(entry->func.method); - assert((jl_value_t*)mt != jl_nothing); - jl_method_table_activate(mt, entry); + //uint64_t t0 = uv_hrtime(); + jl_method_table_activate(entry); + //jl_printf(JL_STDERR, "%f ", (double)(uv_hrtime() - t0) / 1e6); + //jl_static_show(JL_STDERR, entry->func.value); + //jl_printf(JL_STDERR, "\n"); } } } -static void jl_copy_roots(jl_array_t *method_roots_list, uint64_t key) +static int jl_copy_roots(jl_array_t *method_roots_list, uint64_t key) { size_t i, l = jl_array_nrows(method_roots_list); + int failed = 0; for (i = 0; i < l; i+=2) { jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(method_roots_list, i); jl_array_t *roots = (jl_array_t*)jl_array_ptr_ref(method_roots_list, i+1); if (roots) { assert(jl_is_array(roots)); + if (m->root_blocks) { + // check for key collision + uint64_t *blocks = jl_array_data(m->root_blocks, uint64_t); + size_t nx2 = jl_array_nrows(m->root_blocks); + for (size_t i = 0; i < nx2; i+=2) { + if (blocks[i] == key) { + // found duplicate block + failed = -1; + } + } + } + jl_append_method_roots(m, key, roots); } } + return failed; } static jl_value_t *read_verify_mod_list(ios_t *s, jl_array_t *depmods) diff --git a/src/subtype.c b/src/subtype.c index 691f86f5384f3..d2331b289818c 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -65,9 +65,9 @@ typedef struct { // Most of the complexity is due to the "diagonal rule", requiring us to // identify which type vars range over only concrete types. typedef struct jl_varbinding_t { - jl_tvar_t *var; - jl_value_t *lb; - jl_value_t *ub; + jl_tvar_t *var; // store NULL to "delete" this from env (temporarily) + jl_value_t *JL_NONNULL lb; + jl_value_t *JL_NONNULL ub; int8_t right; // whether this variable came from the right side of `A <: B` int8_t occurs_inv; // occurs in invariant position int8_t occurs_cov; // # of occurrences in covariant position @@ -139,7 +139,7 @@ static jl_varbinding_t *lookup(jl_stenv_t *e, jl_tvar_t *v) JL_GLOBALLY_ROOTED J static int statestack_get(jl_unionstate_t *st, int i) JL_NOTSAFEPOINT { - assert(i >= 0 && i <= 32767); // limited by the depth bit. + assert(i >= 0 && i < 32767); // limited by the depth bit. // get the `i`th bit in an array of 32-bit words jl_bits_stack_t *stack = &st->stack; while (i >= sizeof(stack->data) * 8) { @@ -153,7 +153,7 @@ static int statestack_get(jl_unionstate_t *st, int i) JL_NOTSAFEPOINT static void statestack_set(jl_unionstate_t *st, int i, int val) JL_NOTSAFEPOINT { - assert(i >= 0 && i <= 32767); // limited by the depth bit. + assert(i >= 0 && i < 32767); // limited by the depth bit. jl_bits_stack_t *stack = &st->stack; while (i >= sizeof(stack->data) * 8) { if (__unlikely(stack->next == NULL)) { @@ -356,7 +356,7 @@ static void free_stenv(jl_stenv_t *e) JL_NOTSAFEPOINT static void restore_env(jl_stenv_t *e, jl_savedenv_t *se, int root) JL_NOTSAFEPOINT { - jl_value_t **roots = NULL; + jl_value_t *JL_NONNULL *roots = NULL; int nroots = 0; if (root) { if (se->gcframe.nroots == JL_GC_ENCODE_PUSHARGS(1)) { @@ -1069,7 +1069,8 @@ static int forall_exists_equal(jl_value_t *x, jl_value_t *y, jl_stenv_t *e); static int subtype_tuple_varargs( jl_vararg_t *vtx, jl_vararg_t *vty, - size_t vx, size_t vy, + jl_value_t *lastx, jl_value_t *lasty, + size_t vx, size_t vy, size_t x_reps, jl_stenv_t *e, int param) { jl_value_t *xp0 = jl_unwrap_vararg(vtx); jl_value_t *xp1 = jl_unwrap_vararg_num(vtx); @@ -1111,12 +1112,30 @@ static int subtype_tuple_varargs( } } } - - // in Vararg{T1} <: Vararg{T2}, need to check subtype twice to - // simulate the possibility of multiple arguments, which is needed - // to implement the diagonal rule correctly. - if (!subtype(xp0, yp0, e, param)) return 0; - if (!subtype(xp0, yp0, e, 1)) return 0; + int x_same = vx > 1 || (lastx && obviously_egal(xp0, lastx)); + int y_same = vy > 1 || (lasty && obviously_egal(yp0, lasty)); + // keep track of number of consecutive identical subtyping + x_reps = y_same && x_same ? x_reps + 1 : 1; + if (x_reps > 2) { + // an identical type on the left doesn't need to be compared to the same + // element type on the right more than twice. + } + else if (x_same && e->Runions.depth == 0 && y_same && + !jl_has_free_typevars(xp0) && !jl_has_free_typevars(yp0)) { + // fast path for repeated elements + } + else if ((e->Runions.depth == 0 ? !jl_has_free_typevars(xp0) : jl_is_concrete_type(xp0)) && !jl_has_free_typevars(yp0)) { + // fast path for separable sub-formulas + if (!jl_subtype(xp0, yp0)) + return 0; + } + else { + // in Vararg{T1} <: Vararg{T2}, need to check subtype twice to + // simulate the possibility of multiple arguments, which is needed + // to implement the diagonal rule correctly. + if (!subtype(xp0, yp0, e, param)) return 0; + if (x_reps < 2 && !subtype(xp0, yp0, e, 1)) return 0; + } constrain_length: if (!yp1) { @@ -1163,12 +1182,14 @@ static int subtype_tuple_varargs( if (bxp1) { if (bxp1->intvalued == 0) bxp1->intvalued = 1; + assert(bxp1->lb); // make static analyzer happy if (jl_is_long(bxp1->lb)) xp1 = bxp1->lb; } if (byp1) { if (byp1->intvalued == 0) byp1->intvalued = 1; + assert(byp1->lb); // make static analyzer happy if (jl_is_long(byp1->lb)) yp1 = byp1->lb; } @@ -1246,7 +1267,8 @@ static int subtype_tuple_tail(jl_datatype_t *xd, jl_datatype_t *yd, int8_t R, jl return subtype_tuple_varargs( (jl_vararg_t*)xi, (jl_vararg_t*)yi, - vx, vy, e, param); + lastx, lasty, + vx, vy, x_reps, e, param); } if (j >= ly) @@ -1267,7 +1289,7 @@ static int subtype_tuple_tail(jl_datatype_t *xd, jl_datatype_t *yd, int8_t R, jl (yi == lastx && !vx && vy && jl_is_concrete_type(xi)))) { // fast path for repeated elements } - else if (e->Runions.depth == 0 && !jl_has_free_typevars(xi) && !jl_has_free_typevars(yi)) { + else if ((e->Runions.depth == 0 ? !jl_has_free_typevars(xi) : jl_is_concrete_type(xi)) && !jl_has_free_typevars(yi)) { // fast path for separable sub-formulas if (!jl_subtype(xi, yi)) return 0; @@ -1393,7 +1415,7 @@ static int subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param) x = pick_union_element(x, e, 0); } if (jl_is_uniontype(y)) { - if (x == ((jl_uniontype_t*)y)->a || x == ((jl_uniontype_t*)y)->b) + if (obviously_in_union(y, x)) return 1; if (jl_is_unionall(x)) return subtype_unionall(y, (jl_unionall_t*)x, e, 0, param); @@ -1448,11 +1470,14 @@ static int subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param) } if (jl_is_unionall(y)) { jl_varbinding_t *xb = lookup(e, (jl_tvar_t*)x); - if (xb == NULL ? !e->ignore_free : !xb->right) { + jl_value_t *xub = xb == NULL ? ((jl_tvar_t *)x)->ub : xb->ub; + if ((xb == NULL ? !e->ignore_free : !xb->right) && xub != y) { // We'd better unwrap `y::UnionAll` eagerly if `x` isa ∀-var. // This makes sure the following cases work correct: // 1) `∀T <: Union{∃S, SomeType{P}} where {P}`: `S == Any` ==> `S >: T` // 2) `∀T <: Union{∀T, SomeType{P}} where {P}`: + // note: if xub == y we'd better try `subtype_var` as `subtype_left_var` + // hit `==` based fast path. return subtype_unionall(x, (jl_unionall_t*)y, e, 1, param); } } @@ -1590,6 +1615,8 @@ static int has_exists_typevar(jl_value_t *x, jl_stenv_t *e) JL_NOTSAFEPOINT return env != NULL && jl_has_bound_typevars(x, env); } +static int forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param); + static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param, int limit_slow) { int16_t oldRmore = e->Runions.more; @@ -1603,7 +1630,18 @@ static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t return jl_subtype(x, y); int has_exists = (!kindx && has_exists_typevar(x, e)) || (!kindy && has_exists_typevar(y, e)); - if (has_exists && (is_exists_typevar(x, e) != is_exists_typevar(y, e))) { + if (!has_exists) { + // We can use ∀_∃_subtype safely for ∃ free inputs. + // This helps to save some bits in union stack. + jl_saved_unionstate_t oldRunions; push_unionstate(&oldRunions, &e->Runions); + e->Lunions.used = e->Runions.used = 0; + e->Lunions.depth = e->Runions.depth = 0; + e->Lunions.more = e->Runions.more = 0; + sub = forall_exists_subtype(x, y, e, param); + pop_unionstate(&e->Runions, &oldRunions); + return sub; + } + if (is_exists_typevar(x, e) != is_exists_typevar(y, e)) { e->Lunions.used = 0; while (1) { e->Lunions.more = 0; @@ -1617,7 +1655,7 @@ static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t if (limit_slow == -1) limit_slow = kindx || kindy; jl_savedenv_t se; - save_env(e, &se, has_exists); + save_env(e, &se, 1); int count, limited = 0, ini_count = 0; jl_saved_unionstate_t latestLunions = {0, 0, 0, NULL}; while (1) { @@ -1635,13 +1673,13 @@ static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t limited = 1; if (!sub || !next_union_state(e, 0)) break; - if (limited || !has_exists || e->Runions.more == oldRmore) { + if (limited || e->Runions.more == oldRmore) { // re-save env and freeze the ∃decision for previous ∀Union // Note: We could ignore the rest `∃Union` decisions if `x` and `y` // contain no ∃ typevar, as they have no effect on env. ini_count = count; push_unionstate(&latestLunions, &e->Lunions); - re_save_env(e, &se, has_exists); + re_save_env(e, &se, 1); e->Runions.more = oldRmore; } } @@ -1649,12 +1687,12 @@ static int local_forall_exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t break; assert(e->Runions.more > oldRmore); next_union_state(e, 1); - restore_env(e, &se, has_exists); // also restore Rdepth here + restore_env(e, &se, 1); // also restore Rdepth here e->Runions.more = oldRmore; } if (!sub) assert(e->Runions.more == oldRmore); - else if (limited || !has_exists) + else if (limited) e->Runions.more = oldRmore; free_env(&se); return sub; @@ -2539,9 +2577,6 @@ static jl_value_t *intersect_aside(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, static jl_value_t *intersect_union(jl_value_t *x, jl_uniontype_t *u, jl_stenv_t *e, int8_t R, int param) { - // band-aid for #56040 - if (!jl_is_uniontype(x) && obviously_in_union((jl_value_t *)u, x)) - return x; int no_free = !jl_has_free_typevars(x) && !jl_has_free_typevars((jl_value_t*)u); if (param == 2 || no_free) { jl_value_t *a=NULL, *b=NULL; @@ -2678,7 +2713,7 @@ static void set_bound(jl_value_t **bound, jl_value_t *val, jl_tvar_t *v, jl_sten // subtype, treating all vars as existential static int subtype_in_env_existential(jl_value_t *x, jl_value_t *y, jl_stenv_t *e) { - if (x == jl_bottom_type || y == (jl_value_t*)jl_any_type) + if (x == jl_bottom_type || y == (jl_value_t*)jl_any_type || obviously_in_union(y, x)) return 1; int8_t *rs = (int8_t*)alloca(current_env_length(e)); jl_varbinding_t *v = e->vars; @@ -2801,7 +2836,7 @@ static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int jl_value_t *ub = R ? intersect_aside(a, bb->ub, e, bb->depth0) : intersect_aside(bb->ub, a, e, bb->depth0); if (ub == jl_bottom_type) return jl_bottom_type; - if (bb->constraintkind == 1 || e->triangular) { + if (bb->constraintkind == 1 || (e->triangular && param == 1)) { if (e->triangular && check_unsat_bound(ub, b, e)) return jl_bottom_type; set_bound(&bb->ub, ub, b, e); @@ -4116,12 +4151,14 @@ static jl_value_t *intersect(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int pa if (jl_subtype(y, x)) return y; } if (jl_is_uniontype(x)) { - if (y == ((jl_uniontype_t*)x)->a || y == ((jl_uniontype_t*)x)->b) + if (obviously_in_union(x, y)) return y; + if (jl_is_uniontype(y) && obviously_in_union(y, x)) + return x; return intersect_union(y, (jl_uniontype_t*)x, e, 0, param); } if (jl_is_uniontype(y)) { - if (x == ((jl_uniontype_t*)y)->a || x == ((jl_uniontype_t*)y)->b) + if (obviously_in_union(y, x)) return x; if (jl_is_unionall(x) && (jl_has_free_typevars(x) || jl_has_free_typevars(y))) return intersect_unionall(y, (jl_unionall_t*)x, e, 0, param); diff --git a/src/support/dtypes.h b/src/support/dtypes.h index 6513370da4dae..2ee2d3529c10d 100644 --- a/src/support/dtypes.h +++ b/src/support/dtypes.h @@ -25,6 +25,9 @@ #include #include #define WIN32_LEAN_AND_MEAN +/* Clang does not like fvisibility=hidden with windows headers. This adds the visibility attribute there. + Arguably this is a clang bug. */ +#define DECLSPEC_IMPORT __declspec(dllimport) __attribute__ ((visibility("default"))) #include #if defined(_COMPILER_MICROSOFT_) && !defined(_SSIZE_T_) && !defined(_SSIZE_T_DEFINED) @@ -37,21 +40,6 @@ typedef intptr_t ssize_t; #endif /* defined(_COMPILER_MICROSOFT_) && !defined(_SSIZE_T_) && !defined(_SSIZE_T_DEFINED) */ -#if !defined(_COMPILER_GCC_) - -#define strtoull _strtoui64 -#define strtoll _strtoi64 -#define strcasecmp _stricmp -#define strncasecmp _strnicmp -#define snprintf _snprintf -#define stat _stat - -#define STDIN_FILENO 0 -#define STDOUT_FILENO 1 -#define STDERR_FILENO 2 - -#endif /* !_COMPILER_GCC_ */ - #endif /* _OS_WINDOWS_ */ @@ -73,13 +61,13 @@ typedef intptr_t ssize_t; #ifdef _OS_WINDOWS_ #define STDCALL __stdcall # ifdef JL_LIBRARY_EXPORTS_INTERNAL -# define JL_DLLEXPORT __declspec(dllexport) +# define JL_DLLEXPORT __declspec(dllexport) __attribute__ ((visibility("default"))) # endif # ifdef JL_LIBRARY_EXPORTS_CODEGEN -# define JL_DLLEXPORT_CODEGEN __declspec(dllexport) +# define JL_DLLEXPORT_CODEGEN __declspec(dllexport) __attribute__ ((visibility("default"))) # endif #define JL_HIDDEN -#define JL_DLLIMPORT __declspec(dllimport) +#define JL_DLLIMPORT __declspec(dllimport) __attribute__ ((visibility("default"))) #else #define STDCALL #define JL_DLLIMPORT __attribute__ ((visibility("default"))) @@ -123,11 +111,7 @@ typedef intptr_t ssize_t; #define STATIC_INLINE static inline #define FORCE_INLINE static inline __attribute__((always_inline)) -#ifdef _OS_WINDOWS_ -#define EXTERN_INLINE_DECLARE inline -#else #define EXTERN_INLINE_DECLARE inline __attribute__ ((visibility("default"))) -#endif #define EXTERN_INLINE_DEFINE extern inline JL_DLLEXPORT #if defined(_OS_WINDOWS_) && !defined(_COMPILER_GCC_) diff --git a/src/support/ios.h b/src/support/ios.h index 6eab9e21c45b6..bd26ee8d28181 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -14,8 +14,6 @@ extern "C" { // this flag controls when data actually moves out to the underlying I/O // channel. memory streams are a special case of this where the data // never moves out. - -//make it compatible with UV Handles typedef enum { bm_none=1000, bm_line, bm_block, bm_mem } bufmode_t; typedef enum { bst_none, bst_rd, bst_wr } bufstate_t; diff --git a/src/task.c b/src/task.c index 210f9fe718e49..6d48380e14bf8 100644 --- a/src/task.c +++ b/src/task.c @@ -37,6 +37,10 @@ #include "threading.h" #include "julia_assert.h" +#ifdef _COMPILER_TSAN_ENABLED_ +#include +#endif + #ifdef __cplusplus extern "C" { #endif @@ -335,7 +339,7 @@ void JL_NORETURN jl_finish_task(jl_task_t *ct) // let the runtime know this task is dead and find a new task to run jl_function_t *done = jl_atomic_load_relaxed(&task_done_hook_func); if (done == NULL) { - done = (jl_function_t*)jl_get_global(jl_base_module, jl_symbol("task_done_hook")); + done = (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("task_done_hook"), ct->world_age); if (done != NULL) jl_atomic_store_release(&task_done_hook_func, done); } @@ -993,7 +997,7 @@ the xoshiro256 state. There are two problems with that fix, however: need four variations of the internal RNG stream for the four xoshiro256 registers. That means we'd have to apply the PCG finalizer, add it to our dot product accumulator field in the child task, then apply the - MurmurHash3 finalizer to that dot product and use the result to purturb + MurmurHash3 finalizer to that dot product and use the result to perturb the main RNG state. We avoid both problems by recognizing that the mixing function can be much less diff --git a/src/threading.c b/src/threading.c index 0310941c40678..06c2475ff0869 100644 --- a/src/threading.c +++ b/src/threading.c @@ -1,15 +1,17 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license - #include #include #include #include #include - #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" +#ifdef _COMPILER_TSAN_ENABLED_ +#include +#endif + #ifdef USE_ITTAPI #include "ittapi/ittnotify.h" #endif @@ -411,9 +413,11 @@ _Atomic(jl_function_t*) init_task_lock_func JL_GLOBALLY_ROOTED = NULL; static void jl_init_task_lock(jl_task_t *ct) { + size_t last_age = ct->world_age; + ct->world_age = jl_get_world_counter(); jl_function_t *done = jl_atomic_load_relaxed(&init_task_lock_func); if (done == NULL) { - done = (jl_function_t*)jl_get_global(jl_base_module, jl_symbol("init_task_lock")); + done = (jl_function_t*)jl_get_global_value(jl_base_module, jl_symbol("init_task_lock"), ct->world_age); if (done != NULL) jl_atomic_store_release(&init_task_lock_func, done); } @@ -426,6 +430,7 @@ static void jl_init_task_lock(jl_task_t *ct) jl_no_exc_handler(jl_current_exception(ct), ct); } } + ct->world_age = last_age; } JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void) @@ -448,7 +453,6 @@ JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void) JL_GC_PROMISE_ROOTED(ct); uv_random(NULL, NULL, &ct->rngState, sizeof(ct->rngState), 0, NULL); jl_atomic_fetch_add(&jl_gc_disable_counter, -1); - ct->world_age = jl_get_world_counter(); // root_task sets world_age to 1 jl_init_task_lock(ct); return &ct->gcstack; } @@ -457,15 +461,14 @@ JL_DLLEXPORT jl_gcframe_t **jl_autoinit_and_adopt_thread(void) { if (!jl_is_initialized()) { void *retaddr = __builtin_extract_return_addr(__builtin_return_address(0)); - void *sysimg_handle = jl_find_dynamic_library_by_addr(retaddr, /* throw_err */ 0); - - if (sysimg_handle == NULL) { + void *handle = jl_find_dynamic_library_by_addr(retaddr, 0); + if (handle == NULL) { fprintf(stderr, "error: runtime auto-initialization failed due to bad sysimage lookup\n" " (this should not happen, please file a bug report)\n"); - abort(); + exit(1); } - - assert(0 && "TODO: implement auto-init"); + jl_init_with_image_handle(handle); + return &jl_get_current_task()->gcstack; } return jl_adopt_thread(); @@ -562,18 +565,20 @@ static void jl_delete_thread(void *value) JL_NOTSAFEPOINT_ENTER // this here by blocking. This also synchronizes our read of `current_task` // (which is the flag we currently use to check the liveness state of a thread). #ifdef _OS_WINDOWS_ - jl_lock_profile_wr(); + int havelock = jl_lock_profile_wr(); + assert(havelock); (void)havelock; #elif defined(JL_DISABLE_LIBUNWIND) // nothing #elif defined(__APPLE__) - jl_lock_profile_wr(); + int havelock = jl_lock_profile_wr(); + assert(havelock); (void)havelock; #else pthread_mutex_lock(&in_signal_lock); #endif jl_atomic_store_relaxed(&ptls->current_task, NULL); // indicate dead // finally, release all of the locks we had grabbed #ifdef _OS_WINDOWS_ - jl_unlock_profile_wr(); + if (havelock) jl_unlock_profile_wr(); #elif defined(JL_DISABLE_LIBUNWIND) // nothing #elif defined(__APPLE__) @@ -739,6 +744,8 @@ void jl_init_threading(void) if (errno != 0 || endptr == cp || nthreads <= 0) nthreads = 1; cp = endptr; + if (nthreads == 1) // User asked for 1 thread so lets assume they dont want an interactive thread + nthreadsi = 0; } if (*cp == ',') { cp++; @@ -928,7 +935,16 @@ void _jl_mutex_init(jl_mutex_t *lock, const char *name) JL_NOTSAFEPOINT { jl_atomic_store_relaxed(&lock->owner, (jl_task_t*)NULL); lock->count = 0; +#if defined(_COMPILER_TSAN_ENABLED_) && defined(ENABLE_TIMINGS) + __tsan_mutex_pre_divert(lock, 0); +#endif jl_profile_lock_init(lock, name); +#ifdef _COMPILER_TSAN_ENABLED_ +#ifdef ENABLE_TIMINGS + __tsan_mutex_post_divert(lock, 0); +#endif + __tsan_mutex_create(lock, __tsan_mutex_write_reentrant); +#endif } void _jl_mutex_wait(jl_task_t *self, jl_mutex_t *lock, int safepoint) @@ -938,11 +954,17 @@ void _jl_mutex_wait(jl_task_t *self, jl_mutex_t *lock, int safepoint) lock->count++; return; } +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_divert(lock, 0); +#endif // Don't use JL_TIMING for instant acquires, results in large blowup of events jl_profile_lock_start_wait(lock); if (owner == NULL && jl_atomic_cmpswap(&lock->owner, &owner, self)) { lock->count = 1; jl_profile_lock_acquired(lock); +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_divert(lock, 0); +#endif return; } JL_TIMING(LOCK_SPIN, LOCK_SPIN); @@ -950,6 +972,9 @@ void _jl_mutex_wait(jl_task_t *self, jl_mutex_t *lock, int safepoint) if (owner == NULL && jl_atomic_cmpswap(&lock->owner, &owner, self)) { lock->count = 1; jl_profile_lock_acquired(lock); +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_divert(lock, 0); +#endif return; } if (jl_running_under_rr(0)) { @@ -970,6 +995,9 @@ void _jl_mutex_wait(jl_task_t *self, jl_mutex_t *lock, int safepoint) jl_cpu_suspend(); owner = jl_atomic_load_relaxed(&lock->owner); } +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_divert(lock, 0); +#endif } static void jl_lock_frame_push(jl_task_t *self, jl_mutex_t *lock) @@ -995,23 +1023,43 @@ static void jl_lock_frame_pop(jl_task_t *self) void _jl_mutex_lock(jl_task_t *self, jl_mutex_t *lock) { +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_lock(lock, __tsan_mutex_write_reentrant); +#endif JL_SIGATOMIC_BEGIN_self(); _jl_mutex_wait(self, lock, 1); jl_lock_frame_push(self, lock); +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_lock(lock, __tsan_mutex_write_reentrant, 1); +#endif } int _jl_mutex_trylock_nogc(jl_task_t *self, jl_mutex_t *lock) { +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_lock(lock, __tsan_mutex_try_lock | __tsan_mutex_write_reentrant); +#endif jl_task_t *owner = jl_atomic_load_acquire(&lock->owner); + int ret = 0; if (owner == self) { lock->count++; - return 1; + ret = 1; + goto done; } if (owner == NULL && jl_atomic_cmpswap(&lock->owner, &owner, self)) { lock->count = 1; - return 1; + ret = 1; + goto done; } - return 0; +done: +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_lock(lock, + __tsan_mutex_try_lock | + (ret ? 0 : __tsan_mutex_try_lock_failed) | + __tsan_mutex_write_reentrant, + 1); +#endif + return ret; } int _jl_mutex_trylock(jl_task_t *self, jl_mutex_t *lock) @@ -1027,6 +1075,9 @@ int _jl_mutex_trylock(jl_task_t *self, jl_mutex_t *lock) void _jl_mutex_unlock_nogc(jl_mutex_t *lock) { #ifndef __clang_gcanalyzer__ +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_pre_unlock(lock, 0); +#endif assert(jl_atomic_load_relaxed(&lock->owner) == jl_current_task && "Unlocking a lock in a different thread."); if (--lock->count == 0) { @@ -1041,6 +1092,9 @@ void _jl_mutex_unlock_nogc(jl_mutex_t *lock) } jl_profile_lock_release_end(lock); } +#ifdef _COMPILER_TSAN_ENABLED_ + __tsan_mutex_post_unlock(lock, 0); +#endif #endif } diff --git a/src/timing.c b/src/timing.c index 265e50ad3dd74..f0abd19a1e347 100644 --- a/src/timing.c +++ b/src/timing.c @@ -16,6 +16,15 @@ jl_module_t *jl_module_root(jl_module_t *m); extern "C" { #endif +JL_DLLEXPORT int jl_timing_enabled(void) { +#ifdef ENABLE_TIMINGS + return 1; +#else + return 0; +#endif +} + + #ifdef ENABLE_TIMINGS #ifndef HAVE_TIMING_SUPPORT @@ -185,6 +194,7 @@ void jl_init_timing(void) error |= jl_timing_set_enable("METHOD_LOOKUP_FAST", 0); error |= jl_timing_set_enable("AST_COMPRESS", 0); error |= jl_timing_set_enable("AST_UNCOMPRESS", 0); + error |= jl_timing_set_enable("TYPE_CACHE_INSERT", 0); if (error) jl_error("invalid timing subsystem encountered in jl_init_timing"); #endif diff --git a/src/timing.h b/src/timing.h index 61118cc3b41ab..833f5f68d34f5 100644 --- a/src/timing.h +++ b/src/timing.h @@ -55,6 +55,8 @@ JL_DLLEXPORT jl_timing_event_t *_jl_timing_event_create(const char *subsystem, c JL_DLLEXPORT void _jl_timing_block_init(char *buf, size_t size, jl_timing_event_t *event); JL_DLLEXPORT void _jl_timing_block_start(jl_timing_block_t *cur_block); JL_DLLEXPORT void _jl_timing_block_end(jl_timing_block_t *cur_block); +JL_DLLEXPORT int jl_timing_enabled(void); + #ifdef __cplusplus } @@ -187,6 +189,7 @@ JL_DLLEXPORT void jl_timing_puts(jl_timing_block_t *cur_block, const char *str); X(STACKWALK) \ X(DL_OPEN) \ X(JULIA_INIT) \ + X(CORE_COMPILER) \ #define JL_TIMING_COUNTERS \ diff --git a/src/toplevel.c b/src/toplevel.c index 98d627c5a9a30..9c19164d37812 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -26,42 +26,34 @@ extern "C" { #endif // current line number in a file -JL_DLLEXPORT int jl_lineno = 0; // need to update jl_critical_error if this is TLS +JL_DLLEXPORT _Atomic(int) jl_lineno = 0; // need to update jl_critical_error if this is TLS // current file name -JL_DLLEXPORT const char *jl_filename = "none"; // need to update jl_critical_error if this is TLS +JL_DLLEXPORT _Atomic(const char *) jl_filename = "none"; // need to update jl_critical_error if this is TLS htable_t jl_current_modules; jl_mutex_t jl_modules_mutex; // During incremental compilation, the following gets set -JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module = NULL; // the toplevel module currently being defined +jl_module_t *jl_precompile_toplevel_module = NULL; // the first toplevel module being defined -JL_DLLEXPORT void jl_add_standard_imports(jl_module_t *m) +jl_module_t *jl_add_standard_imports(jl_module_t *m) { jl_module_t *base_module = jl_base_relative_to(m); assert(base_module != NULL); // using Base - jl_module_using(m, base_module); + jl_module_initial_using(m, base_module); + return base_module; } // create a new top-level module void jl_init_main_module(void) { assert(jl_main_module == NULL); - jl_main_module = jl_new_module(jl_symbol("Main"), NULL); - jl_main_module->parent = jl_main_module; - jl_set_const(jl_main_module, jl_symbol("Core"), - (jl_value_t*)jl_core_module); - jl_set_const(jl_core_module, jl_symbol("Main"), - (jl_value_t*)jl_main_module); + jl_main_module = jl_new_module_(jl_symbol("Main"), NULL, 0, 1); // baremodule Main; end + jl_set_initial_const(jl_core_module, jl_symbol("Main"), (jl_value_t*)jl_main_module, 0); // const Main.Core = Core + jl_set_initial_const(jl_main_module, jl_symbol("Core"), (jl_value_t*)jl_core_module, 0); // const Core.Main = Main } -static jl_function_t *jl_module_get_initializer(jl_module_t *m JL_PROPAGATES_ROOT) -{ - return (jl_function_t*)jl_get_global(m, jl_symbol("__init__")); -} - - void jl_module_run_initializer(jl_module_t *m) { JL_TIMING(INIT_MODULE, INIT_MODULE); @@ -70,9 +62,12 @@ void jl_module_run_initializer(jl_module_t *m) size_t last_age = ct->world_age; JL_TRY { ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - jl_function_t *f = jl_module_get_initializer(m); - if (f != NULL) + jl_value_t *f = jl_get_global_value(m, jl_symbol("__init__"), ct->world_age); + if (f != NULL) { + JL_GC_PUSH1(&f); jl_apply(&f, 1); + JL_GC_POP(); + } ct->world_age = last_age; } JL_CATCH { @@ -109,10 +104,10 @@ jl_array_t *jl_get_loaded_modules(void) return NULL; } -static int jl_is__toplevel__mod(jl_module_t *mod) +static int jl_is__toplevel__mod(jl_module_t *mod, jl_task_t *ct) { return jl_base_module && - (jl_value_t*)mod == jl_get_global(jl_base_module, jl_symbol("__toplevel__")); + (jl_value_t*)mod == jl_get_global_value(jl_base_module, jl_symbol("__toplevel__"), ct->world_age); } // TODO: add locks around global state mutation operations @@ -134,43 +129,19 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex jl_type_error("module", (jl_value_t*)jl_symbol_type, (jl_value_t*)name); } - int is_parent__toplevel__ = jl_is__toplevel__mod(parent_module); + int is_parent__toplevel__ = jl_is__toplevel__mod(parent_module, ct); // If we have `Base`, don't also try to import `Core` - the `Base` exports are a superset. // While we allow multiple imports of the same binding from different modules, various error printing // performs reflection on which module a binding came from and we'd prefer users see "Base" here. - jl_module_t *newm = jl_new_module__(name, is_parent__toplevel__ ? NULL : parent_module); + jl_module_t *newm = jl_new_module_(name, is_parent__toplevel__ ? NULL : parent_module, std_imports && jl_base_module != NULL ? 0 : 1, 1); jl_value_t *form = (jl_value_t*)newm; JL_GC_PUSH1(&form); JL_LOCK(&jl_modules_mutex); OBJHASH_PIN(newm) ptrhash_put(&jl_current_modules, (void*)newm, (void*)((uintptr_t)HT_NOTFOUND + 1)); JL_UNLOCK(&jl_modules_mutex); - - jl_add_default_names(newm, std_imports && jl_base_module != NULL ? 0 : 1, 1); - - jl_module_t *old_toplevel_module = jl_precompile_toplevel_module; - // copy parent environment info into submodule newm->uuid = parent_module->uuid; - if (is_parent__toplevel__) { - newm->parent = newm; - jl_register_root_module(newm); - if (jl_options.incremental) { - jl_precompile_toplevel_module = newm; - } - } - else { - jl_declare_constant_val(NULL, parent_module, name, (jl_value_t*)newm); - } - - if (parent_module == jl_main_module && name == jl_symbol("Base")) { - // pick up Base module during bootstrap - jl_base_module = newm; - } - - size_t last_age = ct->world_age; - - // add standard imports unless baremodule jl_array_t *exprs = ((jl_expr_t*)jl_exprarg(ex, 2))->args; int lineno = 0; const char *filename = "none"; @@ -183,30 +154,45 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex filename = jl_symbol_name((jl_sym_t*)file); } } - if (std_imports) { - if (jl_base_module != NULL) { - jl_add_standard_imports(newm); - jl_datatype_t *include_into = (jl_datatype_t *)jl_get_global(jl_base_module, jl_symbol("IncludeInto")); - if (include_into) { - form = jl_new_struct(include_into, newm); - jl_set_const(newm, jl_symbol("include"), form); - } + newm->file = jl_symbol(filename); + jl_gc_wb_knownold(newm, newm->file); + newm->line = lineno; + + // add standard imports unless baremodule + if (std_imports && jl_base_module != NULL) { + jl_module_t *base = jl_add_standard_imports(newm); + jl_datatype_t *include_into = (jl_datatype_t *)jl_get_global(base, jl_symbol("IncludeInto")); + if (include_into) { + form = jl_new_struct(include_into, newm); + jl_set_initial_const(newm, jl_symbol("include"), form, 0); } jl_datatype_t *eval_into = (jl_datatype_t *)jl_get_global(jl_core_module, jl_symbol("EvalInto")); if (eval_into) { form = jl_new_struct(eval_into, newm); - jl_set_const(newm, jl_symbol("eval"), form); + jl_set_initial_const(newm, jl_symbol("eval"), form, 0); } } - newm->file = jl_symbol(filename); - jl_gc_wb_knownold(newm, newm->file); - newm->line = lineno; + size_t last_age = ct->world_age; + + if (parent_module == jl_main_module && name == jl_symbol("Base") && jl_base_module == NULL) { + // pick up Base module during bootstrap + jl_base_module = newm; + } + + if (is_parent__toplevel__) { + jl_register_root_module(newm); + if (jl_options.incremental && jl_precompile_toplevel_module == NULL) { + jl_precompile_toplevel_module = newm; + } + } + else { + jl_declare_constant_val(NULL, parent_module, name, (jl_value_t*)newm); + } for (int i = 0; i < jl_array_nrows(exprs); i++) { // process toplevel form - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - form = jl_expand_stmt_with_loc(jl_array_ptr_ref(exprs, i), newm, filename, lineno); + form = jl_svecref(jl_lower(jl_array_ptr_ref(exprs, i), newm, filename, lineno, ~(size_t)0, 0), 0); ct->world_age = jl_atomic_load_acquire(&jl_world_counter); (void)jl_toplevel_eval_flex(newm, form, 1, 1, &filename, &lineno); } @@ -255,25 +241,22 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex } } - jl_precompile_toplevel_module = old_toplevel_module; - JL_GC_POP(); return (jl_value_t*)newm; } -static jl_value_t *jl_eval_dot_expr(jl_module_t *m, jl_value_t *x, jl_value_t *f, int fast, const char **toplevel_filename, int *toplevel_lineno) +static jl_value_t *jl_eval_dot_expr(jl_task_t *ct, jl_module_t *m, jl_value_t *x, jl_value_t *f, int fast, const char **toplevel_filename, int *toplevel_lineno) { - jl_task_t *ct = jl_current_task; jl_value_t **args; JL_GC_PUSHARGS(args, 3); args[1] = jl_toplevel_eval_flex(m, x, fast, 0, toplevel_filename, toplevel_lineno); args[2] = jl_toplevel_eval_flex(m, f, fast, 0, toplevel_filename, toplevel_lineno); if (jl_is_module(args[1])) { JL_TYPECHK(getglobal, symbol, args[2]); - args[0] = jl_eval_global_var((jl_module_t*)args[1], (jl_sym_t*)args[2]); + args[0] = jl_eval_global_var((jl_module_t*)args[1], (jl_sym_t*)args[2], ct->world_age); } else { - args[0] = jl_eval_global_var(jl_base_relative_to(m), jl_symbol("getproperty")); + args[0] = jl_eval_global_var(jl_base_relative_to(m), jl_symbol("getproperty"), ct->world_age); size_t last_age = ct->world_age; ct->world_age = jl_atomic_load_acquire(&jl_world_counter); args[0] = jl_apply(args, 3); @@ -319,7 +302,7 @@ void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type, in goto check_type; } check_safe_newbinding(gm, gs); - if (bpart->min_world == new_world) { + if (jl_atomic_load_relaxed(&bpart->min_world) == new_world) { bpart->kind = new_kind | (bpart->kind & PARTITION_MASK_FLAG); bpart->restriction = global_type; if (global_type) @@ -427,7 +410,7 @@ static void expr_attributes(jl_value_t *v, jl_array_t *body, int *has_ccall, int jl_module_t *mod = jl_globalref_mod(f); jl_sym_t *name = jl_globalref_name(f); jl_binding_t *b = jl_get_binding(mod, name); - called = jl_get_binding_value_if_const(b); + called = jl_get_latest_binding_value_if_const(b); } else if (jl_is_quotenode(f)) { called = jl_quotenode_value(f); @@ -436,7 +419,7 @@ static void expr_attributes(jl_value_t *v, jl_array_t *body, int *has_ccall, int if (jl_is_intrinsic(called) && jl_unbox_int32(called) == (int)llvmcall) { *has_ccall = 1; } - if (called == jl_builtin__typebody) { // TODO: rely on latestworld instead of function callee detection here (or add it to jl_is_toplevel_only_expr) + if (called == BUILTIN(_typebody)) { // TODO: rely on latestworld instead of function callee detection here (or add it to jl_is_toplevel_only_expr) *has_defs = 1; } } @@ -488,110 +471,10 @@ static void body_attributes(jl_array_t *body, int *has_ccall, int *has_defs, int *forced_compile = jl_has_meta(body, jl_force_compile_sym); } -extern size_t jl_require_world; -static jl_module_t *call_require(jl_task_t *ct, jl_module_t *mod, jl_sym_t *var) JL_GLOBALLY_ROOTED -{ - JL_TIMING(LOAD_IMAGE, LOAD_Require); - jl_timing_printf(JL_TIMING_DEFAULT_BLOCK, "%s", jl_symbol_name(var)); - - int build_mode = jl_options.incremental && jl_generating_output(); - jl_module_t *m = NULL; - static jl_value_t *require_func = NULL; - if (require_func == NULL && jl_base_module != NULL) { - require_func = jl_get_global(jl_base_module, jl_symbol("require")); - } - if (require_func != NULL) { - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - if (build_mode && jl_require_world < ct->world_age) - ct->world_age = jl_require_world; - jl_value_t *reqargs[3]; - reqargs[0] = require_func; - reqargs[1] = (jl_value_t*)mod; - reqargs[2] = (jl_value_t*)var; - m = (jl_module_t*)jl_apply(reqargs, 3); - } - if (m == NULL || !jl_is_module(m)) { - jl_errorf("failed to load module %s", jl_symbol_name(var)); - } - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - return m; -} - -// either: -// - sets *name and returns the module to import *name from -// - sets *name to NULL and returns a module to import -// also updates world_age -static jl_module_t *eval_import_path(jl_task_t *ct, jl_module_t *where, jl_module_t *from JL_PROPAGATES_ROOT, - jl_array_t *args, jl_sym_t **name, const char *keyword) JL_GLOBALLY_ROOTED -{ - if (jl_array_nrows(args) == 0) - jl_errorf("malformed \"%s\" statement", keyword); - jl_sym_t *var = (jl_sym_t*)jl_array_ptr_ref(args, 0); - size_t i = 1; - jl_module_t *m = NULL; - *name = NULL; - if (!jl_is_symbol(var)) - jl_type_error(keyword, (jl_value_t*)jl_symbol_type, (jl_value_t*)var); - - if (from != NULL) { - m = from; - i = 0; - } - else if (var != jl_dot_sym) { - // `A.B`: call the loader to obtain the root A in the current environment. - if (jl_core_module && var == jl_core_module->name) { - m = jl_core_module; - } - else if (jl_base_module && var == jl_base_module->name) { - m = jl_base_module; - } - else { - m = call_require(ct, where, var); - } - if (i == jl_array_nrows(args)) - return m; - } - else { - // `.A.B.C`: strip off leading dots by following parent links - m = where; - while (1) { - if (i >= jl_array_nrows(args)) - jl_error("invalid module path"); - var = (jl_sym_t*)jl_array_ptr_ref(args, i); - if (var != jl_dot_sym) - break; - i++; - assert(m); - m = m->parent; - } - } - - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - - while (1) { - var = (jl_sym_t*)jl_array_ptr_ref(args, i); - if (!jl_is_symbol(var)) - jl_type_error(keyword, (jl_value_t*)jl_symbol_type, (jl_value_t*)var); - if (var == jl_dot_sym) - jl_errorf("invalid %s path: \".\" in identifier path", keyword); - if (i == jl_array_nrows(args)-1) - break; - m = (jl_module_t*)jl_eval_global_var(m, var); - JL_GC_PROMISE_ROOTED(m); - if (!jl_is_module(m)) - jl_errorf("invalid %s path: \"%s\" does not name a module", keyword, jl_symbol_name(var)); - i++; - } - *name = var; - return m; -} - int jl_is_toplevel_only_expr(jl_value_t *e) JL_NOTSAFEPOINT { return jl_is_expr(e) && (((jl_expr_t*)e)->head == jl_module_sym || - ((jl_expr_t*)e)->head == jl_import_sym || - ((jl_expr_t*)e)->head == jl_using_sym || ((jl_expr_t*)e)->head == jl_export_sym || ((jl_expr_t*)e)->head == jl_public_sym || ((jl_expr_t*)e)->head == jl_thunk_sym || @@ -609,10 +492,9 @@ int jl_needs_lowering(jl_value_t *e) JL_NOTSAFEPOINT return 0; jl_expr_t *ex = (jl_expr_t*)e; jl_sym_t *head = ex->head; - if (head == jl_module_sym || head == jl_import_sym || head == jl_using_sym || - head == jl_export_sym || head == jl_public_sym || head == jl_thunk_sym || - head == jl_toplevel_sym || head == jl_error_sym || head == jl_incomplete_sym || - head == jl_method_sym) { + if (head == jl_module_sym || head == jl_export_sym || head == jl_public_sym || + head == jl_thunk_sym || head == jl_toplevel_sym || head == jl_error_sym || + head == jl_incomplete_sym || head == jl_method_sym) { return 0; } if (head == jl_global_sym || head == jl_const_sym) { @@ -629,11 +511,15 @@ int jl_needs_lowering(jl_value_t *e) JL_NOTSAFEPOINT JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst_for_uninferred(jl_method_instance_t *mi, jl_code_info_t *src) { + jl_svec_t *edges = jl_emptysvec; + if (src->edges && jl_is_svec(src->edges)) + edges = (jl_svec_t*)src->edges; + // Do not compress this, we expect it to be shortlived. jl_code_instance_t *ci = jl_new_codeinst(mi, (jl_value_t*)jl_uninferred_sym, (jl_value_t*)jl_any_type, (jl_value_t*)jl_any_type, jl_nothing, (jl_value_t*)src, 0, src->min_world, src->max_world, - 0, NULL, NULL, NULL); + 0, NULL, NULL, edges); return ci; } @@ -652,62 +538,6 @@ JL_DLLEXPORT jl_method_instance_t *jl_method_instance_for_thunk(jl_code_info_t * return mi; } -static void import_module(jl_task_t *ct, jl_module_t *JL_NONNULL m, jl_module_t *import, jl_sym_t *asname) -{ - assert(m); - jl_sym_t *name = asname ? asname : import->name; - // TODO: this is a bit race-y with what error message we might print - jl_binding_t *b = jl_get_module_binding(m, name, 1); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, ct->world_age); - enum jl_partition_kind kind = jl_binding_kind(bpart); - if (!jl_bkind_is_some_implicit(kind) && kind != PARTITION_KIND_DECLARED) { - // Unlike regular constant declaration, we allow this as long as we eventually end up at a constant. - jl_walk_binding_inplace(&b, &bpart, ct->world_age); - if (jl_bkind_is_some_constant(jl_binding_kind(bpart))) { - // Already declared (e.g. on another thread) or imported. - if (bpart->restriction == (jl_value_t*)import) - return; - } - jl_errorf("importing %s into %s conflicts with an existing global", - jl_symbol_name(name), jl_symbol_name(m->name)); - } - jl_declare_constant_val2(b, m, name, (jl_value_t*)import, PARTITION_KIND_CONST_IMPORT); -} - -// in `import A.B: x, y, ...`, evaluate the `A.B` part if it exists -static jl_module_t *eval_import_from(jl_task_t *ct, jl_module_t *m JL_PROPAGATES_ROOT, jl_expr_t *ex, const char *keyword) -{ - if (jl_expr_nargs(ex) == 1 && jl_is_expr(jl_exprarg(ex, 0))) { - jl_expr_t *fr = (jl_expr_t*)jl_exprarg(ex, 0); - if (fr->head == jl_colon_sym) { - if (jl_expr_nargs(fr) > 0 && jl_is_expr(jl_exprarg(fr, 0))) { - jl_expr_t *path = (jl_expr_t*)jl_exprarg(fr, 0); - if (((jl_expr_t*)path)->head == jl_dot_sym) { - jl_sym_t *name = NULL; - jl_module_t *from = eval_import_path(ct, m, NULL, path->args, &name, keyword); - if (name != NULL) { - from = (jl_module_t*)jl_eval_global_var(from, name); - if (!from || !jl_is_module(from)) - jl_errorf("invalid %s path: \"%s\" does not name a module", keyword, jl_symbol_name(name)); - } - return from; - } - } - jl_errorf("malformed \"%s:\" statement", keyword); - } - } - return NULL; -} - -static void check_macro_rename(jl_sym_t *from, jl_sym_t *to, const char *keyword) -{ - char *n1 = jl_symbol_name(from), *n2 = jl_symbol_name(to); - if (n1[0] == '@' && n2[0] != '@') - jl_errorf("cannot rename macro \"%s\" to non-macro \"%s\" in \"%s\"", n1, n2, keyword); - if (n1[0] != '@' && n2[0] == '@') - jl_errorf("cannot rename non-macro \"%s\" to macro \"%s\" in \"%s\"", n1, n2, keyword); -} - // Eval `throw(ErrorException(msg)))` in module `m`. // Used in `jl_toplevel_eval_flex` instead of `jl_throw` so that the error // location in julia code gets into the backtrace. @@ -715,7 +545,7 @@ static void jl_eval_throw(jl_module_t *m, jl_value_t *exc, const char *filename, { jl_value_t *throw_ex = (jl_value_t*)jl_exprn(jl_call_sym, 2); JL_GC_PUSH1(&throw_ex); - jl_exprargset(throw_ex, 0, jl_builtin_throw); + jl_exprargset(throw_ex, 0, BUILTIN(throw)); jl_exprargset(throw_ex, 1, exc); jl_toplevel_eval_flex(m, throw_ex, 0, 0, &filename, &lineno); JL_GC_POP(); @@ -740,7 +570,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val2( JL_LOCK(&world_counter_lock); size_t new_world = jl_atomic_load_relaxed(&jl_world_counter) + 1; jl_binding_partition_t *bpart = jl_declare_constant_val3(b, mod, var, val, constant_kind, new_world); - if (bpart->min_world == new_world) + if (jl_atomic_load_relaxed(&bpart->min_world) == new_world) jl_atomic_store_release(&jl_world_counter, new_world); JL_UNLOCK(&world_counter_lock); return bpart; @@ -771,6 +601,16 @@ JL_DLLEXPORT void jl_eval_const_decl(jl_module_t *m, jl_value_t *arg, jl_value_t JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_value_t *e, int fast, int expanded, const char **toplevel_filename, int *toplevel_lineno) { jl_task_t *ct = jl_current_task; + if (jl_is_globalref(e)) { + return jl_eval_globalref((jl_globalref_t*)e, ct->world_age); + } + if (jl_is_symbol(e)) { + char *n = jl_symbol_name((jl_sym_t*)e), *n0 = n; + while (*n == '_') ++n; + if (*n == 0 && n > n0) + jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, "all-underscore identifiers are write-only and their values cannot be used in expressions"); + return jl_eval_global_var(m, (jl_sym_t*)e, ct->world_age); + } if (!jl_is_expr(e)) { if (jl_is_linenode(e)) { *toplevel_lineno = jl_linenode_line(e); @@ -780,16 +620,10 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val *toplevel_filename = jl_symbol_name((jl_sym_t*)file); } // Not thread safe. For debugging and last resort error messages (jl_critical_error) only. - jl_filename = *toplevel_filename; - jl_lineno = *toplevel_lineno; + jl_atomic_store_relaxed(&jl_filename, *toplevel_filename); + jl_atomic_store_relaxed(&jl_lineno, *toplevel_lineno); return jl_nothing; } - if (jl_is_symbol(e)) { - char *n = jl_symbol_name((jl_sym_t*)e), *n0 = n; - while (*n == '_') ++n; - if (*n == 0 && n > n0) - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, "all-underscore identifiers are write-only and their values cannot be used in expressions"); - } return jl_interpret_toplevel_expr_in(m, e, NULL, NULL); } @@ -802,7 +636,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val jl_value_t *rhs = jl_exprarg(ex, 1); // only handle `a.b` syntax here, so qualified names can be eval'd in pure contexts if (jl_is_quotenode(rhs) && jl_is_symbol(jl_fieldref(rhs, 0))) { - return jl_eval_dot_expr(m, lhs, rhs, fast, toplevel_filename, toplevel_lineno); + return jl_eval_dot_expr(ct, m, lhs, rhs, fast, toplevel_filename, toplevel_lineno); } } @@ -816,10 +650,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val JL_GC_PUSH4(&mfunc, &thk, &ex, &root); size_t last_age = ct->world_age; - if (!expanded && jl_needs_lowering(e)) { - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - ex = (jl_expr_t*)jl_expand_with_loc_warn(e, m, *toplevel_filename, *toplevel_lineno); - ct->world_age = last_age; + if (!expanded && (jl_needs_lowering(e))) { + ex = (jl_expr_t*)jl_svecref(jl_lower(e, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1), 0); } jl_sym_t *head = jl_is_expr(ex) ? ex->head : NULL; @@ -828,121 +660,31 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val JL_GC_POP(); return val; } - else if (head == jl_using_sym) { - jl_sym_t *name = NULL; - jl_module_t *from = eval_import_from(ct, m, ex, "using"); - size_t i = 0; - if (from) { - i = 1; - ex = (jl_expr_t*)jl_exprarg(ex, 0); - } - for (; i < jl_expr_nargs(ex); i++) { - jl_value_t *a = jl_exprarg(ex, i); - if (jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_dot_sym) { - name = NULL; - jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)a)->args, &name, "using"); - if (from) { - // `using A: B` and `using A: B.c` syntax - jl_module_use(ct, m, import, name); - } - else { - jl_module_t *u = import; - if (name != NULL) - u = (jl_module_t*)jl_eval_global_var(import, name); - if (!jl_is_module(u)) - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, - "invalid using path: \"%s\" does not name a module", - jl_symbol_name(name)); - // `using A` and `using A.B` syntax - jl_module_using(m, u); - if (m == jl_main_module && name == NULL) { - // TODO: for now, `using A` in Main also creates an explicit binding for `A` - // This will possibly be extended to all modules. - import_module(ct, m, u, NULL); - } - } - continue; - } - else if (from && jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_as_sym && jl_expr_nargs(a) == 2 && - jl_is_expr(jl_exprarg(a, 0)) && ((jl_expr_t*)jl_exprarg(a, 0))->head == jl_dot_sym) { - jl_sym_t *asname = (jl_sym_t*)jl_exprarg(a, 1); - if (jl_is_symbol(asname)) { - jl_expr_t *path = (jl_expr_t*)jl_exprarg(a, 0); - name = NULL; - jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)path)->args, &name, "using"); - assert(name); - check_macro_rename(name, asname, "using"); - // `using A: B as C` syntax - jl_module_use_as(ct, m, import, name, asname); - continue; - } - } - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, - "syntax: malformed \"using\" statement"); - } - JL_GC_POP(); - ct->world_age = last_age; - return jl_nothing; - } - else if (head == jl_import_sym) { - jl_sym_t *name = NULL; - jl_module_t *from = eval_import_from(ct, m, ex, "import"); - size_t i = 0; - if (from) { - i = 1; - ex = (jl_expr_t*)jl_exprarg(ex, 0); - } - for (; i < jl_expr_nargs(ex); i++) { - jl_value_t *a = jl_exprarg(ex, i); - if (jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_dot_sym) { - name = NULL; - jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)a)->args, &name, "import"); - if (name == NULL) { - // `import A` syntax - import_module(ct, m, import, NULL); - } - else { - // `import A.B` or `import A: B` syntax - jl_module_import(ct, m, import, name); - } - continue; - } - else if (jl_is_expr(a) && ((jl_expr_t*)a)->head == jl_as_sym && jl_expr_nargs(a) == 2 && - jl_is_expr(jl_exprarg(a, 0)) && ((jl_expr_t*)jl_exprarg(a, 0))->head == jl_dot_sym) { - jl_sym_t *asname = (jl_sym_t*)jl_exprarg(a, 1); - if (jl_is_symbol(asname)) { - jl_expr_t *path = (jl_expr_t*)jl_exprarg(a, 0); - name = NULL; - jl_module_t *import = eval_import_path(ct, m, from, ((jl_expr_t*)path)->args, &name, "import"); - if (name == NULL) { - // `import A as B` syntax - import_module(ct, m, import, asname); - } - else { - check_macro_rename(name, asname, "import"); - // `import A.B as C` syntax - jl_module_import_as(ct, m, import, name, asname); - } - continue; - } - } - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, - "syntax: malformed \"import\" statement"); - } - JL_GC_POP(); - ct->world_age = last_age; - return jl_nothing; - } else if (head == jl_export_sym || head == jl_public_sym) { int exp = (head == jl_export_sym); - for (size_t i = 0; i < jl_array_nrows(ex->args); i++) { - jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, i); - if (!jl_is_symbol(name)) - jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, - exp ? "syntax: malformed \"export\" statement" : - "syntax: malformed \"public\" statement"); - jl_module_public(m, name, exp); + volatile int any_new = 0; + JL_LOCK(&world_counter_lock); + size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; + JL_TRY { + for (size_t i = 0; i < jl_array_nrows(ex->args); i++) { + jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, i); + if (!jl_is_symbol(name)) + jl_eval_errorf(m, *toplevel_filename, *toplevel_lineno, + exp ? "syntax: malformed \"export\" statement" : + "syntax: malformed \"public\" statement"); + if (jl_module_public_(m, name, exp, new_world)) + any_new = 1; + } } + JL_CATCH { + if (any_new) + jl_atomic_store_release(&jl_world_counter, new_world); + JL_UNLOCK(&world_counter_lock); + jl_rethrow(); + } + if (any_new) + jl_atomic_store_release(&jl_world_counter, new_world); + JL_UNLOCK(&world_counter_lock); JL_GC_POP(); return jl_nothing; } @@ -966,8 +708,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val for (i = 0; i < jl_array_nrows(ex->args); i++) { root = jl_array_ptr_ref(ex->args, i); if (jl_needs_lowering(root)) { - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - root = jl_expand_with_loc_warn(root, m, *toplevel_filename, *toplevel_lineno); + root = jl_svecref(jl_lower(root, m, *toplevel_filename, *toplevel_lineno, ~(size_t)0, 1), 0); } ct->world_age = jl_atomic_load_acquire(&jl_world_counter); res = jl_toplevel_eval_flex(m, root, fast, 1, toplevel_filename, toplevel_lineno); @@ -987,7 +728,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val } else if (jl_is_symbol(ex)) { JL_GC_POP(); - return jl_eval_global_var(m, (jl_sym_t*)ex); + return jl_eval_global_var(m, (jl_sym_t*)ex, ct->world_age); } else if (head == NULL) { JL_GC_POP(); @@ -1017,7 +758,7 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val size_t world = jl_atomic_load_acquire(&jl_world_counter); ct->world_age = world; if (!has_defs && jl_get_module_infer(m) != 0) { - (void)jl_type_infer(mfunc, world, SOURCE_MODE_ABI); + (void)jl_type_infer(mfunc, world, SOURCE_MODE_ABI, jl_options.trim); } result = jl_invoke(/*func*/NULL, /*args*/NULL, /*nargs*/0, mfunc); ct->world_age = last_age; @@ -1040,8 +781,8 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_val JL_DLLEXPORT jl_value_t *jl_toplevel_eval(jl_module_t *m, jl_value_t *v) { - const char *filename = jl_filename; - int lineno = jl_lineno; + const char *filename = jl_atomic_load_relaxed(&jl_filename); + int lineno = jl_atomic_load_relaxed(&jl_lineno); return jl_toplevel_eval_flex(m, v, 1, 0, &filename, &lineno); } @@ -1064,7 +805,7 @@ JL_DLLEXPORT void jl_check_top_level_effect(jl_module_t *m, char *fname) } } JL_UNLOCK(&jl_modules_mutex); - if (!open && !jl_is__toplevel__mod(m)) { + if (!open && !jl_is__toplevel__mod(m, jl_current_task)) { const char* name = jl_symbol_name(m->name); jl_errorf("Evaluation into the closed module `%s` breaks incremental compilation " "because the side effects will not be permanent. " @@ -1079,23 +820,23 @@ JL_DLLEXPORT jl_value_t *jl_toplevel_eval_in(jl_module_t *m, jl_value_t *ex) { jl_check_top_level_effect(m, "eval"); jl_value_t *v = NULL; - int last_lineno = jl_lineno; - const char *last_filename = jl_filename; + int last_lineno = jl_atomic_load_relaxed(&jl_lineno); + const char *last_filename = jl_atomic_load_relaxed(&jl_filename); jl_task_t *ct = jl_current_task; - jl_lineno = 1; - jl_filename = "none"; + jl_atomic_store_relaxed(&jl_lineno, 1); + jl_atomic_store_relaxed(&jl_filename, "none"); size_t last_age = ct->world_age; JL_TRY { - ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); v = jl_toplevel_eval(m, ex); } JL_CATCH { - jl_lineno = last_lineno; - jl_filename = last_filename; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); + jl_atomic_store_relaxed(&jl_filename, last_filename); jl_rethrow(); } - jl_lineno = last_lineno; - jl_filename = last_filename; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); + jl_atomic_store_relaxed(&jl_filename, last_filename); ct->world_age = last_age; assert(v); return v; @@ -1127,12 +868,12 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, } jl_task_t *ct = jl_current_task; - int last_lineno = jl_lineno; - const char *last_filename = jl_filename; + int last_lineno = jl_atomic_load_relaxed(&jl_lineno); + const char *last_filename = jl_atomic_load_relaxed(&jl_filename); int lineno = 0; - jl_lineno = 0; + jl_atomic_store_relaxed(&jl_lineno, 0); const char *filename_str = jl_string_data(filename); - jl_filename = filename_str; + jl_atomic_store_relaxed(&jl_filename, filename_str); JL_TRY { size_t last_age = ct->world_age; @@ -1142,29 +883,27 @@ static jl_value_t *jl_parse_eval_all(jl_module_t *module, jl_value_t *text, if (jl_is_linenode(expression)) { // filename is already set above. lineno = jl_linenode_line(expression); - jl_lineno = lineno; + jl_atomic_store_relaxed(&jl_lineno, lineno); continue; } - ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); - expression = jl_expand_with_loc_warn(expression, module, - jl_string_data(filename), lineno); - ct->world_age = jl_atomic_load_relaxed(&jl_world_counter); + expression = jl_svecref(jl_lower(expression, module, jl_string_data(filename), lineno, ~(size_t)0, 1), 0); + ct->world_age = jl_atomic_load_acquire(&jl_world_counter); result = jl_toplevel_eval_flex(module, expression, 1, 1, &filename_str, &lineno); } ct->world_age = last_age; } JL_CATCH { result = jl_box_long(lineno); // (ab)use result to root error line - jl_lineno = last_lineno; - jl_filename = last_filename; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); + jl_atomic_store_relaxed(&jl_filename, last_filename); if (jl_loaderror_type == NULL) jl_rethrow(); else jl_rethrow_other(jl_new_struct(jl_loaderror_type, filename, result, jl_current_exception(ct))); } - jl_lineno = last_lineno; - jl_filename = last_filename; + jl_atomic_store_relaxed(&jl_lineno, last_lineno); + jl_atomic_store_relaxed(&jl_filename, last_filename); JL_GC_POP(); return result; } diff --git a/src/typemap.c b/src/typemap.c index b8b699e101fe5..4a77af32afd13 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -23,29 +23,29 @@ static int jl_is_any(jl_value_t *t1) return t1 == (jl_value_t*)jl_any_type; } -static jl_value_t *jl_type_extract_name(jl_value_t *t1 JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +static jl_value_t *jl_type_extract_name(jl_value_t *t1 JL_PROPAGATES_ROOT, int invariant) JL_NOTSAFEPOINT { if (jl_is_unionall(t1)) t1 = jl_unwrap_unionall(t1); if (jl_is_vararg(t1)) { - return jl_type_extract_name(jl_unwrap_vararg(t1)); + return jl_type_extract_name(jl_unwrap_vararg(t1), invariant); } else if (jl_is_typevar(t1)) { - return jl_type_extract_name(((jl_tvar_t*)t1)->ub); + return jl_type_extract_name(((jl_tvar_t*)t1)->ub, 0); } else if (t1 == jl_bottom_type || t1 == (jl_value_t*)jl_typeofbottom_type || t1 == (jl_value_t*)jl_typeofbottom_type->super) { return (jl_value_t*)jl_typeofbottom_type->name; // put Union{} and typeof(Union{}) and Type{Union{}} together for convenience } else if (jl_is_datatype(t1)) { jl_datatype_t *dt = (jl_datatype_t*)t1; - if (!jl_is_kind(t1)) - return (jl_value_t*)dt->name; - return NULL; + if (jl_is_kind(t1) && !invariant) + return (jl_value_t*)jl_type_typename; + return (jl_value_t*)dt->name; } else if (jl_is_uniontype(t1)) { jl_uniontype_t *u1 = (jl_uniontype_t*)t1; - jl_value_t *tn1 = jl_type_extract_name(u1->a); - jl_value_t *tn2 = jl_type_extract_name(u1->b); + jl_value_t *tn1 = jl_type_extract_name(u1->a, invariant); + jl_value_t *tn2 = jl_type_extract_name(u1->b, invariant); if (tn1 == tn2) return tn1; // TODO: if invariant is false, instead find the nearest common ancestor @@ -71,7 +71,7 @@ static int jl_type_extract_name_precise(jl_value_t *t1, int invariant) } else if (jl_is_datatype(t1)) { jl_datatype_t *dt = (jl_datatype_t*)t1; - if ((invariant || !dt->name->abstract) && !jl_is_kind(t1)) + if (invariant || !dt->name->abstract || dt->name == jl_type_typename) return 1; return 0; } @@ -81,8 +81,8 @@ static int jl_type_extract_name_precise(jl_value_t *t1, int invariant) return 0; if (!jl_type_extract_name_precise(u1->b, invariant)) return 0; - jl_value_t *tn1 = jl_type_extract_name(u1->a); - jl_value_t *tn2 = jl_type_extract_name(u1->b); + jl_value_t *tn1 = jl_type_extract_name(u1->a, invariant); + jl_value_t *tn2 = jl_type_extract_name(u1->b, invariant); if (tn1 == tn2) return 1; return 0; @@ -286,20 +286,20 @@ static _Atomic(jl_value_t*) *mtcache_hash_lookup_bp(jl_genericmemory_t *cache JL return pml; } -static void mtcache_hash_insert(_Atomic(jl_genericmemory_t*) *cache, jl_value_t *parent, jl_value_t *key, jl_typemap_t *val) +static void mtcache_hash_insert(_Atomic(jl_genericmemory_t*) *pcache, jl_value_t *parent, jl_value_t *key, jl_typemap_t *val) { int inserted = 0; - jl_genericmemory_t *a = jl_atomic_load_relaxed(cache); + jl_genericmemory_t *a = jl_atomic_load_relaxed(pcache); if (a == (jl_genericmemory_t*)jl_an_empty_memory_any) { a = jl_alloc_memory_any(16); - jl_atomic_store_release(cache, a); + jl_atomic_store_release(pcache, a); if (parent) jl_gc_wb(parent, a); } a = jl_eqtable_put(a, key, val, &inserted); assert(inserted); - if (a != jl_atomic_load_relaxed(cache)) { - jl_atomic_store_release(cache, a); + if (a != jl_atomic_load_relaxed(pcache)) { + jl_atomic_store_release(pcache, a); if (parent) jl_gc_wb(parent, a); } @@ -469,7 +469,7 @@ static int jl_typemap_intersection_memory_visitor(jl_genericmemory_t *a, jl_valu tydt = (jl_datatype_t*)ttype; } else if (ttype) { - ttype = jl_type_extract_name(ttype); + ttype = jl_type_extract_name(ttype, tparam & 1); tydt = ttype ? (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)ttype)->wrapper) : NULL; } if (tydt == jl_any_type) @@ -569,10 +569,8 @@ int has_covariant_var(jl_datatype_t *ttypes, jl_tvar_t *tv) void typemap_slurp_search(jl_typemap_entry_t *ml, struct typemap_intersection_env *closure) { - // n.b. we could consider mt->max_args here too, so this optimization - // usually works even if the user forgets the `slurp...` argument, but - // there is discussion that parameter may be going away? (and it is - // already not accurately up-to-date for all tables currently anyways) + // TODO: we should consider nparams(closure->type) here too, so this optimization + // usually works even if the user forgets the `slurp...` argument if (closure->search_slurp && ml->va) { jl_value_t *sig = jl_unwrap_unionall((jl_value_t*)ml->sig); size_t nargs = jl_nparams(sig); @@ -641,7 +639,7 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, if (maybe_type && !maybe_kind) { typetype = jl_unwrap_unionall(ty); typetype = jl_is_type_type(typetype) ? jl_tparam0(typetype) : NULL; - name = typetype ? jl_type_extract_name(typetype) : NULL; + name = typetype ? jl_type_extract_name(typetype, 1) : NULL; if (!typetype) exclude_typeofbottom = !jl_subtype((jl_value_t*)jl_typeofbottom_type, ty); else if (jl_is_typevar(typetype)) @@ -717,7 +715,7 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, } } else { - jl_value_t *name = jl_type_extract_name(ty); + jl_value_t *name = jl_type_extract_name(ty, 0); if (name && jl_type_extract_name_precise(ty, 0)) { // direct lookup of leaf types jl_value_t *ml = mtcache_hash_lookup(cachearg1, name); @@ -782,7 +780,7 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, } jl_genericmemory_t *name1 = jl_atomic_load_relaxed(&cache->name1); if (name1 != (jl_genericmemory_t*)jl_an_empty_memory_any) { - jl_value_t *name = jl_type_extract_name(ty); + jl_value_t *name = jl_type_extract_name(ty, 0); if (name && jl_type_extract_name_precise(ty, 0)) { jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)name)->wrapper); // direct lookup of concrete types @@ -1003,7 +1001,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( // now look at the optimized TypeName caches jl_genericmemory_t *tname = jl_atomic_load_relaxed(&cache->tname); if (tname != (jl_genericmemory_t*)jl_an_empty_memory_any) { - jl_value_t *a0 = ty && jl_is_type_type(ty) ? jl_type_extract_name(jl_tparam0(ty)) : NULL; + jl_value_t *a0 = ty && jl_is_type_type(ty) ? jl_type_extract_name(jl_tparam0(ty), 1) : NULL; if (a0) { // TODO: if we start analyzing Union types in jl_type_extract_name, then a0 might be over-approximated here, leading us to miss possible subtypes jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper); while (1) { @@ -1042,7 +1040,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( jl_genericmemory_t *name1 = jl_atomic_load_relaxed(&cache->name1); if (name1 != (jl_genericmemory_t*)jl_an_empty_memory_any) { if (ty) { - jl_value_t *a0 = jl_type_extract_name(ty); + jl_value_t *a0 = jl_type_extract_name(ty, 0); if (a0) { // TODO: if we start analyzing Union types in jl_type_extract_name, then a0 might be over-approximated here, leading us to miss possible subtypes jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper); while (1) { @@ -1200,7 +1198,7 @@ jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_v } jl_genericmemory_t *tname = jl_atomic_load_relaxed(&cache->tname); if (jl_is_kind(ty) && tname != (jl_genericmemory_t*)jl_an_empty_memory_any) { - jl_value_t *name = jl_type_extract_name(a1); + jl_value_t *name = jl_type_extract_name(a1, 1); if (name) { if (ty != (jl_value_t*)jl_datatype_type) a1 = jl_unwrap_unionall(((jl_typename_t*)name)->wrapper); @@ -1295,9 +1293,10 @@ static void jl_typemap_memory_insert_( static jl_value_t *jl_method_convert_list_to_cache( jl_typemap_t *map, jl_typemap_entry_t *ml, int8_t tparam, int8_t offs, int8_t doublesplit) { - jl_value_t *cache = doublesplit ? jl_an_empty_memory_any : (jl_value_t*)jl_new_typemap_level(); + _Atomic(jl_genericmemory_t*) dblcache = (jl_genericmemory_t*)jl_an_empty_memory_any; + jl_typemap_level_t *cache = doublesplit ? NULL : jl_new_typemap_level(); jl_typemap_entry_t *next = NULL; - JL_GC_PUSH3(&cache, &next, &ml); + JL_GC_PUSH4(&cache, &dblcache, &next, &ml); while (ml != (void*)jl_nothing) { next = jl_atomic_load_relaxed(&ml->next); jl_atomic_store_relaxed(&ml->next, (jl_typemap_entry_t*)jl_nothing); @@ -1318,14 +1317,14 @@ static jl_value_t *jl_method_convert_list_to_cache( assert(jl_is_type_type(key)); key = jl_tparam0(key); } - jl_typemap_memory_insert_(map, (_Atomic(jl_genericmemory_t*)*)&cache, key, ml, NULL, 0, offs, NULL); + jl_typemap_memory_insert_(map, &dblcache, key, ml, NULL, 0, offs, NULL); } else - jl_typemap_level_insert_(map, (jl_typemap_level_t*)cache, ml, offs); + jl_typemap_level_insert_(map, cache, ml, offs); ml = next; } JL_GC_POP(); - return cache; + return doublesplit ? (jl_value_t*)jl_atomic_load_relaxed(&dblcache) : (jl_value_t*)cache; } static void jl_typemap_list_insert_( @@ -1447,12 +1446,12 @@ static void jl_typemap_level_insert_( jl_value_t *a0; t1 = jl_unwrap_unionall(t1); if (jl_is_type_type(t1)) { - a0 = jl_type_extract_name(jl_tparam0(t1)); + a0 = jl_type_extract_name(jl_tparam0(t1), 1); jl_datatype_t *super = a0 ? (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper) : jl_any_type; jl_typemap_memory_insert_(map, &cache->tname, (jl_value_t*)super->name, newrec, (jl_value_t*)cache, 1, offs, NULL); return; } - a0 = jl_type_extract_name(t1); + a0 = jl_type_extract_name(t1, 0); if (a0 && a0 != (jl_value_t*)jl_any_type->name) { jl_typemap_memory_insert_(map, &cache->name1, a0, newrec, (jl_value_t*)cache, 0, offs, NULL); return; diff --git a/stdlib/Artifacts/src/Artifacts.jl b/stdlib/Artifacts/src/Artifacts.jl index e21db58b9445e..3b20e5feb3da3 100644 --- a/stdlib/Artifacts/src/Artifacts.jl +++ b/stdlib/Artifacts/src/Artifacts.jl @@ -543,6 +543,14 @@ function jointail(dir, tail) end function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, ::Val{LazyArtifacts}) where LazyArtifacts + world = Base._require_world_age[] + if world == typemax(UInt) + world = Base.get_world_counter() + end + return Base.invoke_in_world(world, __artifact_str, __module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, Val(LazyArtifacts))::String +end + +function __artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, ::Val{LazyArtifacts}) where LazyArtifacts pkg = Base.PkgId(__module__) if pkg.uuid !== nothing # Process overrides for this UUID, if we know what it is @@ -562,10 +570,11 @@ function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dic meta = artifact_meta(name, artifact_dict, artifacts_toml; platform) if meta !== nothing && get(meta, "lazy", false) if LazyArtifacts isa Module && isdefined(LazyArtifacts, :ensure_artifact_installed) - if nameof(LazyArtifacts) in (:Pkg, :Artifacts) + if nameof(LazyArtifacts) in (:Pkg, :Artifacts, :PkgArtifacts) Base.depwarn("using Pkg instead of using LazyArtifacts is deprecated", :var"@artifact_str", force=true) end - return jointail(LazyArtifacts.ensure_artifact_installed(string(name), meta, artifacts_toml; platform), path_tail) + path_base = (@invokelatest LazyArtifacts.ensure_artifact_installed(string(name), meta, artifacts_toml; platform))::String + return jointail(path_base, path_tail) end error("Artifact $(repr(name)) is a lazy artifact; package developers must call `using LazyArtifacts` in $(__module__) before using lazy artifacts.") end @@ -697,7 +706,7 @@ macro artifact_str(name, platform=nothing) # Check if the user has provided `LazyArtifacts`, and thus supports lazy artifacts # If not, check to see if `Pkg` or `Pkg.Artifacts` has been imported. LazyArtifacts = nothing - for module_name in (:LazyArtifacts, :Pkg, :Artifacts) + for module_name in (:LazyArtifacts, :Pkg, :Artifacts, :PkgArtifacts) if isdefined(__module__, module_name) LazyArtifacts = GlobalRef(__module__, module_name) break diff --git a/stdlib/Base64/src/encode.jl b/stdlib/Base64/src/encode.jl index 588b49aa28d97..d55421c52ff55 100644 --- a/stdlib/Base64/src/encode.jl +++ b/stdlib/Base64/src/encode.jl @@ -24,7 +24,7 @@ julia> write(iob64_encode, "Hello!") julia> close(iob64_encode); -julia> str = String(take!(io)) +julia> str = takestring!(io) "SGVsbG8h" julia> String(base64decode(str)) @@ -211,6 +211,6 @@ function base64encode(f::Function, args...; context=nothing) f(IOContext(b, context), args...) end close(b) - return String(take!(s)) + return takestring!(s) end base64encode(args...; context=nothing) = base64encode(write, args...; context=context) diff --git a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl index b32a229d653ed..9a0729c50d01f 100644 --- a/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl +++ b/stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl @@ -13,67 +13,123 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libgcc_s_path::String = "" -libgfortran_path::String = "" -libstdcxx_path::String = "" -libgomp_path::String = "" -if Sys.iswindows() - const _libatomic_path = BundledLazyLibraryPath("libatomic-1.dll") - const _libquadmath_path = BundledLazyLibraryPath("libquadmath-0.dll") - if arch(HostPlatform()) == "x86_64" - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s_seh-1.dll") +libatomic_path::String = "" +const libatomic = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libatomic-1.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libatomic.1.dylib") + elseif Sys.isfreebsd() + BundledLazyLibraryPath("libatomic.so.3") + elseif Sys.islinux() + BundledLazyLibraryPath("libatomic.so.1") else - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s_sjlj-1.dll") + error("CompilerSupportLibraries_jll: Library 'libatomic' is not available for $(Sys.KERNEL)") end - const _libgfortran_path = BundledLazyLibraryPath(string("libgfortran-", libgfortran_version(HostPlatform()).major, ".dll")) - const _libstdcxx_path = BundledLazyLibraryPath("libstdc++-6.dll") - const _libgomp_path = BundledLazyLibraryPath("libgomp-1.dll") - const _libssp_path = BundledLazyLibraryPath("libssp-0.dll") -elseif Sys.isapple() - const _libatomic_path = BundledLazyLibraryPath("libatomic.1.dylib") - const _libquadmath_path = BundledLazyLibraryPath("libquadmath.0.dylib") - if arch(HostPlatform()) == "aarch64" || libgfortran_version(HostPlatform()) == v"5" - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s.1.1.dylib") +) + +if Sys.iswindows() || Sys.isapple() || arch(HostPlatform()) ∈ ("x86_64", "i686") + global libquadmath_path::String = "" + const libquadmath = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libquadmath-0.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libquadmath.0.dylib") + elseif (Sys.islinux() || Sys.isfreebsd()) && arch(HostPlatform()) ∈ ("x86_64", "i686") + BundledLazyLibraryPath("libquadmath.so.0") + else + error("CompilerSupportLibraries_jll: Library 'libquadmath' is not available for $(Sys.KERNEL)") + end + ) +end + +libgcc_s_path::String = "" +const libgcc_s = LazyLibrary( + if Sys.iswindows() + if arch(HostPlatform()) == "x86_64" + BundledLazyLibraryPath("libgcc_s_seh-1.dll") + else + BundledLazyLibraryPath("libgcc_s_sjlj-1.dll") + end + elseif Sys.isapple() + if arch(HostPlatform()) == "aarch64" || libgfortran_version(HostPlatform()) == v"5" + BundledLazyLibraryPath("libgcc_s.1.1.dylib") + else + BundledLazyLibraryPath("libgcc_s.1.dylib") + end + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libgcc_s.so.1") else - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s.1.dylib") + error("CompilerSupportLibraries_jll: Library 'libgcc_s' is not available for $(Sys.KERNEL)") end - const _libgfortran_path = BundledLazyLibraryPath(string("libgfortran.", libgfortran_version(HostPlatform()).major, ".dylib")) - const _libstdcxx_path = BundledLazyLibraryPath("libstdc++.6.dylib") - const _libgomp_path = BundledLazyLibraryPath("libgomp.1.dylib") - const _libssp_path = BundledLazyLibraryPath("libssp.0.dylib") -else - if Sys.isfreebsd() - const _libatomic_path = BundledLazyLibraryPath("libatomic.so.3") +) + +libgfortran_path::String = "" +const libgfortran = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath(string("libgfortran-", libgfortran_version(HostPlatform()).major, ".dll")) + elseif Sys.isapple() + BundledLazyLibraryPath(string("libgfortran.", libgfortran_version(HostPlatform()).major, ".dylib")) + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath(string("libgfortran.so.", libgfortran_version(HostPlatform()).major)) else - const _libatomic_path = BundledLazyLibraryPath("libatomic.so.1") - end - const _libgcc_s_path = BundledLazyLibraryPath("libgcc_s.so.1") - const _libgfortran_path = BundledLazyLibraryPath(string("libgfortran.so.", libgfortran_version(HostPlatform()).major)) - const _libstdcxx_path = BundledLazyLibraryPath("libstdc++.so.6") - const _libgomp_path = BundledLazyLibraryPath("libgomp.so.1") - if libc(HostPlatform()) != "musl" - const _libssp_path = BundledLazyLibraryPath("libssp.so.0") + error("CompilerSupportLibraries_jll: Library 'libgfortran' is not available for $(Sys.KERNEL)") + end; + dependencies = @static if @isdefined(libquadmath) + LazyLibrary[libgcc_s, libquadmath] + else + LazyLibrary[libgcc_s] end - if arch(HostPlatform()) ∈ ("x86_64", "i686") - const _libquadmath_path = BundledLazyLibraryPath("libquadmath.so.0") +) + +libstdcxx_path::String = "" +const libstdcxx = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libstdc++-6.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libstdc++.6.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libstdc++.so.6") + else + error("CompilerSupportLibraries_jll: Library 'libstdcxx' is not available for $(Sys.KERNEL)") + end; + dependencies = LazyLibrary[libgcc_s] +) + +libgomp_path::String = "" +const libgomp = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libgomp-1.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libgomp.1.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libgomp.so.1") + else + error("CompilerSupportLibraries_jll: Library 'libgomp' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + LazyLibrary[libgcc_s] + else + LazyLibrary[] end -end +) -if @isdefined(_libatomic_path) - const libatomic = LazyLibrary(_libatomic_path) -end -const libgcc_s = LazyLibrary(_libgcc_s_path) -libgfortran_deps = [libgcc_s] -if @isdefined _libquadmath_path - const libquadmath = LazyLibrary(_libquadmath_path) - push!(libgfortran_deps, libquadmath) -end -const libgfortran = LazyLibrary(_libgfortran_path, dependencies=libgfortran_deps) -const libstdcxx = LazyLibrary(_libstdcxx_path, dependencies=[libgcc_s]) -const libgomp = LazyLibrary(_libgomp_path) -if @isdefined _libssp_path - const libssp = LazyLibrary(_libssp_path) +# only define if isfile +let + if Sys.iswindows() || Sys.isapple() || libc(HostPlatform()) != "musl" + _libssp_path = if Sys.iswindows() + BundledLazyLibraryPath("libssp-0.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libssp.0.dylib") + elseif Sys.islinux() && libc(HostPlatform()) != "musl" + BundledLazyLibraryPath("libssp.so.0") + end + if isfile(string(_libssp_path)) + global libssp_path::String = "" + @eval const libssp = LazyLibrary($(_libssp_path)) + end + end end # Conform to LazyJLLWrappers API @@ -95,22 +151,25 @@ end is_available() = true function __init__() - if @isdefined _libatomic_path - global libatomic_path = string(_libatomic_path) + global libatomic_path = string(libatomic.path) + global libgcc_s_path = string(libgcc_s.path) + global libgomp_path = string(libgomp.path) + if @isdefined libquadmath_path + global libquadmath_path = string(libquadmath.path) end - global libgcc_s_path = string(_libgcc_s_path) - global libgomp_path = string(_libgomp_path) - if @isdefined _libquadmath_path - global libquadmath_path = string(_libquadmath_path) + if @isdefined libssp_path + global libssp_path = string(libssp.path) end - if @isdefined _libssp_path - global libssp_path = string(_libssp_path) - end - global libgfortran_path = string(_libgfortran_path) - global libstdcxx_path = string(_libstdcxx_path) + global libgfortran_path = string(libgfortran.path) + global libstdcxx_path = string(libstdcxx.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libgcc_s_path) push!(LIBPATH_list, LIBPATH[]) end +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end + end # module CompilerSupportLibraries_jll diff --git a/stdlib/Dates/src/Dates.jl b/stdlib/Dates/src/Dates.jl index a4600a5f82043..e63ec0ec2a487 100644 --- a/stdlib/Dates/src/Dates.jl +++ b/stdlib/Dates/src/Dates.jl @@ -5,7 +5,7 @@ The `Dates` module provides `Date`, `DateTime`, `Time` types, and related functions. -The types are not aware of time zones, based on UT seconds +The types are not aware of time zones, are based on UT seconds (86400 seconds a day, avoiding leap seconds), and use the proleptic Gregorian calendar, as specified in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601). For time zone functionality, see the TimeZones.jl package. @@ -81,4 +81,6 @@ export Period, DatePeriod, TimePeriod, # io.jl ISODateTimeFormat, ISODateFormat, ISOTimeFormat, DateFormat, RFC1123Format, @dateformat_str +public format + end # module diff --git a/stdlib/Dates/src/conversions.jl b/stdlib/Dates/src/conversions.jl index edb521d31f831..65df4c06b64db 100644 --- a/stdlib/Dates/src/conversions.jl +++ b/stdlib/Dates/src/conversions.jl @@ -85,7 +85,7 @@ Return a `DateTime` corresponding to the user's system time as UTC/GMT. For other time zones, see the TimeZones.jl package. # Examples -```julia +```jldoctest; filter = r"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d{3})?" => "2023-01-04T10:52:24.864" julia> now(UTC) 2023-01-04T10:52:24.864 ``` diff --git a/stdlib/Dates/src/periods.jl b/stdlib/Dates/src/periods.jl index da4cb495c5d94..42072f329ba11 100644 --- a/stdlib/Dates/src/periods.jl +++ b/stdlib/Dates/src/periods.jl @@ -122,7 +122,7 @@ coarserperiod(::Type{Month}) = (Year, 12) CompoundPeriod A `CompoundPeriod` is useful for expressing time periods that are not a fixed multiple of -smaller periods. For example, "a year and a day" is not a fixed number of days, but can +smaller periods. For example, "a year and a day" is not a fixed number of days, but can be expressed using a `CompoundPeriod`. In fact, a `CompoundPeriod` is automatically generated by addition of different period types, e.g. `Year(1) + Day(1)` produces a `CompoundPeriod` result. diff --git a/stdlib/DelimitedFiles.version b/stdlib/DelimitedFiles.version index d741690a96838..c3bd5449d11f4 100644 --- a/stdlib/DelimitedFiles.version +++ b/stdlib/DelimitedFiles.version @@ -1,4 +1,4 @@ DELIMITEDFILES_BRANCH = main -DELIMITEDFILES_SHA1 = db79c842f95f55b1f8d8037c0d3363ab21cd3b90 +DELIMITEDFILES_SHA1 = a982d5cf46061593c98b4d80fcc64dd42e6cba74 DELIMITEDFILES_GIT_URL := https://github.com/JuliaData/DelimitedFiles.jl.git DELIMITEDFILES_TAR_URL = https://api.github.com/repos/JuliaData/DelimitedFiles.jl/tarball/$1 diff --git a/stdlib/Distributed.version b/stdlib/Distributed.version index be52c0d684b50..99b5238a55491 100644 --- a/stdlib/Distributed.version +++ b/stdlib/Distributed.version @@ -1,4 +1,4 @@ DISTRIBUTED_BRANCH = master -DISTRIBUTED_SHA1 = 51e52978481835413d15b589919aba80dd85f890 +DISTRIBUTED_SHA1 = 3679026d7b510befdedfa8c6497e3cb032f9cea1 DISTRIBUTED_GIT_URL := https://github.com/JuliaLang/Distributed.jl DISTRIBUTED_TAR_URL = https://api.github.com/repos/JuliaLang/Distributed.jl/tarball/$1 diff --git a/stdlib/Downloads.version b/stdlib/Downloads.version index 16675209860f4..4829d63b31668 100644 --- a/stdlib/Downloads.version +++ b/stdlib/Downloads.version @@ -1,4 +1,4 @@ DOWNLOADS_BRANCH = master -DOWNLOADS_SHA1 = 1199e9039dcce6072601c3f29cea8da4a0acbff6 +DOWNLOADS_SHA1 = 06916258c3ff7bd37a0ff8b525f2bb58ce1ba1b3 DOWNLOADS_GIT_URL := https://github.com/JuliaLang/Downloads.jl.git DOWNLOADS_TAR_URL = https://api.github.com/repos/JuliaLang/Downloads.jl/tarball/$1 diff --git a/stdlib/FileWatching/src/FileWatching.jl b/stdlib/FileWatching/src/FileWatching.jl index 7c743ce634193..ddaf36dfd33a4 100644 --- a/stdlib/FileWatching/src/FileWatching.jl +++ b/stdlib/FileWatching/src/FileWatching.jl @@ -488,12 +488,11 @@ end function getproperty(fdw::FDWatcher, s::Symbol) # support deprecated field names - s === :readable && return fdw.mask.readable - s === :writable && return fdw.mask.writable + s === :readable && return getfield(fdw, :mask).readable + s === :writable && return getfield(fdw, :mask).writable return getfield(fdw, s) end - close(t::_FDWatcher, mask::FDEvent) = close(t, mask.readable, mask.writable) function close(t::_FDWatcher, readable::Bool, writable::Bool) iolock_begin() @@ -517,17 +516,19 @@ end function uvfinalize(uv::Union{FileMonitor, FolderMonitor}) iolock_begin() - if uv.handle != C_NULL - disassociate_julia_struct(uv) # close (and free) without notify - ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), uv.handle) + handle = @atomicswap :monotonic uv.handle = C_NULL + if handle != C_NULL + disassociate_julia_struct(handle) # close (and free) without notify + ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), handle) end iolock_end() end function close(t::Union{FileMonitor, FolderMonitor}) iolock_begin() - if t.handle != C_NULL - ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), t.handle) + handle = t.handle + if handle != C_NULL + ccall(:jl_close_uv, Cvoid, (Ptr{Cvoid},), handle) end iolock_end() end diff --git a/stdlib/FileWatching/src/pidfile.jl b/stdlib/FileWatching/src/pidfile.jl index 6862aaa9f8453..3a3ac7e754817 100644 --- a/stdlib/FileWatching/src/pidfile.jl +++ b/stdlib/FileWatching/src/pidfile.jl @@ -5,7 +5,8 @@ export mkpidlock, trymkpidlock using Base: IOError, UV_EEXIST, UV_ESRCH, UV_ENOENT, - Process + Process, + unsafe_takestring using Base.Filesystem: File, open, JL_O_CREAT, JL_O_RDWR, JL_O_RDONLY, JL_O_EXCL, @@ -304,12 +305,12 @@ function open_exclusive(path::String; end function _rand_filename(len::Int=4) # modified from Base.Libc - slug = Base.StringVector(len) + slug = Base.StringMemory(len) chars = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" for i = 1:len slug[i] = chars[(Libc.rand() % length(chars)) + 1] end - return String(slug) + return unsafe_takestring(slug) end function tryrmopenfile(path::String) diff --git a/stdlib/GMP_jll/Project.toml b/stdlib/GMP_jll/Project.toml index a31688d0a9c07..c17e5311a7d80 100644 --- a/stdlib/GMP_jll/Project.toml +++ b/stdlib/GMP_jll/Project.toml @@ -4,6 +4,7 @@ version = "6.3.0+2" [deps] Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [compat] diff --git a/stdlib/GMP_jll/src/GMP_jll.jl b/stdlib/GMP_jll/src/GMP_jll.jl index ae8b3c0b3e7d5..12a3fc21bd893 100644 --- a/stdlib/GMP_jll/src/GMP_jll.jl +++ b/stdlib/GMP_jll/src/GMP_jll.jl @@ -3,50 +3,68 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/GMP_jll.jl baremodule GMP_jll using Base, Libdl - -const PATH_list = String[] -const LIBPATH_list = String[] +if !Sys.isapple() + using CompilerSupportLibraries_jll +end export libgmp, libgmpxx # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libgmp_handle::Ptr{Cvoid} = C_NULL + libgmp_path::String = "" -libgmpxx_handle::Ptr{Cvoid} = C_NULL +const libgmp = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libgmp-10.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libgmp.10.dylib") + else + BundledLazyLibraryPath("libgmp.so.10") + end +) + libgmpxx_path::String = "" +const libgmpxx = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libgmpxx-4.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libgmpxx.4.dylib") + else + BundledLazyLibraryPath("libgmpxx.so.4") + end, + dependencies = if Sys.isfreebsd() + LazyLibrary[libgmp, libgcc_s] + elseif Sys.isapple() + LazyLibrary[libgmp] + else + LazyLibrary[libgmp, libstdcxx, libgcc_s] + end +) -if Sys.iswindows() - const libgmp = "libgmp-10.dll" - const libgmpxx = "libgmpxx-4.dll" -elseif Sys.isapple() - const libgmp = "@rpath/libgmp.10.dylib" - const libgmpxx = "@rpath/libgmpxx.4.dylib" -else - const libgmp = "libgmp.so.10" - const libgmpxx = "libgmpxx.so.4" +function eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + dlopen(libgmp) + dlopen(libgmpxx) end +is_available() = true function __init__() - global libgmp_handle = dlopen(libgmp) - global libgmp_path = dlpath(libgmp_handle) - global libgmpxx_handle = dlopen(libgmpxx) - global libgmpxx_path = dlpath(libgmpxx_handle) + global libgmp_path = string(libgmp.path) + global libgmpxx_path = string(libgmpxx.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libgmp_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libgmp_path() = libgmp_path -get_libgmpxx_path() = libgmpxx_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module GMP_jll diff --git a/stdlib/GMP_jll/test/runtests.jl b/stdlib/GMP_jll/test/runtests.jl index b2b35b98cbe17..ad0e2dc8a4944 100644 --- a/stdlib/GMP_jll/test/runtests.jl +++ b/stdlib/GMP_jll/test/runtests.jl @@ -3,6 +3,6 @@ using Test, Libdl, GMP_jll @testset "GMP_jll" begin - vn = VersionNumber(unsafe_string(unsafe_load(cglobal((:__gmp_version, libgmp), Ptr{Cchar})))) + vn = VersionNumber(unsafe_string(unsafe_load(cglobal(dlsym(libgmp, :__gmp_version), Ptr{Cchar})))) @test vn == v"6.3.0" end diff --git a/stdlib/InteractiveUtils/src/InteractiveUtils.jl b/stdlib/InteractiveUtils/src/InteractiveUtils.jl index 6b75a228b2761..874e8d16490ae 100644 --- a/stdlib/InteractiveUtils/src/InteractiveUtils.jl +++ b/stdlib/InteractiveUtils/src/InteractiveUtils.jl @@ -53,7 +53,7 @@ function varinfo(m::Module=Base.active_module(), pattern::Regex=r""; all::Bool = if !isdefined(m2, v) || !occursin(pattern, string(v)) continue end - value = getfield(m2, v) + value = getglobal(m2, v) isbuiltin = value === Base || value === Base.active_module() || value === Core if recursive && !isbuiltin && isa(value, Module) && value !== m2 && nameof(value) === v && parentmodule(value) === m2 push!(workqueue, (value, "$prep$v.")) @@ -120,7 +120,7 @@ function versioninfo(io::IO=stdout; verbose::Bool=false) Note: This is an unofficial build, please report bugs to the project responsible for this build and not to the Julia project unless you can - reproduce the issue using official builds available at https://julialang.org/downloads + reproduce the issue using official builds available at https://julialang.org """ ) end @@ -148,7 +148,7 @@ function versioninfo(io::IO=stdout; verbose::Bool=false) if verbose cpuio = IOBuffer() # print cpu_summary with correct alignment Sys.cpu_summary(cpuio) - for (i, line) in enumerate(split(chomp(String(take!(cpuio))), "\n")) + for (i, line) in enumerate(split(chomp(takestring!(cpuio)), "\n")) prefix = i == 1 ? " CPU: " : " " println(io, prefix, line) end @@ -200,7 +200,7 @@ end # `methodswith` -- shows a list of methods using the type given """ - methodswith(typ[, module or function]; supertypes::Bool=false]) + methodswith(typ[, module or function]; supertypes::Bool=false) Return an array of methods with an argument of type `typ`. @@ -232,8 +232,8 @@ end function _methodswith(@nospecialize(t::Type), m::Module, supertypes::Bool) meths = Method[] for nm in names(m) - if isdefined(m, nm) - f = getfield(m, nm) + if isdefinedglobal(m, nm) + f = getglobal(m, nm) if isa(f, Base.Callable) methodswith(t, f, meths; supertypes = supertypes) end @@ -264,8 +264,8 @@ function _subtypes_in!(mods::Array, x::Type) m = pop!(mods) xt = xt::DataType for s in names(m, all = true) - if !isdeprecated(m, s) && isdefined(m, s) - t = getfield(m, s) + if !isdeprecated(m, s) && isdefinedglobal(m, s) + t = getglobal(m, s) dt = isa(t, UnionAll) ? unwrap_unionall(t) : t if isa(dt, DataType) if dt.name.name === s && dt.name.module == m && supertype(dt).name == xt.name diff --git a/stdlib/InteractiveUtils/src/codeview.jl b/stdlib/InteractiveUtils/src/codeview.jl index 1aa83a19285ff..c8f909096c819 100644 --- a/stdlib/InteractiveUtils/src/codeview.jl +++ b/stdlib/InteractiveUtils/src/codeview.jl @@ -20,6 +20,28 @@ const llstyle = Dict{Symbol, Tuple{Bool, Union{Symbol, Int}}}( :funcname => (false, :light_yellow), ) +struct ArgInfo + oc::Union{Core.OpaqueClosure,Nothing} + tt::Type{<:Tuple} + + # Construct from a function object + argtypes + function ArgInfo(@nospecialize(f), @nospecialize(t)) + if isa(f, Core.Builtin) + throw(ArgumentError("argument is not a generic function")) + elseif f isa Core.OpaqueClosure + return new(f, Base.to_tuple_type(t)) + else + return new(nothing, signature_type(f, t)) + end + end + + # Construct from argtypes (incl. arg0) + function ArgInfo(@nospecialize(argtypes::Union{Tuple,Type{<:Tuple}})) + tt = Base.to_tuple_type(argtypes) + return new(nothing, tt) + end +end + function printstyled_ll(io::IO, x, s::Symbol, trailing_spaces="") printstyled(io, x, bold=llstyle[s][1], color=llstyle[s][2]) print(io, trailing_spaces) @@ -143,7 +165,7 @@ See the [`@code_warntype`](@ref man-code-warntype) section in the Performance Ti See also: [`@code_warntype`](@ref), [`code_typed`](@ref), [`code_lowered`](@ref), [`code_llvm`](@ref), [`code_native`](@ref). """ -function code_warntype(io::IO, @nospecialize(f), @nospecialize(tt=Base.default_tt(f)); +function code_warntype(io::IO, arginfo::ArgInfo; world=Base.get_world_counter(), interp::Base.Compiler.AbstractInterpreter=Base.Compiler.NativeInterpreter(world), debuginfo::Symbol=:default, optimize::Bool=false, kwargs...) @@ -152,13 +174,14 @@ function code_warntype(io::IO, @nospecialize(f), @nospecialize(tt=Base.default_t debuginfo = Base.IRShow.debuginfo(debuginfo) lineprinter = Base.IRShow.__debuginfo[debuginfo] nargs::Int = 0 - if isa(f, Core.OpaqueClosure) - isa(f.source, Method) && (nargs = f.source.nargs) - print_warntype_codeinfo(io, Base.code_typed_opaque_closure(f, tt)[1]..., nargs; + if arginfo.oc !== nothing + (; oc, tt) = arginfo + isa(oc.source, Method) && (nargs = oc.source.nargs) + print_warntype_codeinfo(io, Base.code_typed_opaque_closure(oc, tt)[1]..., nargs; lineprinter, label_dynamic_calls = optimize) return nothing end - tt = Base.signature_type(f, tt) + tt = arginfo.tt matches = findall(tt, Base.Compiler.method_table(interp)) matches === nothing && Base.raise_match_failure(:code_warntype, tt) for match in matches.matches @@ -176,6 +199,8 @@ function code_warntype(io::IO, @nospecialize(f), @nospecialize(tt=Base.default_t end nothing end +code_warntype(io::IO, @nospecialize(f), @nospecialize(tt=Base.default_tt(f)); kwargs...) = code_warntype(io, ArgInfo(f, tt); kwargs...) +code_warntype(io::IO, @nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) = code_warntype(io, ArgInfo(argtypes); kwargs...) code_warntype(args...; kwargs...) = (@nospecialize; code_warntype(stdout, args...; kwargs...)) using Base: CodegenParams @@ -189,33 +214,30 @@ const OC_MISMATCH_WARNING = # Printing code representations in IR and assembly -function _dump_function(@nospecialize(f), @nospecialize(t), native::Bool, wrapper::Bool, +function _dump_function(arginfo::ArgInfo, native::Bool, wrapper::Bool, raw::Bool, dump_module::Bool, syntax::Symbol, optimize::Bool, debuginfo::Symbol, binary::Bool, params::CodegenParams=CodegenParams(debug_info_kind=Cint(0), debug_info_level=Cint(2), safepoint_on_entry=raw, gcstack_arg=raw)) ccall(:jl_is_in_pure_context, Bool, ()) && error("code reflection cannot be used from generated functions") - if isa(f, Core.Builtin) - throw(ArgumentError("argument is not a generic function")) - end warning = "" # get the MethodInstance for the method match - if !isa(f, Core.OpaqueClosure) + if arginfo.oc === nothing world = Base.get_world_counter() - match = Base._which(signature_type(f, t); world) + match = Base._which(arginfo.tt; world) mi = Base.specialize_method(match) # TODO: use jl_is_cacheable_sig instead of isdispatchtuple isdispatchtuple(mi.specTypes) || (warning = GENERIC_SIG_WARNING) else - world = UInt64(f.world) - tt = Base.to_tuple_type(t) - if !isdefined(f.source, :source) + (; oc, tt) = arginfo + world = UInt64(oc.world) + if !isdefined(oc.source, :source) # OC was constructed from inferred source. There's only one # specialization and we can't infer anything more precise either. - world = f.source.primary_world - mi = f.source.specializations::Core.MethodInstance - Base.hasintersect(typeof(f).parameters[1], tt) || (warning = OC_MISMATCH_WARNING) + world = oc.source.primary_world + mi = oc.source.specializations::Core.MethodInstance + Base.hasintersect(typeof(oc).parameters[1], tt) || (warning = OC_MISMATCH_WARNING) else - mi = Base.specialize_method(f.source, Tuple{typeof(f.captures), tt.parameters...}, Core.svec()) + mi = Base.specialize_method(oc.source, Tuple{typeof(oc.captures), tt.parameters...}, Core.svec()) isdispatchtuple(mi.specTypes) || (warning = GENERIC_SIG_WARNING) end end @@ -236,19 +258,19 @@ function _dump_function(@nospecialize(f), @nospecialize(t), native::Bool, wrappe end if isempty(str) # if that failed (or we want metadata), use LLVM to generate more accurate assembly output - if !isa(f, Core.OpaqueClosure) + if arginfo.oc === nothing src = Base.Compiler.typeinf_code(Base.Compiler.NativeInterpreter(world), mi, true) else - src, rt = Base.get_oc_code_rt(nothing, f, tt, true) + src, rt = Base.get_oc_code_rt(nothing, arginfo.oc, arginfo.tt, true) end src isa Core.CodeInfo || error("failed to infer source for $mi") str = _dump_function_native_assembly(mi, src, wrapper, syntax, debuginfo, binary, raw, params) end else - if !isa(f, Core.OpaqueClosure) + if arginfo.oc === nothing src = Base.Compiler.typeinf_code(Base.Compiler.NativeInterpreter(world), mi, true) else - src, rt = Base.get_oc_code_rt(nothing, f, tt, true) + src, rt = Base.get_oc_code_rt(nothing, arginfo.oc, arginfo.tt, true) end src isa Core.CodeInfo || error("failed to infer source for $mi") str = _dump_function_llvm(mi, src, wrapper, !raw, dump_module, optimize, debuginfo, params) @@ -311,16 +333,18 @@ Keyword argument `debuginfo` may be one of source (default) or none, to specify See also: [`@code_llvm`](@ref), [`code_warntype`](@ref), [`code_typed`](@ref), [`code_lowered`](@ref), [`code_native`](@ref). """ -function code_llvm(io::IO, @nospecialize(f), @nospecialize(types=Base.default_tt(f)); +function code_llvm(io::IO, arginfo::ArgInfo; raw::Bool=false, dump_module::Bool=false, optimize::Bool=true, debuginfo::Symbol=:default, params::CodegenParams=CodegenParams(debug_info_kind=Cint(0), debug_info_level=Cint(2), safepoint_on_entry=raw, gcstack_arg=raw)) - d = _dump_function(f, types, false, false, raw, dump_module, :intel, optimize, debuginfo, false, params) + d = _dump_function(arginfo, false, false, raw, dump_module, :intel, optimize, debuginfo, false, params) if highlighting[:llvm] && get(io, :color, false)::Bool print_llvm(io, d) else print(io, d) end end +code_llvm(io::IO, @nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) = code_llvm(io, ArgInfo(argtypes); kwargs...) +code_llvm(io::IO, @nospecialize(f), @nospecialize(types=Base.default_tt(f)); kwargs...) = code_llvm(io, ArgInfo(f, types); kwargs...) code_llvm(args...; kwargs...) = (@nospecialize; code_llvm(stdout, args...; kwargs...)) """ @@ -337,17 +361,19 @@ generic function and type signature to `io`. See also: [`@code_native`](@ref), [`code_warntype`](@ref), [`code_typed`](@ref), [`code_lowered`](@ref), [`code_llvm`](@ref). """ -function code_native(io::IO, @nospecialize(f), @nospecialize(types=Base.default_tt(f)); +function code_native(io::IO, arginfo::ArgInfo; dump_module::Bool=true, syntax::Symbol=:intel, raw::Bool=false, debuginfo::Symbol=:default, binary::Bool=false, params::CodegenParams=CodegenParams(debug_info_kind=Cint(0), debug_info_level=Cint(2), safepoint_on_entry=raw, gcstack_arg=raw)) - d = _dump_function(f, types, true, false, raw, dump_module, syntax, true, debuginfo, binary, params) + d = _dump_function(arginfo, true, false, raw, dump_module, syntax, true, debuginfo, binary, params) if highlighting[:native] && get(io, :color, false)::Bool print_native(io, d) else print(io, d) end end +code_native(io::IO, @nospecialize(argtypes::Union{Tuple,Type{<:Tuple}}); kwargs...) = code_native(io, ArgInfo(argtypes); kwargs...) +code_native(io::IO, @nospecialize(f), @nospecialize(types=Base.default_tt(f)); kwargs...) = code_native(io, ArgInfo(f, types); kwargs...) code_native(args...; kwargs...) = (@nospecialize; code_native(stdout, args...; kwargs...)) ## colorized IR and assembly printing diff --git a/stdlib/InteractiveUtils/src/editless.jl b/stdlib/InteractiveUtils/src/editless.jl index 6d1d75f1072ea..e7ac51a2d2092 100644 --- a/stdlib/InteractiveUtils/src/editless.jl +++ b/stdlib/InteractiveUtils/src/editless.jl @@ -269,7 +269,8 @@ function edit(@nospecialize f) end edit(m::Method) = edit(functionloc(m)...) edit(@nospecialize(f), idx::Integer) = edit(methods(f).ms[idx]) -edit(f, t) = (@nospecialize; edit(functionloc(f, t)...)) +edit(f, t) = (@nospecialize; edit(functionloc(f, t)...)) +edit(@nospecialize argtypes::Union{Tuple, Type{<:Tuple}}) = edit(functionloc(argtypes)...) edit(file::Nothing, line::Integer) = error("could not find source file for function") edit(m::Module) = edit(pathof(m)) diff --git a/stdlib/InteractiveUtils/src/macros.jl b/stdlib/InteractiveUtils/src/macros.jl index 68afc40976275..753e84beda06e 100644 --- a/stdlib/InteractiveUtils/src/macros.jl +++ b/stdlib/InteractiveUtils/src/macros.jl @@ -2,14 +2,57 @@ # macro wrappers for various reflection functions -using Base: typesof, insert!, replace_ref_begin_end!, - infer_return_type, infer_exception_type, infer_effects, code_ircode +using Base: insert!, replace_ref_begin_end!, + infer_return_type, infer_exception_type, infer_effects, code_ircode, isexpr # defined in Base so it's possible to time all imports, including InteractiveUtils and its deps # via. `Base.@time_imports` etc. import Base: @time_imports, @trace_compile, @trace_dispatch -separate_kwargs(args...; kwargs...) = (args, values(kwargs)) +typesof_expr(args::Vector{Any}, where_params::Union{Nothing, Vector{Any}} = nothing) = rewrap_where(:(Tuple{$(get_typeof.(args)...)}), where_params) + +function extract_where_parameters(ex::Expr) + isexpr(ex, :where) || return ex, nothing + ex.args[1], ex.args[2:end] +end + +function rewrap_where(ex::Expr, where_params::Union{Nothing, Vector{Any}}) + isnothing(where_params) && return ex + Expr(:where, ex, esc.(where_params)...) +end + +function get_typeof(@nospecialize ex) + isexpr(ex, :(::), 1) && return esc(ex.args[1]) + isexpr(ex, :(::), 2) && return esc(ex.args[2]) + if isexpr(ex, :..., 1) + splatted = ex.args[1] + isexpr(splatted, :(::), 1) && return Expr(:curly, :Vararg, esc(splatted.args[1])) + return :(Any[Core.Typeof(x) for x in $(esc(splatted))]...) + end + return :(Core.Typeof($(esc(ex)))) +end + +function is_broadcasting_call(ex) + isa(ex, Expr) || return false + # Standard broadcasting: f.(x) + isexpr(ex, :.) && length(ex.args) ≥ 2 && isexpr(ex.args[2], :tuple) && return true + # Infix broadcasting: x .+ y, x .<< y, etc. + if isexpr(ex, :call) + f = ex.args[1] + f == :.. && return false + string(f)[1] == '.' && return true + end + return false +end +is_broadcasting_expr(ex) = is_broadcasting_call(ex) || is_broadcasting_assignment(ex) +function is_broadcasting_assignment(ex) + isa(ex, Expr) || return false + isexpr(ex, :.) && return false + head = string(ex.head) + # x .= y, x .+= y, x .<<= y, etc. + head[begin] == '.' && head[end] == '=' && return true + return false +end """ Transform a dot expression into one where each argument has been replaced by a @@ -17,117 +60,247 @@ variable "xj" (with j an integer from 1 to the returned i). The list `args` contains the original arguments that have been replaced. """ function recursive_dotcalls!(ex, args, i=1) - if !(ex isa Expr) || ((ex.head !== :. || !(ex.args[2] isa Expr)) && - (ex.head !== :call || string(ex.args[1])[1] != '.')) - newarg = Symbol('x', i) - if Meta.isexpr(ex, :...) - push!(args, only(ex.args)) - return Expr(:..., newarg), i+1 + if is_broadcasting_expr(ex) + if is_broadcasting_assignment(ex) + (start, branches) = (1, ex.args) + elseif isexpr(ex, :.) + (start, branches) = (1, ex.args[2].args) + else + (start, branches) = (2, ex.args) + end + for j in start:length(branches)::Int + branch, i = recursive_dotcalls!(branches[j], args, i) + branches[j] = branch + end + return ex, i + elseif isexpr(ex, :parameters) + for j in eachindex(ex.args) + param, i = recursive_dotcalls!(ex.args[j], args, i) + ex.args[j] = param + end + return ex, i + end + newarg = Symbol('x', i) + if isexpr(ex, :...) + newarg = Expr(:..., newarg) + push!(args, only(ex.args)) + elseif isexpr(ex, :kw) + newarg = Expr(:kw, ex.args[1], newarg) + push!(args, ex.args[end]) + else + push!(args, ex) + end + return newarg, i+1 +end + +function extract_farg(@nospecialize arg) + !isexpr(arg, :(::), 1) && return esc(arg) + fT = esc(arg.args[1]) + :($construct_callable($fT)) +end + +function construct_callable(@nospecialize(func::Type)) + # Support function singleton types such as `(::typeof(f))(args...)` + Base.issingletontype(func) && isdefined(func, :instance) && return func.instance + # Don't support type annotations otherwise, we don't want to give wrong answers + # for callables such as `(::Returns{Int})(args...)` where using `Returns{Int}` + # would give us code for the constructor, not for the callable object. + throw(ArgumentError("If a function type is explicitly provided, it must be a singleton whose only instance is the callable object")) +end + +function separate_kwargs(exs::Vector{Any}) + args = [] + kwargs = [] + for ex in exs + if isexpr(ex, :kw) + push!(kwargs, ex) + elseif isexpr(ex, :parameters) + for kw in ex.args + push!(kwargs, kw) + end else push!(args, ex) - return newarg, i+1 end end - (start, branches) = ex.head === :. ? (1, ex.args[2].args) : (2, ex.args) - length_branches = length(branches)::Int - for j in start:length_branches - branch, i = recursive_dotcalls!(branches[j], args, i) - branches[j] = branch + args, kwargs +end + +function are_kwargs_valid(kwargs::Vector{Any}) + for kwarg in kwargs + isexpr(kwarg, :..., 1) && continue + isexpr(kwarg, :kw, 2) && isa(kwarg.args[1], Symbol) && continue + isexpr(kwarg, :(::), 2) && continue + isa(kwarg, Symbol) && continue + return false + end + return true +end + +# Generate an expression that merges `kwargs` onto a single `NamedTuple` +function generate_merged_namedtuple_type(kwargs::Vector{Any}) + nts = Any[] + ntargs = Pair{Symbol, Any}[] + for ex in kwargs + if isexpr(ex, :..., 1) + if !isempty(ntargs) + # Construct a `NamedTuple` containing the previous parameters. + push!(nts, generate_namedtuple_type(ntargs)) + empty!(ntargs) + end + push!(nts, Expr(:call, typeof_nt, esc(ex.args[1]))) + elseif isexpr(ex, :kw, 2) + push!(ntargs, ex.args[1]::Symbol => get_typeof(ex.args[2])) + elseif isexpr(ex, :(::), 2) + push!(ntargs, ex.args[1]::Symbol => get_typeof(ex)) + else + ex::Symbol + push!(ntargs, ex => get_typeof(ex)) + end + end + !isempty(ntargs) && push!(nts, generate_namedtuple_type(ntargs)) + return :($merge_namedtuple_types($(nts...))) +end + +function generate_namedtuple_type(ntargs::Vector{Pair{Symbol, Any}}) + names = Expr(:tuple) + tt = Expr(:curly, :Tuple) + for (name, type) in ntargs + push!(names.args, QuoteNode(name)) + push!(tt.args, type) + end + return :(NamedTuple{$names, $tt}) +end + +typeof_nt(nt::NamedTuple) = typeof(nt) +typeof_nt(nt::Base.Pairs) = typeof(values(nt)) + +function merge_namedtuple_types(nt::Type{<:NamedTuple}, nts::Type{<:NamedTuple}...) + @nospecialize + isempty(nts) && return nt + names = Symbol[] + types = Any[] + for nt in (nt, nts...) + for (name, type) in zip(fieldnames(nt), fieldtypes(nt)) + i = findfirst(==(name), names) + if isnothing(i) + push!(names, name) + push!(types, type) + else + types[i] = type + end + end end - return ex, i + NamedTuple{Tuple(names), Tuple{types...}} end -function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) - if Meta.isexpr(ex0, :ref) +is_code_macro(fcn) = startswith(string(fcn), "code_") + +function gen_call_with_extracted_types(__module__, fcn, ex0, kws = Expr[]; is_source_reflection = !is_code_macro(fcn), supports_binding_reflection = false) + if isexpr(ex0, :ref) ex0 = replace_ref_begin_end!(ex0) end # assignments get bypassed: @edit a = f(x) <=> @edit f(x) if isa(ex0, Expr) && ex0.head == :(=) && isa(ex0.args[1], Symbol) && isempty(kws) - return gen_call_with_extracted_types(__module__, fcn, ex0.args[2]) + return gen_call_with_extracted_types(__module__, fcn, ex0.args[2], kws; is_source_reflection, supports_binding_reflection) + end + where_params = nothing + if isa(ex0, Expr) + ex0, where_params = extract_where_parameters(ex0) end if isa(ex0, Expr) - if ex0.head === :do && Meta.isexpr(get(ex0.args, 1, nothing), :call) + if ex0.head === :do && isexpr(get(ex0.args, 1, nothing), :call) + # Normalize `f(args...) do ... end` calls to `f(do_anonymous_function, args...)` if length(ex0.args) != 2 return Expr(:call, :error, "ill-formed do call") end - i = findlast(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args[1].args) + i = findlast(@nospecialize(a)->(isexpr(a, :kw) || isexpr(a, :parameters)), ex0.args[1].args) args = copy(ex0.args[1].args) insert!(args, (isnothing(i) ? 2 : 1+i::Int), ex0.args[2]) ex0 = Expr(:call, args...) end - if ex0.head === :. || (ex0.head === :call && ex0.args[1] !== :.. && string(ex0.args[1])[1] == '.') - codemacro = startswith(string(fcn), "code_") - if codemacro && (ex0.head === :call || ex0.args[2] isa Expr) - # Manually wrap a dot call in a function - args = Any[] - ex, i = recursive_dotcalls!(copy(ex0), args) - xargs = [Symbol('x', j) for j in 1:i-1] - dotfuncname = gensym("dotfunction") - dotfuncdef = Expr(:local, Expr(:(=), Expr(:call, dotfuncname, xargs...), ex)) - return quote - $(esc(dotfuncdef)) - local args = $typesof($(map(esc, args)...)) - $(fcn)($(esc(dotfuncname)), args; $(kws...)) - end - elseif !codemacro - fully_qualified_symbol = true # of the form A.B.C.D - ex1 = ex0 - while ex1 isa Expr && ex1.head === :. - fully_qualified_symbol = (length(ex1.args) == 2 && - ex1.args[2] isa QuoteNode && - ex1.args[2].value isa Symbol) - fully_qualified_symbol || break - ex1 = ex1.args[1] + if is_broadcasting_expr(ex0) && !is_source_reflection + # Manually wrap top-level broadcasts in a function. + # We don't do that if `fcn` reflects into the source, + # because that destroys provenance information. + args = Any[] + ex, i = recursive_dotcalls!(copy(ex0), args) + xargs = [Symbol('x', j) for j in 1:i-1] + dotfuncname = gensym("dotfunction") + dotfuncdef = :(local $dotfuncname($(xargs...)) = $ex) + return quote + $(esc(dotfuncdef)) + $(gen_call_with_extracted_types(__module__, fcn, :($dotfuncname($(args...))), kws; is_source_reflection, supports_binding_reflection)) + end + elseif isexpr(ex0, :.) && is_source_reflection + # If `ex0` has the form A.B (or some chain A.B.C.D) and `fcn` reflects into the source, + # `A` (or `A.B.C`) may be a module, in which case `fcn` is probably more interested in + # the binding rather than the `getproperty` call. + # If binding reflection is not supported, we generate an error; `getproperty(::Module, field)` + # is not going to be interesting to reflect into, so best to allow future non-breaking support + # for binding reflection in case the macro may eventually support that. + fully_qualified_symbol = true + ex1 = ex0 + while ex1 isa Expr && ex1.head === :. + fully_qualified_symbol = (length(ex1.args) == 2 && + ex1.args[2] isa QuoteNode && + ex1.args[2].value isa Symbol) + fully_qualified_symbol || break + ex1 = ex1.args[1] + end + fully_qualified_symbol &= ex1 isa Symbol + if fully_qualified_symbol || isexpr(ex1, :(::), 1) + call_reflection = :($(fcn)(Base.getproperty, $(typesof_expr(ex0.args, where_params)))) + isexpr(ex0.args[1], :(::), 1) && return call_reflection + if supports_binding_reflection + binding_reflection = :($fcn(arg1, $(ex0.args[2]))) + else + binding_reflection = :(error("expression is not a function call")) end - fully_qualified_symbol &= ex1 isa Symbol - if fully_qualified_symbol - return quote - local arg1 = $(esc(ex0.args[1])) - if isa(arg1, Module) - $(if string(fcn) == "which" - :(which(arg1, $(ex0.args[2]))) - else - :(error("expression is not a function call")) - end) - else - local args = $typesof($(map(esc, ex0.args)...)) - $(fcn)(Base.getproperty, args) - end + return quote + local arg1 = $(esc(ex0.args[1])) + if isa(arg1, Module) + $binding_reflection + else + $call_reflection end - else - return Expr(:call, :error, "dot expressions are not lowered to " - * "a single function call, so @$fcn cannot analyze " - * "them. You may want to use Meta.@lower to identify " - * "which function call to target.") end end end - if any(a->(Meta.isexpr(a, :kw) || Meta.isexpr(a, :parameters)), ex0.args) - return quote - local arg1 = $(esc(ex0.args[1])) - local args, kwargs = $separate_kwargs($(map(esc, ex0.args[2:end])...)) - $(fcn)(Core.kwcall, - Tuple{typeof(kwargs), Core.Typeof(arg1), map(Core.Typeof, args)...}; - $(kws...)) + if is_broadcasting_expr(ex0) + return Expr(:call, :error, "dot expressions are not lowered to " + * "a single function call, so @$fcn cannot analyze " + * "them. You may want to use Meta.@lower to identify " + * "which function call to target.") + end + if any(@nospecialize(a)->(isexpr(a, :kw) || isexpr(a, :parameters)), ex0.args) + args, kwargs = separate_kwargs(ex0.args) + are_kwargs_valid(kwargs) || return quote + error("keyword argument format unrecognized; they must be of the form `x` or `x = `") + $(esc(ex0)) # trigger syntax errors if any end + nt = generate_merged_namedtuple_type(kwargs) + tt = rewrap_where(:(Tuple{$nt, $(get_typeof.(args)...)}), where_params) + return :($(fcn)(Core.kwcall, $tt; $(kws...))) elseif ex0.head === :call + argtypes = Any[get_typeof(arg) for arg in ex0.args[2:end]] if ex0.args[1] === :^ && length(ex0.args) >= 3 && isa(ex0.args[3], Int) - return Expr(:call, fcn, :(Base.literal_pow), - Expr(:call, typesof, esc(ex0.args[1]), esc(ex0.args[2]), - esc(Val(ex0.args[3])))) + farg = :(Base.literal_pow) + pushfirst!(argtypes, :(typeof(^))) + argtypes[3] = :(Val{$(ex0.args[3])}) + else + farg = extract_farg(ex0.args[1]) end - return Expr(:call, fcn, esc(ex0.args[1]), - Expr(:call, typesof, map(esc, ex0.args[2:end])...), - kws...) + tt = rewrap_where(:(Tuple{$(argtypes...)}), where_params) + return Expr(:call, fcn, farg, tt, kws...) elseif ex0.head === :(=) && length(ex0.args) == 2 lhs, rhs = ex0.args if isa(lhs, Expr) if lhs.head === :(.) return Expr(:call, fcn, Base.setproperty!, - Expr(:call, typesof, map(esc, lhs.args)..., esc(rhs)), kws...) + typesof_expr(Any[lhs.args..., rhs], where_params), kws...) elseif lhs.head === :ref return Expr(:call, fcn, Base.setindex!, - Expr(:call, typesof, esc(lhs.args[1]), esc(rhs), map(esc, lhs.args[2:end])...), kws...) + typesof_expr(Any[lhs.args[1], rhs, lhs.args[2:end]...], where_params), kws...) end end elseif ex0.head === :vcat || ex0.head === :typed_vcat @@ -138,23 +311,19 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) f, hf = Base.typed_vcat, Base.typed_hvcat args = ex0.args[2:end] end - if any(a->isa(a,Expr) && a.head === :row, args) + if any(@nospecialize(a)->isa(a,Expr) && a.head === :row, args) rows = Any[ (isa(x,Expr) && x.head === :row ? x.args : Any[x]) for x in args ] lens = map(length, rows) - return Expr(:call, fcn, hf, - Expr(:call, typesof, - (ex0.head === :vcat ? [] : Any[esc(ex0.args[1])])..., - Expr(:tuple, lens...), - map(esc, vcat(rows...))...), kws...) + args = Any[Expr(:tuple, lens...); vcat(rows...)] + ex0.head === :typed_vcat && pushfirst!(args, ex0.args[1]) + return Expr(:call, fcn, hf, typesof_expr(args, where_params), kws...) else - return Expr(:call, fcn, f, - Expr(:call, typesof, map(esc, ex0.args)...), kws...) + return Expr(:call, fcn, f, typesof_expr(ex0.args, where_params), kws...) end else for (head, f) in (:ref => Base.getindex, :hcat => Base.hcat, :(.) => Base.getproperty, :vect => Base.vect, Symbol("'") => Base.adjoint, :typed_hcat => Base.typed_hcat, :string => string) if ex0.head === head - return Expr(:call, fcn, f, - Expr(:call, typesof, map(esc, ex0.args)...), kws...) + return Expr(:call, fcn, f, typesof_expr(ex0.args, where_params), kws...) end end end @@ -170,16 +339,14 @@ function gen_call_with_extracted_types(__module__, fcn, ex0, kws=Expr[]) exret = Expr(:none) if ex.head === :call - if any(e->(isa(e, Expr) && e.head === :(...)), ex0.args) && + if any(@nospecialize(x) -> isexpr(x, :...), ex0.args) && (ex.args[1] === GlobalRef(Core,:_apply_iterate) || ex.args[1] === GlobalRef(Base,:_apply_iterate)) # check for splatting exret = Expr(:call, ex.args[2], fcn, - Expr(:tuple, esc(ex.args[3]), - Expr(:call, typesof, map(esc, ex.args[4:end])...))) + Expr(:tuple, extract_farg(ex.args[3]), typesof_expr(ex.args[4:end], where_params))) else - exret = Expr(:call, fcn, esc(ex.args[1]), - Expr(:call, typesof, map(esc, ex.args[2:end])...), kws...) + exret = Expr(:call, fcn, extract_farg(ex.args[1]), typesof_expr(ex.args[2:end], where_params), kws...) end end if ex.head === :thunk || exret.head === :none @@ -196,7 +363,7 @@ Same behaviour as `gen_call_with_extracted_types` except that keyword arguments of the form "foo=bar" are passed on to the called function as well. The keyword arguments must be given before the mandatory argument. """ -function gen_call_with_extracted_types_and_kwargs(__module__, fcn, ex0) +function gen_call_with_extracted_types_and_kwargs(__module__, fcn, ex0; is_source_reflection = !is_code_macro(fcn), supports_binding_reflection = false) kws = Expr[] arg = ex0[end] # Mandatory argument for i in 1:length(ex0)-1 @@ -210,13 +377,15 @@ function gen_call_with_extracted_types_and_kwargs(__module__, fcn, ex0) return Expr(:call, :error, "@$fcn expects only one non-keyword argument") end end - return gen_call_with_extracted_types(__module__, fcn, arg, kws) + return gen_call_with_extracted_types(__module__, fcn, arg, kws; is_source_reflection, supports_binding_reflection) end for fname in [:which, :less, :edit, :functionloc] @eval begin macro ($fname)(ex0) - gen_call_with_extracted_types(__module__, $(Expr(:quote, fname)), ex0) + gen_call_with_extracted_types(__module__, $(Expr(:quote, fname)), ex0, Expr[]; + is_source_reflection = true, + supports_binding_reflection = $(fname === :which)) end end end @@ -229,13 +398,13 @@ end for fname in [:code_warntype, :code_llvm, :code_native, :infer_return_type, :infer_effects, :infer_exception_type] @eval macro ($fname)(ex0...) - gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0) + gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0; is_source_reflection = false) end end for fname in [:code_typed, :code_lowered, :code_ircode] @eval macro ($fname)(ex0...) - thecall = gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0) + thecall = gen_call_with_extracted_types_and_kwargs(__module__, $(QuoteNode(fname)), ex0; is_source_reflection = false) quote local results = $thecall length(results) == 1 ? results[1] : results @@ -440,8 +609,8 @@ in the current environment. When using `@activate`, additional options for a component may be specified in square brackets `@activate Compiler[:option1, :option]` -Currently `@activate Compiler` is the only available component that may be -activatived. +Currently `Compiler` and `JuliaLowering` are the only available components that +may be activatived. For `@activate Compiler`, the following options are available: 1. `:reflection` - Activate the compiler for reflection purposes only. @@ -460,7 +629,7 @@ For `@activate Compiler`, the following options are available: """ macro activate(what) options = Symbol[] - if Meta.isexpr(what, :ref) + if isexpr(what, :ref) Component = what.args[1] for i = 2:length(what.args) arg = what.args[i] @@ -475,7 +644,7 @@ macro activate(what) if !isa(Component, Symbol) error("Usage Error: Component $Component is not a symbol") end - allowed_components = (:Compiler,) + allowed_components = (:Compiler, :JuliaLowering) if !(Component in allowed_components) error("Usage Error: Component $Component is not recognized. Expected one of $allowed_components") end diff --git a/stdlib/InteractiveUtils/test/runtests.jl b/stdlib/InteractiveUtils/test/runtests.jl index 739ed5fac9ef2..f2c47e6561043 100644 --- a/stdlib/InteractiveUtils/test/runtests.jl +++ b/stdlib/InteractiveUtils/test/runtests.jl @@ -328,23 +328,86 @@ try catch err13464 @test startswith(err13464.msg, "expression is not a function call") end + +@testset "Single-argument forms" begin + a = which(+, (Int, Int)) + b = which((typeof(+), Int, Int)) + c = which(Tuple{typeof(+), Int, Int}) + @test a == b == c + + a = functionloc(+, (Int, Int)) + b = functionloc((typeof(+), Int, Int)) + c = functionloc(Tuple{typeof(+), Int, Int}) + @test a == b == c +end + +# PR 57909 +@testset "Support for type annotations as arguments" begin + @test (@which (::Vector{Int})[::Int]).name === :getindex + @test (@which (::Vector{Int})[::Int] = ::Int).name === :setindex! + @test (@which (::Base.RefValue{Int}).x).name === :getproperty + @test (@which (::Base.RefValue{Int}).x = ::Int).name === :setproperty! + @test (@which (::Float64)^2).name === :literal_pow + @test (@which [::Int]).name === :vect + @test (@which [undef_var::Int]).name === :vect + @test (@which [::Int 2]).name === :hcat + @test (@which [::Int; 2]).name === :vcat + @test (@which Int[::Int 2]).name === :typed_hcat + @test (@which Int[::Int; 2]).name === :typed_vcat + @test (@which [::Int 2;3 (::Int)]).name === :hvcat + @test (@which Int[::Int 2;3 (::Int)]).name === :typed_hvcat + @test (@which (::Vector{Float64})').name === :adjoint + @test (@which "$(::Symbol) is a symbol").sig === Tuple{typeof(string), Vararg{Union{Char, String, Symbol}}} + @test (@which +(some_x::Int, some_y::Float64)).name === :+ + @test (@which +(::Any, ::Any, ::Any, ::Any...)).sig === Tuple{typeof(+), Any, Any, Any, Vararg{Any}} + @test (@which +(::Any, ::Any, ::Any, ::Vararg{Any})).sig === Tuple{typeof(+), Any, Any, Any, Vararg{Any}} + n = length(@code_typed +(::Float64, ::Vararg{Float64})) + @test n ≥ 2 + @test length(@code_typed +(::Float64, ::Float64...)) == n + @test (@which +(1, ::Float64)).sig === Tuple{typeof(+), Number, Number} + @test (@which +((1, 2)...)).name === :+ + @test (@which (::typeof(+))(::Int, ::Float64)).sig === Tuple{typeof(+), Number, Number} + @test (@code_typed .+(::Float64, ::Vector{Float64})) isa Pair + @test (@code_typed .+(::Float64, .*(::Vector{Float64}, ::Int))) isa Pair + @test (@which +(::T, ::T) where {T<:Number}).sig === Tuple{typeof(+), T, T} where {T<:Number} + @test (@which round(::Float64; digits=3)).name === :round + @test (@which round(1.2; digits = ::Int)).name === :round + @test (@which round(1.2; digits::Int)).name === :round + @test (@code_typed round(::T; digits = ::T) where {T<:Float64})[2] === Union{} + @test (@code_typed round(::T; digits = ::T) where {T<:Int})[2] === Float64 + base = 10 + kwargs_1 = (; digits = 3) + kwargs_2 = (; sigdigits = 3) + @test (@which round(1.2; kwargs_1...)).name === :round + @test (@which round(1.2; digits = 1, kwargs_1...)).name === :round + @test (@code_typed round(1.2; digits = ::Float64, kwargs_1...))[2] === Float64 # picks `3::Int` from `kwargs_1` + @test (@code_typed round(1.2; kwargs_1..., digits = ::Float64))[2] === Union{} # picks `::Float64` from parameters + @test (@which round(1.2; digits = ::Float64, kwargs_1...)).name === :round + @test (@which round(1.2; sigdigits = ::Int, kwargs_1...)).name === :round + @test (@which round(1.2; kwargs_1..., kwargs_2..., base)).name === :round + @test (@code_typed optimize=false round.([1.0, 2.0]; digits = ::Int64))[2] == Vector{Float64} + @test (@code_typed optimize=false round.(::Vector{Float64}, base = 2; digits = ::Int64))[2] == Vector{Float64} + @test (@code_typed optimize=false round.(base = ::Int64, ::Vector{Float64}; digits = ::Int64))[2] == Vector{Float64} + @test (@code_typed optimize=false [1, 2] .= ::Int)[2] == Vector{Int} + @test (@code_typed optimize=false ::Vector{Int} .= ::Int)[2] == Vector{Int} + @test (@code_typed optimize=false ::Vector{Float64} .= 1 .+ ::Vector{Int})[2] == Vector{Float64} + @test (@code_typed optimize=false ::Vector{Float64} .= 1 .+ round.(base = ::Int, ::Vector{Int}; digits = 3))[2] == Vector{Float64} +end + module MacroTest -export @macrotest +var"@which" = parentmodule(@__MODULE__).var"@which" macro macrotest(x::Int, y::Symbol) end -macro macrotest(x::Int, y::Int) - nothing #This is here because of #15280 -end +macro macrotest(x::Int, y::Int) end end let - using .MacroTest a = 1 - m = getfield(@__MODULE__, Symbol("@macrotest")) - @test which(m, Tuple{LineNumberNode, Module, Int, Symbol}) == @which @macrotest 1 a - @test which(m, Tuple{LineNumberNode, Module, Int, Int}) == @which @macrotest 1 1 + m = MacroTest.var"@macrotest" + @test which(m, Tuple{LineNumberNode, Module, Int, Symbol}) == @eval MacroTest @which @macrotest 1 a + @test which(m, Tuple{LineNumberNode, Module, Int, Int}) == @eval MacroTest @which @macrotest 1 1 @test first(methods(m, Tuple{LineNumberNode, Module, Int, Int})) == @which MacroTest.@macrotest 1 1 - @test functionloc(@which @macrotest 1 1) == @functionloc @macrotest 1 1 + @test functionloc(@eval MacroTest @which @macrotest 1 1) == @functionloc MacroTest.@macrotest 1 1 end mutable struct A18434 @@ -461,7 +524,22 @@ a14637 = A14637(0) @test (@code_typed optimize=true max.([1,7], UInt.([4])))[2] == Vector{UInt} @test (@code_typed Ref.([1,2])[1].x)[2] == Int @test (@code_typed max.(Ref(true).x))[2] == Bool +@test (@code_typed optimize=false round.([1.0, 2.0]; digits = 3))[2] == Vector{Float64} +@test (@code_typed optimize=false round.([1.0, 2.0], base = 2; digits = 3))[2] == Vector{Float64} +@test (@code_typed optimize=false round.(base = 2, [1.0, 2.0], digits = 3))[2] == Vector{Float64} +@test (@code_typed optimize=false [1, 2] .= 2)[2] == Vector{Int} +@test (@code_typed optimize=false [1, 2] .<<= 2)[2] == Vector{Int} +@test (@code_typed optimize=false [1, 2.0] .= 1 .+ [2, 3])[2] == Vector{Float64} +@test (@code_typed optimize=false [1, 2.0] .= 1 .+ round.(base = 1, [1, 3]; digits = 3))[2] == Vector{Float64} +@test (@code_typed optimize=false [1] .+ [2])[2] == Vector{Int} @test !isempty(@code_typed optimize=false max.(Ref.([5, 6])...)) +expansion = string(@macroexpand @code_typed optimize=false max.(Ref.([5, 6])...)) +@test contains(expansion, "(x1) =") # presence of wrapper function +# Make sure broadcasts in nested arguments are not processed. +v = Any[1] +expansion = string(@macroexpand @code_typed v[1] = rand.(Ref(1))) +@test contains(expansion, "Typeof(rand.(Ref(1)))") +@test !contains(expansion, "(x1) =") # Issue # 45889 @test !isempty(@code_typed 3 .+ 6) @@ -526,7 +604,9 @@ end # module ReflectionTest # Issue #18883, code_llvm/code_native for generated functions @generated f18883() = nothing @test !isempty(sprint(code_llvm, f18883, Tuple{})) +@test !isempty(sprint(code_llvm, (typeof(f18883),))) @test !isempty(sprint(code_native, f18883, Tuple{})) +@test !isempty(sprint(code_native, (typeof(f18883),))) ix86 = r"i[356]86" @@ -574,6 +654,9 @@ end @test_throws err @code_lowered "" @test_throws err @code_lowered 1 @test_throws err @code_lowered 1.0 + + @test_throws "dot expressions are not lowered to a single function call" @which a .= 1 + 2 + @test_throws "invalid keyword argument syntax" @eval @which round(1; digits(3)) end using InteractiveUtils: editor @@ -796,6 +879,27 @@ let # `default_tt` should work with any function with one method end); true) end +let # specifying calls as argtypes (incl. arg0) should be supported + @test (code_warntype(devnull, (typeof(function () + sin(42) + end),)); true) + @test (code_warntype(devnull, (typeof(function (a::Int) + sin(42) + end), Int)); true) + @test (code_llvm(devnull, (typeof(function () + sin(42) + end),)); true) + @test (code_llvm(devnull, (typeof(function (a::Int) + sin(42) + end), Int)); true) + @test (code_native(devnull, (typeof(function () + sin(42) + end),)); true) + @test (code_native(devnull, (typeof(function (a::Int) + sin(42) + end), Int)); true) +end + @testset "code_llvm on opaque_closure" begin let ci = code_typed(+, (Int, Int))[1][1] ir = Core.Compiler.inflate_ir(ci) @@ -826,7 +930,13 @@ end @test Base.infer_return_type(sin, (Int,)) == InteractiveUtils.@infer_return_type sin(42) @test Base.infer_exception_type(sin, (Int,)) == InteractiveUtils.@infer_exception_type sin(42) @test first(InteractiveUtils.@code_ircode sin(42)) isa Core.Compiler.IRCode -@test first(InteractiveUtils.@code_ircode optimize_until="Inlining" sin(42)) isa Core.Compiler.IRCode +@test first(InteractiveUtils.@code_ircode optimize_until="CC: INLINING" sin(42)) isa Core.Compiler.IRCode +# Test.@inferred also uses `gen_call_with_extracted_types` +@test Test.@inferred round(1.2) isa Float64 +@test Test.@inferred round(1.3; digits = 3) isa Float64 +# ensure proper inference of the macro output of `@inferred` +@test Base.infer_return_type(x -> Test.@inferred(round(x)), (Float64,)) === Float64 +@test Base.infer_return_type(x -> Test.@inferred(round(x; digits = 3)), (Float64,)) === Float64 @testset "Docstrings" begin @test isempty(Docs.undocumented_names(InteractiveUtils)) diff --git a/stdlib/JuliaSyntaxHighlighting.version b/stdlib/JuliaSyntaxHighlighting.version index 1c9bfb131dc0f..3a94bcf8c92cd 100644 --- a/stdlib/JuliaSyntaxHighlighting.version +++ b/stdlib/JuliaSyntaxHighlighting.version @@ -1,4 +1,4 @@ JULIASYNTAXHIGHLIGHTING_BRANCH = main -JULIASYNTAXHIGHLIGHTING_SHA1 = b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f +JULIASYNTAXHIGHLIGHTING_SHA1 = b666d3c98cca30d20d1e6f98c0e12c9350ffbc4c JULIASYNTAXHIGHLIGHTING_GIT_URL := https://github.com/julialang/JuliaSyntaxHighlighting.jl.git JULIASYNTAXHIGHLIGHTING_TAR_URL = https://api.github.com/repos/julialang/JuliaSyntaxHighlighting.jl/tarball/$1 diff --git a/stdlib/LLD_jll/Project.toml b/stdlib/LLD_jll/Project.toml index c7041ac4a2577..f1ebf691ff3a9 100644 --- a/stdlib/LLD_jll/Project.toml +++ b/stdlib/LLD_jll/Project.toml @@ -1,6 +1,6 @@ name = "LLD_jll" uuid = "d55e3150-da41-5e91-b323-ecfd1eec6109" -version = "19.1.7+1" +version = "20.1.2+0" [deps] Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" @@ -10,7 +10,7 @@ Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [compat] julia = "1.13" -libLLVM_jll = "19.1.7" +libLLVM_jll = "20.1.2" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl b/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl index 429e35b91d3f2..c6e2750895c13 100644 --- a/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl +++ b/stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl @@ -5,38 +5,33 @@ baremodule LLVMLibUnwind_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export llvmlibunwind # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -llvmlibunwind_handle::Ptr{Cvoid} = C_NULL + llvmlibunwind_path::String = "" +const llvmlibunwind = LazyLibrary(BundledLazyLibraryPath("libunwind")) -const llvmlibunwind = "libunwind" +function eager_mode() + dlopen(llvmlibunwind) +end +is_available() = @static Sys.isapple() ? true : false function __init__() - # We only dlopen something on MacOS - @static if Sys.isapple() - global llvmlibunwind_handle = dlopen(llvmlibunwind) - global llvmlibunwind_path = dlpath(llvmlibunwind_handle) - global artifact_dir = dirname(Sys.BINDIR) - LIBPATH[] = dirname(llvmlibunwind_path) - push!(LIBPATH_list, LIBPATH[]) - end + global llvmlibunwind_path = string(llvmlibunwind.path) + global artifact_dir = dirname(Sys.BINDIR) + LIBPATH[] = dirname(llvmlibunwind_path) + push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = @static Sys.isapple() ? true : false -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_llvmlibunwind_path() = llvmlibunwind_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LLVMLibUnwind_jll diff --git a/stdlib/LLVMLibUnwind_jll/test/runtests.jl b/stdlib/LLVMLibUnwind_jll/test/runtests.jl index e984593ab2c25..42afe50a875f6 100644 --- a/stdlib/LLVMLibUnwind_jll/test/runtests.jl +++ b/stdlib/LLVMLibUnwind_jll/test/runtests.jl @@ -1,16 +1,14 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Test, Libdl -using LLVMLibUnwind_jll: llvmlibunwind_handle - +using Test, Libdl, LLVMLibUnwind_jll @testset "LLVMLibUnwind_jll" begin if Sys.isapple() - @test dlsym(llvmlibunwind_handle, :unw_getcontext; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_init_local; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_init_local_dwarf; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_step; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_get_reg; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_set_reg; throw_error=false) !== nothing - @test dlsym(llvmlibunwind_handle, :unw_resume; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_getcontext; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_init_local; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_init_local_dwarf; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_step; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_get_reg; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_set_reg; throw_error=false) !== nothing + @test dlsym(llvmlibunwind, :unw_resume; throw_error=false) !== nothing end end diff --git a/stdlib/LibCURL.version b/stdlib/LibCURL.version index 216ab4e7aca22..9bf12e7a8287b 100644 --- a/stdlib/LibCURL.version +++ b/stdlib/LibCURL.version @@ -1,4 +1,4 @@ LIBCURL_BRANCH = master -LIBCURL_SHA1 = a65b64f6eabc932f63c2c0a4a5fb5d75f3e688d0 +LIBCURL_SHA1 = 038790a793203248362cf2bd8d85e42f8c56a72d LIBCURL_GIT_URL := https://github.com/JuliaWeb/LibCURL.jl.git LIBCURL_TAR_URL = https://api.github.com/repos/JuliaWeb/LibCURL.jl/tarball/$1 diff --git a/stdlib/LibCURL_jll/Project.toml b/stdlib/LibCURL_jll/Project.toml index 46ad6f18aaafa..45cc2511c21f7 100644 --- a/stdlib/LibCURL_jll/Project.toml +++ b/stdlib/LibCURL_jll/Project.toml @@ -1,14 +1,16 @@ name = "LibCURL_jll" uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.12.1+1" +version = "8.15.0+1" [deps] +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" LibSSH2_jll = "29816b5a-b9ab-546f-933c-edad1886dfa8" -nghttp2_jll = "8e850ede-7688-5339-a07c-302acd2aaf8d" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +nghttp2_jll = "8e850ede-7688-5339-a07c-302acd2aaf8d" +Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" @@ -17,4 +19,5 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" test = ["Test"] [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.11" diff --git a/stdlib/LibCURL_jll/src/LibCURL_jll.jl b/stdlib/LibCURL_jll/src/LibCURL_jll.jl index 5c1c2aa14b23a..528ca92ffb9af 100644 --- a/stdlib/LibCURL_jll/src/LibCURL_jll.jl +++ b/stdlib/LibCURL_jll/src/LibCURL_jll.jl @@ -3,47 +3,70 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/LibCURL_jll.jl baremodule LibCURL_jll -using Base, Libdl, nghttp2_jll, LibSSH2_jll, Zlib_jll -if !(Sys.iswindows() || Sys.isapple()) - # On Windows and macOS we use system SSL/crypto libraries +using Base, Libdl, nghttp2_jll, LibSSH2_jll, Zlib_jll, Zstd_jll +if !Sys.iswindows() using OpenSSL_jll end - -const PATH_list = String[] -const LIBPATH_list = String[] +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end export libcurl # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libcurl_handle::Ptr{Cvoid} = C_NULL + libcurl_path::String = "" +const libcurl = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libcurl-4.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libcurl.4.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcurl.so.4") + else + error("LibCURL_jll: Library 'libcurl' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + if Sys.WORD_SIZE == 32 + LazyLibrary[libz, libzstd, libnghttp2, libssh2, libgcc_s] + else + LazyLibrary[libz, libzstd, libnghttp2, libssh2] + end + else + LazyLibrary[libz, libzstd, libnghttp2, libssh2, libssl, libcrypto] + end +) -if Sys.iswindows() - const libcurl = "libcurl-4.dll" -elseif Sys.isapple() - const libcurl = "@rpath/libcurl.4.dylib" -else - const libcurl = "libcurl.so.4" +function eager_mode() + Zlib_jll.eager_mode() + Zstd_jll.eager_mode() + nghttp2_jll.eager_mode() + LibSSH2_jll.eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + @static if @isdefined OpenSSL_jll + OpenSSL_jll.eager_mode() + end + dlopen(libcurl) end +is_available() = true function __init__() - global libcurl_handle = dlopen(libcurl) - global libcurl_path = dlpath(libcurl_handle) + global libcurl_path = string(libcurl.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libcurl_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libcurl_path() = libcurl_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LibCURL_jll diff --git a/stdlib/LibGit2/src/utils.jl b/stdlib/LibGit2/src/utils.jl index eb8b878d30484..6fc4acbd6bf77 100644 --- a/stdlib/LibGit2/src/utils.jl +++ b/stdlib/LibGit2/src/utils.jl @@ -162,7 +162,7 @@ function git_url(; end seekstart(io) - return String(take!(io)) + return takestring!(io) end function credential_identifier(scheme::AbstractString, host::AbstractString) diff --git a/stdlib/LibGit2_jll/Project.toml b/stdlib/LibGit2_jll/Project.toml index 216fe9c3c6b41..960e0e1d7c16b 100644 --- a/stdlib/LibGit2_jll/Project.toml +++ b/stdlib/LibGit2_jll/Project.toml @@ -1,15 +1,19 @@ name = "LibGit2_jll" uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.0+0" +version = "1.9.1+0" [deps] OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" LibSSH2_jll = "29816b5a-b9ab-546f-933c-edad1886dfa8" +PCRE2_jll = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" [compat] julia = "1.9" +CompilerSupportLibraries_jll = "1.3.0" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/stdlib/LibGit2_jll/src/LibGit2_jll.jl b/stdlib/LibGit2_jll/src/LibGit2_jll.jl index c69deb4a9d932..350ed85a503f7 100644 --- a/stdlib/LibGit2_jll/src/LibGit2_jll.jl +++ b/stdlib/LibGit2_jll/src/LibGit2_jll.jl @@ -3,47 +3,69 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/LibGit2_jll.jl baremodule LibGit2_jll -using Base, Libdl, LibSSH2_jll +using Base, Libdl, LibSSH2_jll, PCRE2_jll, Zlib_jll if !(Sys.iswindows() || Sys.isapple()) - # On Windows and macOS we use system SSL/crypto libraries using OpenSSL_jll end - -const PATH_list = String[] -const LIBPATH_list = String[] +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end export libgit2 # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libgit2_handle::Ptr{Cvoid} = C_NULL + libgit2_path::String = "" +const libgit2 = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libgit2.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libgit2.1.9.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libgit2.so.1.9") + else + error("LibGit2_jll: Library 'libgit2' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + if Sys.WORD_SIZE == 32 + LazyLibrary[libssh2, libgcc_s, libpcre2_8, libz] + else + LazyLibrary[libssh2, libpcre2_8, libz] + end + elseif Sys.isfreebsd() || Sys.islinux() + LazyLibrary[libssh2, libssl, libcrypto, libpcre2_8, libz] + else + LazyLibrary[libssh2, libpcre2_8, libz] + end +) -if Sys.iswindows() - const libgit2 = "libgit2.dll" -elseif Sys.isapple() - const libgit2 = "@rpath/libgit2.1.9.dylib" -else - const libgit2 = "libgit2.so.1.9" +function eager_mode() + LibSSH2_jll.eager_mode() + @static if @isdefined OpenSSL_jll + OpenSSL_jll.eager_mode() + end + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + dlopen(libgit2) end +is_available() = true function __init__() - global libgit2_handle = dlopen(libgit2) - global libgit2_path = dlpath(libgit2_handle) + global libgit2_path = string(libgit2.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libgit2_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libgit2_path() = libgit2_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LibGit2_jll diff --git a/stdlib/LibGit2_jll/test/runtests.jl b/stdlib/LibGit2_jll/test/runtests.jl index 06edefe335a2f..a971c1e8aa402 100644 --- a/stdlib/LibGit2_jll/test/runtests.jl +++ b/stdlib/LibGit2_jll/test/runtests.jl @@ -7,5 +7,5 @@ using Test, Libdl, LibGit2_jll minor = Ref{Cint}(0) patch = Ref{Cint}(0) @test ccall((:git_libgit2_version, libgit2), Cint, (Ref{Cint}, Ref{Cint}, Ref{Cint}), major, minor, patch) == 0 - @test VersionNumber(major[], minor[], patch[]) == v"1.9.0" + @test VersionNumber(major[], minor[], patch[]) == v"1.9.1" end diff --git a/stdlib/LibSSH2_jll/Project.toml b/stdlib/LibSSH2_jll/Project.toml index 09f07b559344c..c4bf18ca39d7c 100644 --- a/stdlib/LibSSH2_jll/Project.toml +++ b/stdlib/LibSSH2_jll/Project.toml @@ -6,9 +6,12 @@ version = "1.11.3+1" OpenSSL_jll = "458c3c95-2e84-50aa-8efc-19380b2a3a95" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" [compat] julia = "1.8" +CompilerSupportLibraries_jll = "1.3.0" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl b/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl index e9392fe34a918..6c273bbdecc4d 100644 --- a/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl +++ b/stdlib/LibSSH2_jll/src/LibSSH2_jll.jl @@ -4,47 +4,73 @@ baremodule LibSSH2_jll using Base, Libdl +if Sys.isfreebsd() || Sys.isapple() + using Zlib_jll +end +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end if !Sys.iswindows() - # On Windows we use system SSL/crypto libraries using OpenSSL_jll end -const PATH_list = String[] -const LIBPATH_list = String[] - export libssh2 # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libssh2_handle::Ptr{Cvoid} = C_NULL + libssh2_path::String = "" +const libssh2 = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libssh2.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libssh2.1.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libssh2.so.1") + else + error("LibSSH2_jll: Library 'libssh2' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + if Sys.WORD_SIZE == 32 + LazyLibrary[libgcc_s] + else + LazyLibrary[] + end + elseif Sys.islinux() + LazyLibrary[libcrypto] + elseif Sys.isfreebsd() || Sys.isapple() + LazyLibrary[libz, libcrypto] + end +) -if Sys.iswindows() - const libssh2 = "libssh2.dll" -elseif Sys.isapple() - const libssh2 = "@rpath/libssh2.1.dylib" -else - const libssh2 = "libssh2.so.1" +function eager_mode() + @static if @isdefined Zlib_jll + Zlib_jll.eager_mode() + end + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + @static if @isdefined OpenSSL_jll + OpenSSL_jll.eager_mode() + end + dlopen(libssh2) end +is_available() = true function __init__() - global libssh2_handle = dlopen(libssh2) - global libssh2_path = dlpath(libssh2_handle) + global libssh2_path = string(libssh2.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libssh2_path) push!(LIBPATH_list, LIBPATH[]) end - -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libssh2_path() = libssh2_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LibSSH2_jll diff --git a/stdlib/LibUV_jll/src/LibUV_jll.jl b/stdlib/LibUV_jll/src/LibUV_jll.jl index febc47f168ab9..5bf9b7ef3b0fb 100644 --- a/stdlib/LibUV_jll/src/LibUV_jll.jl +++ b/stdlib/LibUV_jll/src/LibUV_jll.jl @@ -6,5 +6,6 @@ baremodule LibUV_jll using Base, Libdl # NOTE: This file is currently empty, as we link libuv statically for now. +is_available() = true end # module LibUV_jll diff --git a/stdlib/LibUnwind_jll/Project.toml b/stdlib/LibUnwind_jll/Project.toml index b43f1c537ce5a..3e22155b27530 100644 --- a/stdlib/LibUnwind_jll/Project.toml +++ b/stdlib/LibUnwind_jll/Project.toml @@ -1,10 +1,12 @@ name = "LibUnwind_jll" uuid = "745a5e78-f969-53e9-954f-d19f2f74f4e3" -version = "1.8.1+2" +version = "1.8.2+1" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" [compat] julia = "1.6" diff --git a/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl b/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl index f97b18443b6fd..d396540dbab46 100644 --- a/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl +++ b/stdlib/LibUnwind_jll/src/LibUnwind_jll.jl @@ -4,39 +4,49 @@ baremodule LibUnwind_jll using Base, Libdl - -const PATH_list = String[] -const LIBPATH_list = String[] +using Zlib_jll +if !Sys.isfreebsd() + using CompilerSupportLibraries_jll +end export libunwind # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libunwind_handle::Ptr{Cvoid} = C_NULL + libunwind_path::String = "" +const libunwind = LazyLibrary( + BundledLazyLibraryPath("libunwind.so.8"), + dependencies = if Sys.isfreebsd() + LazyLibrary[libz] + else + LazyLibrary[libgcc_s, libz] + end +) -const libunwind = "libunwind.so.8" +function eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + Zlib_jll.eager_mode() + dlopen(libunwind) +end +is_available() = @static(Sys.islinux() || Sys.isfreebsd()) ? true : false function __init__() - # We only do something on Linux/FreeBSD - @static if Sys.islinux() || Sys.isfreebsd() - global libunwind_handle = dlopen(libunwind) - global libunwind_path = dlpath(libunwind_handle) - global artifact_dir = dirname(Sys.BINDIR) - LIBPATH[] = dirname(libunwind_path) - push!(LIBPATH_list, LIBPATH[]) - end + global libunwind_path = string(libunwind.path) + global artifact_dir = dirname(Sys.BINDIR) + LIBPATH[] = dirname(libunwind_path) + push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = @static (Sys.islinux() || Sys.isfreebsd()) ? true : false -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libunwind_path() = libunwind_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module LibUnwind_jll diff --git a/stdlib/LibUnwind_jll/test/runtests.jl b/stdlib/LibUnwind_jll/test/runtests.jl index 1cb33dd6729e3..c87ccad988dec 100644 --- a/stdlib/LibUnwind_jll/test/runtests.jl +++ b/stdlib/LibUnwind_jll/test/runtests.jl @@ -4,6 +4,6 @@ using Test, Libdl, LibUnwind_jll @testset "LibUnwind_jll" begin if !Sys.isapple() && !Sys.iswindows() - @test dlsym(LibUnwind_jll.libunwind_handle, :unw_backtrace; throw_error=false) !== nothing + @test dlsym(LibUnwind_jll.libunwind, :unw_backtrace; throw_error=false) !== nothing end end diff --git a/stdlib/LinearAlgebra.version b/stdlib/LinearAlgebra.version index 57683cd38f060..dc220efcc37ea 100644 --- a/stdlib/LinearAlgebra.version +++ b/stdlib/LinearAlgebra.version @@ -1,4 +1,4 @@ LINEARALGEBRA_BRANCH = master -LINEARALGEBRA_SHA1 = 1ce842652c07b33289046236b09bc59bad43dbeb +LINEARALGEBRA_SHA1 = 2c3fe9b7e0ca4e2c7bf506bd16ae5900f04a8023 LINEARALGEBRA_GIT_URL := https://github.com/JuliaLang/LinearAlgebra.jl.git LINEARALGEBRA_TAR_URL = https://api.github.com/repos/JuliaLang/LinearAlgebra.jl/tarball/$1 diff --git a/stdlib/Logging/src/Logging.jl b/stdlib/Logging/src/Logging.jl index 192885f2f94b7..ab02073f7b40a 100644 --- a/stdlib/Logging/src/Logging.jl +++ b/stdlib/Logging/src/Logging.jl @@ -8,70 +8,33 @@ and available by default. """ module Logging -# Import the CoreLogging implementation into Logging as new const bindings. -# Doing it this way (rather than with import) makes these symbols accessible to -# tab completion. -for sym in [ - :LogLevel, - :AbstractLogger, - :NullLogger, - :handle_message, :shouldlog, :min_enabled_level, :catch_exceptions, - Symbol("@debug"), - Symbol("@info"), - Symbol("@warn"), - Symbol("@error"), - Symbol("@logmsg"), - :with_logger, - :current_logger, - :global_logger, - :disable_logging, - :SimpleLogger] - @eval const $sym = Base.CoreLogging.$sym -end - -# LogLevel aliases (re-)documented here (JuliaLang/julia#40978) -""" - Debug - -Alias for [`LogLevel(-1000)`](@ref LogLevel). -""" -const Debug = Base.CoreLogging.Debug -""" - Info - -Alias for [`LogLevel(0)`](@ref LogLevel). -""" -const Info = Base.CoreLogging.Info -""" - Warn - -Alias for [`LogLevel(1000)`](@ref LogLevel). -""" -const Warn = Base.CoreLogging.Warn -""" - Error - -Alias for [`LogLevel(2000)`](@ref LogLevel). -""" -const Error = Base.CoreLogging.Error -""" - BelowMinLevel - -Alias for [`LogLevel(-1_000_001)`](@ref LogLevel). -""" -const BelowMinLevel = Base.CoreLogging.BelowMinLevel -""" - AboveMaxLevel - -Alias for [`LogLevel(1_000_001)`](@ref LogLevel). -""" -const AboveMaxLevel = Base.CoreLogging.AboveMaxLevel - -using Base.CoreLogging: - closed_stream, ConsoleLogger, default_metafmt - -# Some packages use `Logging.default_logcolor` -const default_logcolor = Base.CoreLogging.default_logcolor +import Base.CoreLogging: + LogLevel, + AbstractLogger, + NullLogger, + handle_message, shouldlog, min_enabled_level, catch_exceptions, + var"@debug", + var"@info", + var"@warn", + var"@error", + var"@logmsg", + with_logger, + current_logger, + global_logger, + disable_logging, + SimpleLogger, + Debug, + Info, + Warn, + Error, + BelowMinLevel, + AboveMaxLevel, + default_logcolor, + closed_stream, + ConsoleLogger, + default_metafmt, + # Some packages use `Logging.default_logcolor` + default_logcolor export AbstractLogger, @@ -95,12 +58,6 @@ export Error, AboveMaxLevel -# The following are also part of the public API, but not exported: -# -# 1. Log levels: -# BelowMinLevel, Debug, Info, Warn, Error, AboveMaxLevel, -# -# 2. AbstractLogger message related functions: -# handle_message, shouldlog, min_enabled_level, catch_exceptions, +public handle_message, shouldlog, min_enabled_level, catch_exceptions end diff --git a/stdlib/Logging/test/runtests.jl b/stdlib/Logging/test/runtests.jl index 2fedbde557078..6d926f4dd0340 100644 --- a/stdlib/Logging/test/runtests.jl +++ b/stdlib/Logging/test/runtests.jl @@ -306,4 +306,47 @@ end @test isempty(undoc) end +@testset "Logging when multithreaded" begin + n = 10000 + cmd = `$(Base.julia_cmd()) -t4 --color=no $(joinpath(@__DIR__, "threads_exec.jl")) $n` + fname = tempname() + @testset "Thread safety" begin + f = open(fname, "w") + @test success(run(pipeline(cmd, stderr=f))) + close(f) + end + + @testset "No tearing in log printing" begin + # Check for print tearing by verifying that each log entry starts and ends correctly + f = open(fname, "r") + entry_start = r"┌ (Info|Warning|Error): iteration" + entry_end = r"└ " + + open_entries = 0 + total_entries = 0 + for line in eachline(fname) + starts = count(entry_start, line) + starts > 1 && error("Interleaved logs: Multiple log entries started on one line") + if starts == 1 + startswith(line, entry_start) || error("Interleaved logs: Log entry started in the middle of a line") + open_entries += 1 + total_entries += 1 + end + + ends = count(entry_end, line) + starts == 1 && ends == 1 && error("Interleaved logs: Log entry started and another ended on one line") + ends > 1 && error("Interleaved logs: Multiple log entries ended on one line") + if ends == 1 + startswith(line, entry_end) || error("Interleaved logs: Log entry ended in the middle of a line") + open_entries -= 1 + end + # Ensure no mismatched log entries + open_entries >= 0 || error("Interleaved logs") + end + + @test open_entries == 0 # Ensure all entries closed properly + @test total_entries == n * 3 # Ensure all logs were printed (3 because @debug is hidden) + end +end + end diff --git a/stdlib/Logging/test/threads_exec.jl b/stdlib/Logging/test/threads_exec.jl new file mode 100644 index 0000000000000..497a22b1c7b22 --- /dev/null +++ b/stdlib/Logging/test/threads_exec.jl @@ -0,0 +1,13 @@ +using Logging + +function test_threads_exec(n) + Threads.@threads for i in 1:n + @debug "iteration" maxlog=1 _id=Symbol("$(i)_debug") i Threads.threadid() + @info "iteration" maxlog=1 _id=Symbol("$(i)_info") i Threads.threadid() + @warn "iteration" maxlog=1 _id=Symbol("$(i)_warn") i Threads.threadid() + @error "iteration" maxlog=1 _id=Symbol("$(i)_error") i Threads.threadid() + end +end + +n = parse(Int, ARGS[1]) +test_threads_exec(n) diff --git a/stdlib/MPFR_jll/Project.toml b/stdlib/MPFR_jll/Project.toml index 50de38f169ff0..5d1524c1455b0 100644 --- a/stdlib/MPFR_jll/Project.toml +++ b/stdlib/MPFR_jll/Project.toml @@ -1,13 +1,15 @@ name = "MPFR_jll" uuid = "3a97d323-0669-5f0c-9066-3539efd106a3" -version = "4.2.1+2" +version = "4.2.2+0" [deps] +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" GMP_jll = "781609d7-10c4-51f6-84f2-b8444358ff6d" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.6" [extras] diff --git a/stdlib/MPFR_jll/src/MPFR_jll.jl b/stdlib/MPFR_jll/src/MPFR_jll.jl index 219ab0cad41be..b7b379c543c7a 100644 --- a/stdlib/MPFR_jll/src/MPFR_jll.jl +++ b/stdlib/MPFR_jll/src/MPFR_jll.jl @@ -3,42 +3,51 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/MPFR_jll.jl baremodule MPFR_jll using Base, Libdl, GMP_jll - -const PATH_list = String[] -const LIBPATH_list = String[] +if Sys.iswindows() + using CompilerSupportLibraries_jll +end export libmpfr # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libmpfr_handle::Ptr{Cvoid} = C_NULL -libmpfr_path::String = "" -if Sys.iswindows() - const libmpfr = "libmpfr-6.dll" -elseif Sys.isapple() - const libmpfr = "@rpath/libmpfr.6.dylib" -else - const libmpfr = "libmpfr.so.6" +libmpfr_path::String = "" +const libmpfr = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libmpfr-6.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libmpfr.6.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libmpfr.so.6") + else + error("MPFR_jll: Library 'libmpfr' is not available for $(Sys.KERNEL)") + end, + dependencies = if Sys.iswindows() + LazyLibrary[libgmp, libgcc_s] + else + LazyLibrary[libgmp] + end +) + +function eager_mode() + GMP_jll.eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + dlopen(libmpfr) end +is_available() = true function __init__() - global libmpfr_handle = dlopen(libmpfr) - global libmpfr_path = dlpath(libmpfr_handle) + global libmpfr_path = string(libmpfr.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libmpfr_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libmpfr_path() = libmpfr_path - end # module MPFR_jll diff --git a/stdlib/MPFR_jll/test/runtests.jl b/stdlib/MPFR_jll/test/runtests.jl index fc931b462fa9c..1dbbbb298e737 100644 --- a/stdlib/MPFR_jll/test/runtests.jl +++ b/stdlib/MPFR_jll/test/runtests.jl @@ -4,5 +4,5 @@ using Test, Libdl, MPFR_jll @testset "MPFR_jll" begin vn = VersionNumber(unsafe_string(ccall((:mpfr_get_version,libmpfr), Cstring, ()))) - @test vn == v"4.2.1" + @test vn == v"4.2.2" end diff --git a/stdlib/Makefile b/stdlib/Makefile index 3975f24b7ae3b..e788f85155887 100644 --- a/stdlib/Makefile +++ b/stdlib/Makefile @@ -19,7 +19,7 @@ $(foreach dir,$(DIRS),$(eval $(call dir_target,$(dir)))) JLLS = DSFMT GMP CURL LIBGIT2 LLVM LIBSSH2 LIBUV OPENSSL MPFR NGHTTP2 \ BLASTRAMPOLINE OPENBLAS OPENLIBM P7ZIP PCRE LIBSUITESPARSE ZLIB \ - LLVMUNWIND CSL UNWIND LLD + ZSTD LLVMUNWIND CSL UNWIND LLD # Initialize this with JLLs that aren't in "deps/$(LibName).version" JLL_NAMES := MozillaCACerts_jll diff --git a/stdlib/Manifest.toml b/stdlib/Manifest.toml index 418931e34ab80..894a672ae2d96 100644 --- a/stdlib/Manifest.toml +++ b/stdlib/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.12.0-DEV" +julia_version = "1.13.0-DEV" manifest_format = "2.0" -project_hash = "1cb1aede0b4f0a2f12806233b9f188a63d6acf04" +project_hash = "6826701002e0b87f8744b1c4bf97e2cff5fc1642" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -23,7 +23,7 @@ version = "1.11.0" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.2.0+0" +version = "1.3.0+1" [[deps.Dates]] deps = ["Printf"] @@ -44,7 +44,7 @@ version = "1.11.0" [[deps.Downloads]] deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" +version = "1.7.0" [[deps.FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" @@ -56,7 +56,7 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" version = "1.11.0" [[deps.GMP_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" version = "6.3.0+2" @@ -73,7 +73,7 @@ version = "1.12.0" [[deps.LLD_jll]] deps = ["Artifacts", "Libdl", "Zlib_jll", "libLLVM_jll"] uuid = "d55e3150-da41-5e91-b323-ecfd1eec6109" -version = "18.1.7+3" +version = "20.1.2+0" [[deps.LLVMLibUnwind_jll]] deps = ["Artifacts", "Libdl"] @@ -91,9 +91,9 @@ uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" version = "0.6.4" [[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.12.1+1" +version = "8.14.1+1" [[deps.LibGit2]] deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] @@ -101,12 +101,12 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" version = "1.11.0" [[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.0+0" +version = "1.9.1+0" [[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "OpenSSL_jll", "Zlib_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" version = "1.11.3+1" @@ -116,9 +116,9 @@ uuid = "183b4373-6708-53ba-ad28-60e28bb38547" version = "2.0.1+20" [[deps.LibUnwind_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "Zlib_jll"] uuid = "745a5e78-f969-53e9-954f-d19f2f74f4e3" -version = "1.8.1+2" +version = "1.8.2+0" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -127,16 +127,16 @@ version = "1.11.0" [[deps.LinearAlgebra]] deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -version = "1.11.0" +version = "1.12.0" [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" [[deps.MPFR_jll]] -deps = ["Artifacts", "GMP_jll", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "GMP_jll", "Libdl"] uuid = "3a97d323-0669-5f0c-9066-3539efd106a3" -version = "4.2.1+1" +version = "4.2.2+0" [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] @@ -149,7 +149,7 @@ version = "1.11.0" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2024.11.26" +version = "2025.5.20" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" @@ -158,27 +158,27 @@ version = "1.3.0" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.28+3" +version = "0.3.29+0" [[deps.OpenLibm_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" version = "0.8.5+0" [[deps.OpenSSL_jll]] deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.15+1" +version = "3.5.0+0" [[deps.PCRE2_jll]] deps = ["Artifacts", "Libdl"] uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.44.0+1" +version = "10.45.0+0" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.12.0" +version = "1.13.0" weakdeps = ["REPL"] [deps.Pkg.extensions] @@ -195,7 +195,7 @@ uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" version = "1.11.0" [[deps.REPL]] -deps = ["InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] +deps = ["FileWatching", "InteractiveUtils", "JuliaSyntaxHighlighting", "Markdown", "Sockets", "StyledStrings", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" version = "1.11.0" @@ -241,7 +241,7 @@ uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" version = "1.11.0" [[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "libblastrampoline_jll"] uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" version = "7.10.1+0" @@ -274,15 +274,20 @@ deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" version = "1.3.1+2" +[[deps.Zstd_jll]] +deps = ["CompilerSupportLibraries_jll", "Libdl"] +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + [[deps.dSFMT_jll]] deps = ["Artifacts", "Libdl"] uuid = "05ff407c-b0c1-5878-9df8-858cc2e60c36" version = "2.2.5+2" [[deps.libLLVM_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl", "Zlib_jll", "Zstd_jll"] uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a" -version = "18.1.7+3" +version = "20.1.2+1" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] @@ -290,7 +295,7 @@ uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" version = "5.12.0+0" [[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" version = "1.65.0+0" diff --git a/stdlib/Markdown/src/Common/block.jl b/stdlib/Markdown/src/Common/block.jl index 247c894769f15..59ab0e58cf65b 100644 --- a/stdlib/Markdown/src/Common/block.jl +++ b/stdlib/Markdown/src/Common/block.jl @@ -114,7 +114,7 @@ function indentcode(stream::IO, block::MD) break end end - code = String(take!(buffer)) + code = takestring!(buffer) !isempty(code) && (push!(block, Code(rstrip(code))); return true) return false end @@ -178,7 +178,7 @@ function blockquote(stream::IO, block::MD) end empty && return false - md = String(take!(buffer)) + md = takestring!(buffer) push!(block, BlockQuote(parse(md, flavor = config(block)).content)) return true end @@ -236,7 +236,7 @@ function admonition(stream::IO, block::MD) end end # Parse the nested block as markdown and create a new Admonition block. - nested = parse(String(take!(buffer)), flavor = config(block)) + nested = parse(takestring!(buffer), flavor = config(block)) push!(block, Admonition(category, title, nested.content)) return true end @@ -326,7 +326,7 @@ function list(stream::IO, block::MD) return true end end -pushitem!(list, buffer) = push!(list.items, parse(String(take!(buffer))).content) +pushitem!(list, buffer) = push!(list.items, parse(takestring!(buffer)).content) # –––––––––––––– # HorizontalRule diff --git a/stdlib/Markdown/src/GitHub/GitHub.jl b/stdlib/Markdown/src/GitHub/GitHub.jl index 2ac75ea43cd19..de18b367988d9 100644 --- a/stdlib/Markdown/src/GitHub/GitHub.jl +++ b/stdlib/Markdown/src/GitHub/GitHub.jl @@ -21,9 +21,9 @@ function fencedcode(stream::IO, block::MD) if startswith(stream, string(ch) ^ n) if !startswith(stream, string(ch)) if flavor == "math" - push!(block, LaTeX(String(take!(buffer)) |> chomp)) + push!(block, LaTeX(takestring!(buffer) |> chomp)) else - push!(block, Code(flavor, String(take!(buffer)) |> chomp)) + push!(block, Code(flavor, takestring!(buffer) |> chomp)) end return true else diff --git a/stdlib/Markdown/src/GitHub/table.jl b/stdlib/Markdown/src/GitHub/table.jl index 7c174007a75ba..fefa667fc7f93 100644 --- a/stdlib/Markdown/src/GitHub/table.jl +++ b/stdlib/Markdown/src/GitHub/table.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -mutable struct Table +mutable struct Table <: MarkdownElement rows::Vector{Vector{Any}} align::Vector{Symbol} end diff --git a/stdlib/Markdown/src/IPython/IPython.jl b/stdlib/Markdown/src/IPython/IPython.jl index 54b628e768a48..cab4abbf65412 100644 --- a/stdlib/Markdown/src/IPython/IPython.jl +++ b/stdlib/Markdown/src/IPython/IPython.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -mutable struct LaTeX +mutable struct LaTeX <: MarkdownElement formula::String end diff --git a/stdlib/Markdown/src/parse/parse.jl b/stdlib/Markdown/src/parse/parse.jl index b38cc23b37dc0..8d691a281f218 100644 --- a/stdlib/Markdown/src/parse/parse.jl +++ b/stdlib/Markdown/src/parse/parse.jl @@ -63,7 +63,7 @@ function parseinline(stream::IO, md::MD, config::Config) char = peek(stream, Char) if haskey(config.inner, char) && (inner = parseinline(stream, md, config.inner[char])) !== nothing - c = String(take!(buffer)) + c = takestring!(buffer) !isempty(c) && push!(content, c) buffer = IOBuffer() push!(content, inner) @@ -71,7 +71,7 @@ function parseinline(stream::IO, md::MD, config::Config) write(buffer, read(stream, Char)) end end - c = String(take!(buffer)) + c = takestring!(buffer) !isempty(c) && push!(content, c) return content end diff --git a/stdlib/Markdown/src/parse/util.jl b/stdlib/Markdown/src/parse/util.jl index aabfcbb3ddc62..cd8158780bd6d 100644 --- a/stdlib/Markdown/src/parse/util.jl +++ b/stdlib/Markdown/src/parse/util.jl @@ -141,7 +141,7 @@ function readuntil(stream::IO, delimiter; newlines = false, match = nothing) while !eof(stream) if startswith(stream, delimiter) if count == 0 - return String(take!(buffer)) + return takestring!(buffer) else count -= 1 write(buffer, delimiter) @@ -187,7 +187,7 @@ function parse_inline_wrapper(stream::IO, delimiter::AbstractString; rep = false if !(char in whitespace || char == '\n' || char in delimiter) && startswith(stream, delimiter^n) trailing = 0 while startswith(stream, delimiter); trailing += 1; end - trailing == 0 && return String(take!(buffer)) + trailing == 0 && return takestring!(buffer) write(buffer, delimiter ^ (n + trailing)) end end diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 7d57bf053940d..c527123d51bb8 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -187,16 +187,16 @@ like HDF5 (which can be used with memory-mapping). """ function mmap(io::IO, ::Type{Array{T,N}}=Vector{UInt8}, - dims::NTuple{N,Integer}=(div(filesize(io)-position(io),sizeof(T)),), + dims::NTuple{N,Integer}=(div(filesize(io)-position(io),Base.aligned_sizeof(T)),), offset::Integer=position(io); grow::Bool=true, shared::Bool=true) where {T,N} # check inputs isopen(io) || throw(ArgumentError("$io must be open to mmap")) isbitstype(T) || throw(ArgumentError("unable to mmap $T; must satisfy isbitstype(T) == true")) - len = sizeof(T) + len = Base.aligned_sizeof(T) for l in dims len, overflow = Base.Checked.mul_with_overflow(promote(len, l)...) - overflow && throw(ArgumentError("requested size prod($((sizeof(T), dims...))) too large, would overflow typeof(size(T)) == $(typeof(len))")) + overflow && throw(ArgumentError("requested size prod($((len, dims...))) too large, would overflow typeof(size(T)) == $(typeof(len))")) end len >= 0 || throw(ArgumentError("requested size must be ≥ 0, got $len")) len == 0 && return Array{T}(undef, ntuple(x->0,Val(N))) @@ -267,7 +267,7 @@ end mmap(file::AbstractString, ::Type{T}=Vector{UInt8}, - dims::NTuple{N,Integer}=(div(filesize(file),sizeof(eltype(T))),), + dims::NTuple{N,Integer}=(div(filesize(file),Base.aligned_sizeof(eltype(T))),), offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true) where {T<:Array,N} = open(io->mmap(io, T, dims, offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::Array{eltype(T),N} diff --git a/stdlib/Mmap/test/runtests.jl b/stdlib/Mmap/test/runtests.jl index f4b6abb147fb7..4ab04bf733190 100644 --- a/stdlib/Mmap/test/runtests.jl +++ b/stdlib/Mmap/test/runtests.jl @@ -44,7 +44,7 @@ s = open(file) @test length(@inferred mmap(s, Vector{Int8}, 12, 0; grow=false)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12, 0; shared=false)) == 12 close(s) -@test_throws ErrorException mmap(file, Vector{Ref}) # must be bit-type +@test_throws ArgumentError mmap(file, Vector{Ref}) # must be bit-type GC.gc(); GC.gc() file = tempname() # new name to reduce chance of issues due slow windows fs @@ -343,6 +343,19 @@ end GC.gc() rm(file) +# test for #58982 - mmap with primitive types +file = tempname() +primitive type PrimType9Bytes 9*8 end +arr = Vector{PrimType9Bytes}(undef, 2) +write(file, arr) +m = mmap(file, Vector{PrimType9Bytes}) +@test length(m) == 2 +@test m[1] == arr[1] +@test m[2] == arr[2] +finalize(m); m = nothing; GC.gc() +rm(file) + + @testset "Docstrings" begin @test isempty(Docs.undocumented_names(Mmap)) end diff --git a/stdlib/MozillaCACerts_jll/Project.toml b/stdlib/MozillaCACerts_jll/Project.toml index 2f9bf67e22a74..e1dbca5814708 100644 --- a/stdlib/MozillaCACerts_jll/Project.toml +++ b/stdlib/MozillaCACerts_jll/Project.toml @@ -1,7 +1,7 @@ name = "MozillaCACerts_jll" uuid = "14a3606d-f60d-562e-9121-12d972cd8159" # Keep in sync with `deps/libgit2.version`. -version = "2024.12.31" +version = "2025.07.15" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/stdlib/NetworkOptions.version b/stdlib/NetworkOptions.version index 69b56e03ed89e..f7cb50a74d106 100644 --- a/stdlib/NetworkOptions.version +++ b/stdlib/NetworkOptions.version @@ -1,4 +1,4 @@ NETWORKOPTIONS_BRANCH = master -NETWORKOPTIONS_SHA1 = c090626d3feee6d6a5c476346d22d6147c9c6d2d +NETWORKOPTIONS_SHA1 = 532992fcc0f1d02df48374969cbae37e34c01360 NETWORKOPTIONS_GIT_URL := https://github.com/JuliaLang/NetworkOptions.jl.git NETWORKOPTIONS_TAR_URL = https://api.github.com/repos/JuliaLang/NetworkOptions.jl/tarball/$1 diff --git a/stdlib/OpenBLAS_jll/Project.toml b/stdlib/OpenBLAS_jll/Project.toml index 07a81d3c1d547..8eafa2f2365c1 100644 --- a/stdlib/OpenBLAS_jll/Project.toml +++ b/stdlib/OpenBLAS_jll/Project.toml @@ -3,10 +3,9 @@ uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" version = "0.3.29+0" [deps] -# See note in `src/OpenBLAS_jll.jl` about this dependency. +Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [compat] julia = "1.11" diff --git a/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl b/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl index bd93b050ebbee..238ad459a2dc9 100644 --- a/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl +++ b/stdlib/OpenBLAS_jll/src/OpenBLAS_jll.jl @@ -2,7 +2,7 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/OpenBLAS_jll.jl baremodule OpenBLAS_jll -using Base, Libdl, Base.BinaryPlatforms +using Base, Libdl using CompilerSupportLibraries_jll export libopenblas @@ -13,7 +13,7 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libopenblas_path::String = "" + if Base.USE_BLAS64 const libsuffix = "64_" @@ -21,14 +21,30 @@ else const libsuffix = "" end -if Sys.iswindows() - const _libopenblas_path = BundledLazyLibraryPath(string("libopenblas", libsuffix, ".dll")) -elseif Sys.isapple() - const _libopenblas_path = BundledLazyLibraryPath(string("libopenblas", libsuffix, ".dylib")) -else - const _libopenblas_path = BundledLazyLibraryPath(string("libopenblas", libsuffix, ".so")) -end -const libopenblas = LazyLibrary(_libopenblas_path, dependencies=[libgfortran]) +libopenblas_path::String = "" +const libopenblas = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath(string("libopenblas", libsuffix, ".dll")) + elseif Sys.isapple() + BundledLazyLibraryPath(string("libopenblas", libsuffix, ".dylib")) + else + BundledLazyLibraryPath(string("libopenblas", libsuffix, ".so")) + end, + dependencies = if Sys.iswindows() + LazyLibrary[libgfortran, libgcc_s] + elseif Sys.isapple() + deps = LazyLibrary[libgfortran] + if isdefined(CompilerSupportLibraries_jll, :libquadmath) + push!(deps, CompilerSupportLibraries_jll.libquadmath) + end + if Sys.ARCH != :aarch64 + push!(deps, CompilerSupportLibraries_jll.libgcc_s) + end + deps + else + LazyLibrary[libgfortran] + end +) # Conform to LazyJLLWrappers API function eager_mode() @@ -38,7 +54,7 @@ end is_available() = true function __init__() - global libopenblas_path = string(_libopenblas_path) + global libopenblas_path = string(libopenblas.path) # make sure OpenBLAS does not set CPU affinity (#1070, #9639) if !(haskey(ENV, "OPENBLAS_MAIN_FREE")) ENV["OPENBLAS_MAIN_FREE"] = "1" @@ -55,7 +71,7 @@ function __init__() ENV["OPENBLAS_DEFAULT_NUM_THREADS"] = "1" end - global libopenblas_path = string(_libopenblas_path) + global libopenblas_path = string(libopenblas.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libopenblas_path) push!(LIBPATH_list, LIBPATH[]) diff --git a/stdlib/OpenLibm_jll/Project.toml b/stdlib/OpenLibm_jll/Project.toml index 431528ee3f400..441ad36e79012 100644 --- a/stdlib/OpenLibm_jll/Project.toml +++ b/stdlib/OpenLibm_jll/Project.toml @@ -3,10 +3,12 @@ uuid = "05823500-19ac-5b8b-9628-191a04bc5112" version = "0.8.5+0" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.0" [extras] diff --git a/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl b/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl index 297cd25512894..264dbbf9af8b9 100644 --- a/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl +++ b/stdlib/OpenLibm_jll/src/OpenLibm_jll.jl @@ -3,42 +3,48 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/OpenLibm_jll.jl baremodule OpenLibm_jll using Base, Libdl - -const PATH_list = String[] -const LIBPATH_list = String[] +if Sys.iswindows() + using CompilerSupportLibraries_jll +end export libopenlibm # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libopenlibm_handle::Ptr{Cvoid} = C_NULL -libopenlibm_path::String = "" -if Sys.iswindows() - const libopenlibm = "libopenlibm.dll" -elseif Sys.isapple() - const libopenlibm = "@rpath/libopenlibm.4.dylib" -else - const libopenlibm = "libopenlibm.so.4" +libopenlibm_path::String = "" +const libopenlibm = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libopenlibm.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libopenlibm.4.dylib") + else + BundledLazyLibraryPath("libopenlibm.so.4") + end, + dependencies = if Sys.iswindows() + LazyLibrary[libgcc_s] + else + LazyLibrary[] + end +) + +function eager_mode() + dlopen(libopenlibm) + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end end +is_available() = true function __init__() - global libopenlibm_handle = dlopen(libopenlibm) - global libopenlibm_path = dlpath(libopenlibm_handle) + global libopenlibm_path = string(libopenlibm.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libopenlibm_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libopenlibm_path() = libopenlibm_path - end # module OpenLibm_jll diff --git a/stdlib/OpenSSL_jll/Project.toml b/stdlib/OpenSSL_jll/Project.toml index 7c8067261c253..d11c3b25a6922 100644 --- a/stdlib/OpenSSL_jll/Project.toml +++ b/stdlib/OpenSSL_jll/Project.toml @@ -1,6 +1,6 @@ name = "OpenSSL_jll" uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.16+0" +version = "3.5.1+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl b/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl index bba9a0a299de9..f4d11ee65b3bf 100644 --- a/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl +++ b/stdlib/OpenSSL_jll/src/OpenSSL_jll.jl @@ -5,54 +5,67 @@ baremodule OpenSSL_jll using Base, Libdl, Base.BinaryPlatforms -const PATH_list = String[] -const LIBPATH_list = String[] - export libcrypto, libssl # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libcrypto_handle::Ptr{Cvoid} = C_NULL -libcrypto_path::String = "" -libssl_handle::Ptr{Cvoid} = C_NULL -libssl_path::String = "" -if Sys.iswindows() - if arch(HostPlatform()) == "x86_64" - const libcrypto = "libcrypto-3-x64.dll" - const libssl = "libssl-3-x64.dll" +libcrypto_path::String = "" +const libcrypto = LazyLibrary( + if Sys.iswindows() + if arch(HostPlatform()) == "x86_64" + BundledLazyLibraryPath("libcrypto-3-x64.dll") + else + BundledLazyLibraryPath("libcrypto-3.dll") + end + elseif Sys.isapple() + BundledLazyLibraryPath("libcrypto.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcrypto.so.3") else - const libcrypto = "libcrypto-3.dll" - const libssl = "libssl-3.dll" + error("OpenSSL_jll: Library 'libcrypto' is not available for $(Sys.KERNEL)") end -elseif Sys.isapple() - const libcrypto = "@rpath/libcrypto.3.dylib" - const libssl = "@rpath/libssl.3.dylib" -else - const libcrypto = "libcrypto.so.3" - const libssl = "libssl.so.3" +) + +libssl_path::String = "" +const libssl = LazyLibrary( + if Sys.iswindows() + if arch(HostPlatform()) == "x86_64" + BundledLazyLibraryPath("libssl-3-x64.dll") + else + BundledLazyLibraryPath("libssl-3.dll") + end + elseif Sys.isapple() + BundledLazyLibraryPath("libssl.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libssl.so.3") + else + error("OpenSSL_jll: Library 'libssl' is not available for $(Sys.KERNEL)") + end; + dependencies = LazyLibrary[libcrypto] +) + +function eager_mode() + dlopen(libcrypto) + dlopen(libssl) end +is_available() = true function __init__() - global libcrypto_handle = dlopen(libcrypto) - global libcrypto_path = dlpath(libcrypto_handle) - global libssl_handle = dlopen(libssl) - global libssl_path = dlpath(libssl_handle) + global libcrypto_path = string(libcrypto.path) + global libssl_path = string(libssl.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libssl_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libcrypto_path() = libcrypto_path -get_libssl_path() = libssl_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module OpenSSL_jll diff --git a/stdlib/OpenSSL_jll/test/runtests.jl b/stdlib/OpenSSL_jll/test/runtests.jl index 8bf67288834c9..9ef99d57134e0 100644 --- a/stdlib/OpenSSL_jll/test/runtests.jl +++ b/stdlib/OpenSSL_jll/test/runtests.jl @@ -6,5 +6,5 @@ using Test, Libdl, OpenSSL_jll major = ccall((:OPENSSL_version_major, libcrypto), Cuint, ()) minor = ccall((:OPENSSL_version_minor, libcrypto), Cuint, ()) patch = ccall((:OPENSSL_version_patch, libcrypto), Cuint, ()) - @test VersionNumber(major, minor, patch) == v"3.0.16" + @test VersionNumber(major, minor, patch) == v"3.5.1" end diff --git a/stdlib/PCRE2_jll/Project.toml b/stdlib/PCRE2_jll/Project.toml index 24ac196a3b8a9..1c6187e06be31 100644 --- a/stdlib/PCRE2_jll/Project.toml +++ b/stdlib/PCRE2_jll/Project.toml @@ -1,6 +1,6 @@ name = "PCRE2_jll" uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" -version = "10.44.0+1" +version = "10.45.0+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" diff --git a/stdlib/PCRE2_jll/src/PCRE2_jll.jl b/stdlib/PCRE2_jll/src/PCRE2_jll.jl index d825ac74db5a8..c6e32bf3e6672 100644 --- a/stdlib/PCRE2_jll/src/PCRE2_jll.jl +++ b/stdlib/PCRE2_jll/src/PCRE2_jll.jl @@ -4,41 +4,43 @@ baremodule PCRE2_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export libpcre2_8 # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libpcre2_8_handle::Ptr{Cvoid} = C_NULL -libpcre2_8_path::String = "" -if Sys.iswindows() - const libpcre2_8 = "libpcre2-8-0.dll" -elseif Sys.isapple() - const libpcre2_8 = "@rpath/libpcre2-8.0.dylib" -else - const libpcre2_8 = "libpcre2-8.so.0" +libpcre2_8_path::String = "" +const libpcre2_8 = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libpcre2-8.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libpcre2-8.0.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libpcre2-8.so.0") + else + error("PCRE2_jll: Library 'libpcre2_8' is not available for $(Sys.KERNEL)") + end +) + +function eager_mode() + dlopen(libpcre2_8) end +is_available() = true function __init__() - global libpcre2_8_handle = dlopen(libpcre2_8) - global libpcre2_8_path = dlpath(libpcre2_8_handle) + global libpcre2_8_path = string(libpcre2_8.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libpcre2_8_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libpcre2_8_path() = libpcre2_8_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module PCRE2_jll diff --git a/stdlib/PCRE2_jll/test/runtests.jl b/stdlib/PCRE2_jll/test/runtests.jl index 21df2ec430e0e..9040eb5b76af0 100644 --- a/stdlib/PCRE2_jll/test/runtests.jl +++ b/stdlib/PCRE2_jll/test/runtests.jl @@ -6,5 +6,5 @@ using Test, Libdl, PCRE2_jll vstr = zeros(UInt8, 32) @test ccall((:pcre2_config_8, libpcre2_8), Cint, (UInt32, Ref{UInt8}), 11, vstr) > 0 vn = VersionNumber(split(unsafe_string(pointer(vstr)), " ")[1]) - @test vn == v"10.44.0" + @test vn == v"10.45.0" end diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index 2fdf646e80bbf..ae88e816ffbc6 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = 26bdeeeb10f3ab0c6cf7478cb8454a800b4578be +PKG_SHA1 = d94f8a1d946f90b00b836afc1dedf034604c6627 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 diff --git a/stdlib/Profile/test/runtests.jl b/stdlib/Profile/test/runtests.jl index b487d8963f156..0ae773ec8f284 100644 --- a/stdlib/Profile/test/runtests.jl +++ b/stdlib/Profile/test/runtests.jl @@ -220,9 +220,19 @@ end import InteractiveUtils +@generated function compile_takes_1_second(x) + t = time_ns() + while time_ns() < t + 1e9 + # busy wait for 1 second + end + return :(x) +end @testset "Module short names" begin Profile.clear() - @profile InteractiveUtils.peakflops() + @profile begin + @eval compile_takes_1_second(1) # to increase chance of profiling hitting compilation code + InteractiveUtils.peakflops() + end io = IOBuffer() ioc = IOContext(io, :displaysize=>(1000,1000)) Profile.print(ioc, C=true) diff --git a/stdlib/Project.toml b/stdlib/Project.toml index 1e03a0f474490..2051377a6967c 100644 --- a/stdlib/Project.toml +++ b/stdlib/Project.toml @@ -54,6 +54,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" +Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" dSFMT_jll = "05ff407c-b0c1-5878-9df8-858cc2e60c36" libLLVM_jll = "8f36deef-c2a5-5394-99ed-8e07531fb29a" libblastrampoline_jll = "8e850b90-86db-534c-a0d3-1478176c7d93" diff --git a/stdlib/REPL/Project.toml b/stdlib/REPL/Project.toml index 5ee3849f507f2..968786d492bcc 100644 --- a/stdlib/REPL/Project.toml +++ b/stdlib/REPL/Project.toml @@ -3,6 +3,7 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" version = "1.11.0" [deps] +FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JuliaSyntaxHighlighting = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 288a9cb1ea91f..ba4a8155b13bf 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -169,7 +169,7 @@ region_active(s::PromptState) = s.region_active region_active(s::ModeState) = :off -input_string(s::PromptState) = String(take!(copy(s.input_buffer)))::String +input_string(s::PromptState) = takestring!(copy(s.input_buffer))::String input_string_newlines(s::PromptState) = count(c->(c == '\n'), input_string(s)) function input_string_newlines_aftercursor(s::PromptState) @@ -391,16 +391,21 @@ function complete_line(s::MIState) end end +# Old complete_line return type: Vector{String}, String, Bool +# New complete_line return type: NamedCompletion{String}, String, Bool +# OR NamedCompletion{String}, Region, Bool +# # due to close coupling of the Pkg ReplExt `complete_line` can still return a vector of strings, # so we convert those in this helper -function complete_line_named(args...; kwargs...)::Tuple{Vector{NamedCompletion},String,Bool} - result = complete_line(args...; kwargs...)::Union{Tuple{Vector{NamedCompletion},String,Bool},Tuple{Vector{String},String,Bool}} - if result isa Tuple{Vector{NamedCompletion},String,Bool} - return result - else - completions, partial, should_complete = result - return map(NamedCompletion, completions), partial, should_complete - end +function complete_line_named(c, s, args...; kwargs...)::Tuple{Vector{NamedCompletion},Region,Bool} + r1, r2, should_complete = complete_line(c, s, args...; kwargs...)::Union{ + Tuple{Vector{String}, String, Bool}, + Tuple{Vector{NamedCompletion}, String, Bool}, + Tuple{Vector{NamedCompletion}, Region, Bool}, + } + completions = (r1 isa Vector{String} ? map(NamedCompletion, r1) : r1) + r = (r2 isa String ? (position(s)-sizeof(r2) => position(s)) : r2) + completions, r, should_complete end # checks for a hint and shows it if appropriate. @@ -426,14 +431,14 @@ function check_show_hint(s::MIState) return end t_completion = Threads.@spawn :default begin - named_completions, partial, should_complete = nothing, nothing, nothing + named_completions, reg, should_complete = nothing, nothing, nothing # only allow one task to generate hints at a time and check around lock # if the user has pressed a key since the hint was requested, to skip old completions next_key_pressed() && return @lock s.hint_generation_lock begin next_key_pressed() && return - named_completions, partial, should_complete = try + named_completions, reg, should_complete = try complete_line_named(st.p.complete, st, s.active_module; hint = true) catch lock_clear_hint() @@ -448,21 +453,19 @@ function check_show_hint(s::MIState) return end # Don't complete for single chars, given e.g. `x` completes to `xor` - if length(partial) > 1 && should_complete + if reg.second - reg.first > 1 && should_complete singlecompletion = length(completions) == 1 p = singlecompletion ? completions[1] : common_prefix(completions) if singlecompletion || p in completions # i.e. complete `@time` even though `@time_imports` etc. exists - # The completion `p` and the input `partial` may not share the same initial + # The completion `p` and the region `reg` may not share the same initial # characters, for instance when completing to subscripts or superscripts. # So, in general, make sure that the hint starts at the correct position by # incrementing its starting position by as many characters as the input. - startind = 1 # index of p from which to start providing the hint - maxind = ncodeunits(p) - for _ in partial - startind = nextind(p, startind) - startind > maxind && break - end + maxind = lastindex(p) + startind = sizeof(content(s, reg)) if startind ≤ maxind # completion on a complete name returns itself so check that there's something to hint + # index of p from which to start providing the hint + startind = nextind(p, startind) hint = p[startind:end] next_key_pressed() && return @lock s.line_modify_lock begin @@ -491,7 +494,7 @@ function clear_hint(s::ModeState) end function complete_line(s::PromptState, repeats::Int, mod::Module; hint::Bool=false) - completions, partial, should_complete = complete_line_named(s.p.complete, s, mod; hint) + completions, reg, should_complete = complete_line_named(s.p.complete, s, mod; hint) isempty(completions) && return false if !should_complete # should_complete is false for cases where we only want to show @@ -499,17 +502,16 @@ function complete_line(s::PromptState, repeats::Int, mod::Module; hint::Bool=fal show_completions(s, completions) elseif length(completions) == 1 # Replace word by completion - prev_pos = position(s) push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion) + edit_splice!(s, reg, completions[1].completion) else p = common_prefix(completions) + partial = content(s, reg.first => min(bufend(s), reg.first + sizeof(p))) if !isempty(p) && p != partial # All possible completions share the same prefix, so we might as - # well complete that - prev_pos = position(s) + # well complete that. push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, p) + edit_splice!(s, reg, p) elseif repeats > 0 show_completions(s, completions) end @@ -830,12 +832,12 @@ function edit_move_right(m::MIState) refresh_line(s) return true else - completions, partial, should_complete = complete_line(s.p.complete, s, m.active_module) - if should_complete && eof(buf) && length(completions) == 1 && length(partial) > 1 + completions, reg, should_complete = complete_line(s.p.complete, s, m.active_module) + if should_complete && eof(buf) && length(completions) == 1 && reg.second - reg.first > 1 # Replace word by completion prev_pos = position(s) push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion) + edit_splice!(s, (prev_pos - reg.second + reg.first) => prev_pos, completions[1].completion) refresh_line(state(s)) return true else @@ -1524,7 +1526,7 @@ function edit_input(s, f = (filename, line, column) -> InteractiveUtils.edit(fil end buf = buffer(s) pos = position(buf) - str = String(take!(buf)) + str = takestring!(buf) lines = readlines(IOBuffer(str); keep=true) # Compute line @@ -1758,7 +1760,7 @@ function normalize_key(key::Union{String,SubString{String}}) write(buf, c) end end - return String(take!(buf)) + return takestring!(buf) end function normalize_keys(keymap::Union{Dict{Char,Any},AnyDict}) @@ -2098,7 +2100,7 @@ function history_set_backward(s::SearchState, backward::Bool) nothing end -input_string(s::SearchState) = String(take!(copy(s.query_buffer))) +input_string(s::SearchState) = takestring!(copy(s.query_buffer)) function reset_state(s::SearchState) if s.query_buffer.size != 0 @@ -2186,7 +2188,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, return ias end -input_string(s::PrefixSearchState) = String(take!(copy(s.response_buffer))) +input_string(s::PrefixSearchState) = takestring!(copy(s.response_buffer)) write_prompt(terminal, s::PrefixSearchState, color::Bool) = write_prompt(terminal, s.histprompt.parent_prompt, color) prompt_string(s::PrefixSearchState) = prompt_string(s.histprompt.parent_prompt.prompt) @@ -2255,12 +2257,12 @@ setmodifiers!(c) = nothing # Search Mode completions function complete_line(s::SearchState, repeats, mod::Module; hint::Bool=false) - completions, partial, should_complete = complete_line(s.histprompt.complete, s, mod; hint) + completions, reg, should_complete = complete_line(s.histprompt.complete, s, mod; hint) # For now only allow exact completions in search mode if length(completions) == 1 prev_pos = position(s) push_undo(s) - edit_splice!(s, (prev_pos - sizeof(partial)) => prev_pos, completions[1].completion) + edit_splice!(s, (prev_pos - reg.second - reg.first) => prev_pos, completions[1].completion) return true end return false diff --git a/stdlib/REPL/src/Pkg_beforeload.jl b/stdlib/REPL/src/Pkg_beforeload.jl index 86b5cd35abd2f..e51cf7550bce5 100644 --- a/stdlib/REPL/src/Pkg_beforeload.jl +++ b/stdlib/REPL/src/Pkg_beforeload.jl @@ -1,7 +1,9 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + ## Pkg stuff needed before Pkg has loaded const Pkg_pkgid = Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg") -load_pkg() = Base.require_stdlib(Pkg_pkgid, "REPLExt") +load_pkg() = Base.require_stdlib(Pkg_pkgid, "REPLExt", REPL) ## Below here copied/tweaked from Pkg Types.jl so that the dummy Pkg prompt # can populate the env correctly before Pkg loads diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index ae2fec4ee10e0..db6f5a26e90ed 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -41,6 +41,8 @@ end using Base.Meta, Sockets, StyledStrings using JuliaSyntaxHighlighting import InteractiveUtils +import FileWatching +import Base.JuliaSyntax: kind, @K_str, @KSet_str, Tokenize.tokenize export AbstractREPL, @@ -86,6 +88,7 @@ import .LineEdit: PromptState, mode_idx +include("SyntaxUtil.jl") include("REPLCompletions.jl") using .REPLCompletions @@ -296,12 +299,12 @@ const install_packages_hooks = Any[] # N.B.: Any functions starting with __repl_entry cut off backtraces when printing in the REPL. # We need to do this for both the actual eval and macroexpand, since the latter can cause custom macro # code to run (and error). -__repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = - ccall(:jl_expand_with_loc, Any, (Any, Any, Ptr{UInt8}, Cint), ast, mod, toplevel_file[], toplevel_line[]) -__repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Cint}) = - ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Cint}), mod, ast, 1, 1, toplevel_file, toplevel_line) +__repl_entry_lower_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Csize_t}) = + Core._lower(ast, mod, toplevel_file[], toplevel_line[])[1] +__repl_entry_eval_expanded_with_loc(mod::Module, @nospecialize(ast), toplevel_file::Ref{Ptr{UInt8}}, toplevel_line::Ref{Csize_t}) = + ccall(:jl_toplevel_eval_flex, Any, (Any, Any, Cint, Cint, Ptr{Ptr{UInt8}}, Ptr{Csize_t}), mod, ast, 1, 1, toplevel_file, toplevel_line) -function toplevel_eval_with_hooks(mod::Module, @nospecialize(ast), toplevel_file=Ref{Ptr{UInt8}}(Base.unsafe_convert(Ptr{UInt8}, :REPL)), toplevel_line=Ref{Cint}(1)) +function toplevel_eval_with_hooks(mod::Module, @nospecialize(ast), toplevel_file=Ref{Ptr{UInt8}}(Base.unsafe_convert(Ptr{UInt8}, :REPL)), toplevel_line=Ref{Csize_t}(1)) if !isexpr(ast, :toplevel) ast = invokelatest(__repl_entry_lower_with_loc, mod, ast, toplevel_file, toplevel_line) check_for_missing_packages_and_run_hooks(ast) @@ -358,26 +361,31 @@ function check_for_missing_packages_and_run_hooks(ast) end function _modules_to_be_loaded!(ast::Expr, mods::Vector{Symbol}) + function add!(ctx) + if ctx.head == :as + ctx = ctx.args[1] + end + if ctx.args[1] != :. # don't include local import `import .Foo` + push!(mods, ctx.args[1]) + end + end ast.head === :quote && return mods # don't search if it's not going to be run during this eval - if ast.head === :using || ast.head === :import - for arg in ast.args - arg = arg::Expr - arg1 = first(arg.args) - if arg1 isa Symbol # i.e. `Foo` - if arg1 != :. # don't include local import `import .Foo` - push!(mods, arg1) - end - else # i.e. `Foo: bar` - sym = first((arg1::Expr).args)::Symbol - if sym != :. # don't include local import `import .Foo: a` - push!(mods, sym) - end + if ast.head == :call + if length(ast.args) == 5 && ast.args[1] === GlobalRef(Base, :_eval_import) + ctx = ast.args[4] + if ctx isa QuoteNode # i.e. `Foo: bar` + ctx = ctx.value + else + ctx = ast.args[5].value end + add!(ctx) + elseif length(ast.args) == 3 && ast.args[1] == GlobalRef(Base, :_eval_using) + add!(ast.args[3].value) end end if ast.head !== :thunk for arg in ast.args - if isexpr(arg, (:block, :if, :using, :import)) + if isexpr(arg, (:block, :if)) _modules_to_be_loaded!(arg, mods) end end @@ -444,8 +452,8 @@ function repl_backend_loop(backend::REPLBackend, get_module::Function) try ret = f() put!(backend.response_channel, Pair{Any, Bool}(ret, false)) - catch err - put!(backend.response_channel, Pair{Any, Bool}(err, true)) + catch + put!(backend.response_channel, Pair{Any, Bool}(current_exceptions(), true)) end else ast = ast_or_func @@ -473,6 +481,8 @@ function Base.showerror(io::IO, e::LimitIOException) print(io, "$LimitIOException: aborted printing after attempting to print more than $(Base.format_bytes(e.maxbytes)) within a `LimitIO`.") end +Base.displaysize(io::LimitIO) = _displaysize(io.io) + function Base.write(io::LimitIO, v::UInt8) io.n > io.maxbytes && throw(LimitIOException(io.maxbytes)) n_bytes = write(io.io, v) @@ -545,6 +555,16 @@ display(d::REPLDisplay, x) = display(d, MIME("text/plain"), x) show_repl(io::IO, mime::MIME"text/plain", x) = show(io, mime, x) +function show_repl(io::IO, mime::MIME"text/plain", c::AbstractChar) + show(io, mime, c) # Call the original Base.show + # Check for LaTeX/emoji alias and print if found and using symbol_latex which is used in help?> mode + latex = symbol_latex(string(c)) + if !isempty(latex) + print(io, ", input as ") + printstyled(io, latex, ""; color=:cyan) + end +end + show_repl(io::IO, ::MIME"text/plain", ex::Expr) = print(io, JuliaSyntaxHighlighting.highlight( sprint(show, ex, context=IOContext(io, :color => false)))) @@ -584,11 +604,11 @@ function print_response(errio::IO, response, backend::Union{REPLBackendRef,Nothi if val !== nothing && show_value val2, iserr = if specialdisplay === nothing # display calls may require being run on the main thread - eval_with_backend(backend) do + call_on_backend(backend) do Base.invokelatest(display, val) end else - eval_with_backend(backend) do + call_on_backend(backend) do Base.invokelatest(display, specialdisplay, val) end end @@ -705,7 +725,7 @@ function run_frontend(repl::BasicREPL, backend::REPLBackendRef) (isa(ast,Expr) && ast.head === :incomplete) || break end if !isempty(line) - response = eval_with_backend(ast, backend) + response = eval_on_backend(ast, backend) print_response(repl, response, !ends_with_semicolon(line), false) end write(repl.terminal, '\n') @@ -799,27 +819,29 @@ end beforecursor(buf::IOBuffer) = String(buf.data[1:buf.ptr-1]) +# Convert inclusive-inclusive 1-based char indexing to inclusive-exclusive byte Region. +to_region(s, r) = first(r)-1 => (length(r) > 0 ? nextind(s, last(r))-1 : first(r)-1) + function complete_line(c::REPLCompletionProvider, s::PromptState, mod::Module; hint::Bool=false) - partial = beforecursor(s.input_buffer) full = LineEdit.input_string(s) - ret, range, should_complete = completions(full, lastindex(partial), mod, c.modifiers.shift, hint) + ret, range, should_complete = completions(full, thisind(full, position(s)), mod, c.modifiers.shift, hint) + range = to_region(full, range) c.modifiers = LineEdit.Modifiers() - return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), partial[range], should_complete + return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), range, should_complete end function complete_line(c::ShellCompletionProvider, s::PromptState; hint::Bool=false) - # First parse everything up to the current position - partial = beforecursor(s.input_buffer) full = LineEdit.input_string(s) - ret, range, should_complete = shell_completions(full, lastindex(partial), hint) - return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), partial[range], should_complete + ret, range, should_complete = shell_completions(full, thisind(full, position(s)), hint) + range = to_region(full, range) + return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), range, should_complete end function complete_line(c::LatexCompletions, s; hint::Bool=false) - partial = beforecursor(LineEdit.buffer(s)) full = LineEdit.input_string(s)::String - ret, range, should_complete = bslash_completions(full, lastindex(partial), hint)[2] - return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), partial[range], should_complete + ret, range, should_complete = bslash_completions(full, thisind(full, position(s)), hint)[2] + range = to_region(full, range) + return unique!(LineEdit.NamedCompletion[named_completion(x) for x in ret]), range, should_complete end with_repl_linfo(f, repl) = f(outstream(repl)) @@ -911,7 +933,7 @@ function hist_from_file(hp::REPLHistoryProvider, path::String) end function add_history(hist::REPLHistoryProvider, s::PromptState) - str = rstrip(String(take!(copy(s.input_buffer)))) + str = rstrip(takestring!(copy(s.input_buffer))) isempty(strip(str)) && return mode = mode_idx(hist, LineEdit.mode(s)) !isempty(hist.history) && @@ -924,7 +946,6 @@ function add_history(hist::REPLHistoryProvider, s::PromptState) # mode: $mode $(replace(str, r"^"ms => "\t")) """ - # TODO: write-lock history file try seekend(hist.history_file) catch err @@ -933,8 +954,15 @@ function add_history(hist::REPLHistoryProvider, s::PromptState) # If this doesn't fix it (e.g. when file is deleted), we'll end up rethrowing anyway hist_open_file(hist) end - print(hist.history_file, entry) - flush(hist.history_file) + if isfile(hist.file_path) + FileWatching.mkpidlock(hist.file_path * ".pid", stale_age=3) do + print(hist.history_file, entry) + flush(hist.history_file) + end + else # handle eg devnull + print(hist.history_file, entry) + flush(hist.history_file) + end nothing end @@ -1038,7 +1066,7 @@ function history_move_prefix(s::LineEdit.PrefixSearchState, prefix::AbstractString, backwards::Bool, cur_idx::Int = hist.cur_idx) - cur_response = String(take!(copy(LineEdit.buffer(s)))) + cur_response = takestring!(copy(LineEdit.buffer(s))) # when searching forward, start at last_idx if !backwards && hist.last_idx > 0 cur_idx = hist.last_idx @@ -1080,7 +1108,7 @@ function history_search(hist::REPLHistoryProvider, query_buffer::IOBuffer, respo qpos = position(query_buffer) qpos > 0 || return true searchdata = beforecursor(query_buffer) - response_str = String(take!(copy(response_buffer))) + response_str = takestring!(copy(response_buffer)) # Alright, first try to see if the current match still works a = position(response_buffer) + 1 # position is zero-indexed @@ -1137,7 +1165,7 @@ end LineEdit.reset_state(hist::REPLHistoryProvider) = history_reset_state(hist) function return_callback(s) - ast = Base.parse_input_line(String(take!(copy(LineEdit.buffer(s)))), depwarn=false) + ast = Base.parse_input_line(takestring!(copy(LineEdit.buffer(s))), depwarn=false) return !(isa(ast, Expr) && ast.head === :incomplete) end @@ -1148,21 +1176,23 @@ find_hist_file() = get(ENV, "JULIA_HISTORY", backend(r::AbstractREPL) = hasproperty(r, :backendref) ? r.backendref : nothing -function eval_with_backend(ast::Expr, backend::REPLBackendRef) +function eval_on_backend(ast, backend::REPLBackendRef) put!(backend.repl_channel, (ast, 1)) # (f, show_value) return take!(backend.response_channel) # (val, iserr) end -function eval_with_backend(f, backend::REPLBackendRef) +function call_on_backend(f, backend::REPLBackendRef) + applicable(f) || error("internal error: f is not callable") put!(backend.repl_channel, (f, 2)) # (f, show_value) 2 indicates function (rather than ast) return take!(backend.response_channel) # (val, iserr) end # if no backend just eval (used by tests) -function eval_with_backend(f, backend::Nothing) +eval_on_backend(ast, backend::Nothing) = error("no backend for eval ast") +function call_on_backend(f, backend::Nothing) try ret = f() return (ret, false) # (val, iserr) - catch err - return (err, true) + catch + return (current_exceptions(), true) end end @@ -1178,7 +1208,7 @@ function respond(f, repl, main; pass_empty::Bool = false, suppress_on_semicolon: local response try ast = Base.invokelatest(f, line) - response = eval_with_backend(ast, backend(repl)) + response = eval_on_backend(ast, backend(repl)) catch response = Pair{Any, Bool}(current_exceptions(), true) end @@ -1683,49 +1713,16 @@ answer_color(r::StreamREPL) = r.answer_color input_color(r::LineEditREPL) = r.envcolors ? Base.input_color() : r.input_color input_color(r::StreamREPL) = r.input_color -let matchend = Dict("\"" => r"\"", "\"\"\"" => r"\"\"\"", "'" => r"'", - "`" => r"`", "```" => r"```", "#" => r"$"m, "#=" => r"=#|#=") - global _rm_strings_and_comments - function _rm_strings_and_comments(code::Union{String,SubString{String}}) - buf = IOBuffer(sizehint = sizeof(code)) - pos = 1 - while true - i = findnext(r"\"(?!\"\")|\"\"\"|'|`(?!``)|```|#(?!=)|#=", code, pos) - isnothing(i) && break - match = SubString(code, i) - j = findnext(matchend[match]::Regex, code, nextind(code, last(i))) - if match == "#=" # possibly nested - nested = 1 - while j !== nothing - nested += SubString(code, j) == "#=" ? +1 : -1 - iszero(nested) && break - j = findnext(r"=#|#=", code, nextind(code, last(j))) - end - elseif match[1] != '#' # quote match: check non-escaped - while j !== nothing - notbackslash = findprev(!=('\\'), code, prevind(code, first(j)))::Int - isodd(first(j) - notbackslash) && break # not escaped - j = findnext(matchend[match]::Regex, code, nextind(code, first(j))) - end - end - isnothing(j) && break - if match[1] == '#' - print(buf, SubString(code, pos, prevind(code, first(i)))) - else - print(buf, SubString(code, pos, last(i)), ' ', SubString(code, j)) - end - pos = nextind(code, last(j)) - end - print(buf, SubString(code, pos, lastindex(code))) - return String(take!(buf)) - end -end - # heuristic function to decide if the presence of a semicolon # at the end of the expression was intended for suppressing output -ends_with_semicolon(code::AbstractString) = ends_with_semicolon(String(code)) -ends_with_semicolon(code::Union{String,SubString{String}}) = - contains(_rm_strings_and_comments(code), r";\s*$") +function ends_with_semicolon(code) + semi = false + for tok in tokenize(code) + kind(tok) in KSet"Whitespace NewlineWs Comment EndMarker" && continue + semi = kind(tok) == K";" + end + return semi +end function banner(io::IO = stdout; short = false) if Base.GIT_VERSION_INFO.tagged_commit @@ -1818,7 +1815,7 @@ function run_frontend(repl::StreamREPL, backend::REPLBackendRef) if have_color print(repl.stream, Base.color_normal) end - response = eval_with_backend(ast, backend) + response = eval_on_backend(ast, backend) print_response(repl, response, !ends_with_semicolon(line), have_color) end end @@ -1869,7 +1866,7 @@ function create_global_out!(mod) end return out end - return getglobal(mod, Out) + return getglobal(mod, :Out) end function capture_result(n::Ref{Int}, @nospecialize(x)) @@ -1886,10 +1883,10 @@ end function set_prompt(repl::LineEditREPL, n::Ref{Int}) julia_prompt = repl.interface.modes[1] - julia_prompt.prompt = function() + julia_prompt.prompt = REPL.contextual_prompt(repl, function() n[] = repl_eval_counter(julia_prompt.hist)+1 string("In [", n[], "]: ") - end + end) nothing end diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index d4e33e8ff5136..4179819e47f1d 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -12,8 +12,10 @@ const CC = Base.Compiler using Base.Meta using Base: propertynames, something, IdSet using Base.Filesystem: _readdirx +using Base.JuliaSyntax: @K_str, @KSet_str, parseall, byte_range, children, is_prefix_call, is_trivia, kind using ..REPL.LineEdit: NamedCompletion +using ..REPL.SyntaxUtil: CursorNode, find_parent, seek_pos, char_range, char_last, children_nt, find_delim abstract type Completion end @@ -184,7 +186,7 @@ function complete_symbol!(suggestions::Vector{Completion}, complete_modules_only::Bool=false, shift::Bool=false) local mod, t, val - complete_internal_only = false + complete_internal_only = isempty(name) if prefix !== nothing res = repl_eval_ex(prefix, context_module) res === nothing && return Completion[] @@ -302,9 +304,8 @@ const sorted_keyvals = ["false", "true"] complete_keyval!(suggestions::Vector{Completion}, s::String) = complete_from_list!(suggestions, KeyvalCompletion, sorted_keyvals, s) -function do_raw_escape(s) - # escape_raw_string with delim='`' and ignoring the rule for the ending \ - return replace(s, r"(\\+)`" => s"\1\\`") +function do_cmd_escape(s) + return Base.escape_raw_string(Base.shell_escape_posixly(s), '`') end function do_shell_escape(s) return Base.shell_escape_posixly(s) @@ -312,21 +313,46 @@ end function do_string_escape(s) return escape_string(s, ('\"','$')) end +function do_string_unescape(s) + s = replace(s, "\\\$"=>"\$") + try + unescape_string(s) + catch e + e isa ArgumentError || rethrow() + s # it is unlikely, but if it isn't a valid string, maybe it was a valid path, and just needs escape_string called? + end +end + +function joinpath_withsep(dir, path; dirsep) + dir == "" && return path + dir[end] == dirsep ? dir * path : dir * dirsep * path +end const PATH_cache_lock = Base.ReentrantLock() const PATH_cache = Set{String}() -PATH_cache_task::Union{Task,Nothing} = nothing # used for sync in tests +PATH_cache_task::Union{Task,Nothing} = nothing +PATH_cache_condition::Union{Threads.Condition, Nothing} = nothing # used for sync in tests next_cache_update::Float64 = 0.0 function maybe_spawn_cache_PATH() - global PATH_cache_task, next_cache_update + global PATH_cache_task, PATH_cache_condition, next_cache_update @lock PATH_cache_lock begin - PATH_cache_task isa Task && !istaskdone(PATH_cache_task) && return + # Extract to local variables to enable flow-sensitive type inference for these global variables + PATH_cache_task_local = PATH_cache_task + PATH_cache_task_local isa Task && !istaskdone(PATH_cache_task_local) && return time() < next_cache_update && return - PATH_cache_task = Threads.@spawn begin - REPLCompletions.cache_PATH() - @lock PATH_cache_lock PATH_cache_task = nothing # release memory when done + PATH_cache_task = PATH_cache_task_local = Threads.@spawn begin + try + REPLCompletions.cache_PATH() + finally + @lock PATH_cache_lock begin + next_cache_update = time() + 10 # earliest next update can run is 10s after + PATH_cache_task = nothing # release memory when done + PATH_cache_condition_local = PATH_cache_condition + PATH_cache_condition_local !== nothing && notify(PATH_cache_condition_local) + end + end end - Base.errormonitor(PATH_cache_task) + Base.errormonitor(PATH_cache_task_local) end end @@ -335,8 +361,6 @@ function cache_PATH() path = get(ENV, "PATH", nothing) path isa String || return - global next_cache_update - # Calling empty! on PATH_cache would be annoying for async typing hints as completions would temporarily disappear. # So keep track of what's added this time and at the end remove any that didn't appear this time from the global cache. this_PATH_cache = Set{String}() @@ -399,7 +423,6 @@ function cache_PATH() @lock PATH_cache_lock begin intersect!(PATH_cache, this_PATH_cache) # remove entries from PATH_cache that weren't found this time - next_cache_update = time() + 10 # earliest next update can run is 10s after end @debug "caching PATH files took $t seconds" length(pathdirs) length(PATH_cache) @@ -409,9 +432,10 @@ end function complete_path(path::AbstractString; use_envpath=false, shell_escape=false, - raw_escape=false, + cmd_escape=false, string_escape=false, - contract_user=false) + contract_user=false, + dirsep=Sys.iswindows() ? '\\' : '/') @assert !(shell_escape && string_escape) if Base.Sys.isunix() && occursin(r"^~(?:/|$)", path) # if the path is just "~", don't consider the expanded username as a prefix @@ -440,7 +464,7 @@ function complete_path(path::AbstractString; for entry in entries if startswith(entry.name, prefix) is_dir = try isdir(entry) catch ex; ex isa Base.IOError ? false : rethrow() end - push!(matches, is_dir ? entry.name * "/" : entry.name) + push!(matches, is_dir ? joinpath_withsep(entry.name, ""; dirsep) : entry.name) end end @@ -456,7 +480,7 @@ function complete_path(path::AbstractString; end matches = ((shell_escape ? do_shell_escape(s) : string_escape ? do_string_escape(s) : s) for s in matches) - matches = ((raw_escape ? do_raw_escape(s) : s) for s in matches) + matches = ((cmd_escape ? do_cmd_escape(s) : s) for s in matches) matches = Completion[PathCompletion(contract_user ? contractuser(s) : s) for s in matches] return matches, dir, !isempty(matches) end @@ -469,7 +493,8 @@ function complete_path(path::AbstractString, contract_user=false) ## TODO: enable this depwarn once Pkg is fixed #Base.depwarn("complete_path with pos argument is deprecated because the return value [2] is incorrect to use", :complete_path) - paths, dir, success = complete_path(path; use_envpath, shell_escape, string_escape) + paths, dir, success = complete_path(path; use_envpath, shell_escape, string_escape, dirsep='/') + if Base.Sys.isunix() && occursin(r"^~(?:/|$)", path) # if the path is just "~", don't consider the expanded username as a prefix if path == "~" @@ -490,91 +515,6 @@ function complete_path(path::AbstractString, return paths, startpos:pos, success end -function complete_expanduser(path::AbstractString, r) - expanded = - try expanduser(path) - catch e - e isa ArgumentError || rethrow() - path - end - return Completion[PathCompletion(expanded)], r, path != expanded -end - -# Returns a range that includes the method name in front of the first non -# closed start brace from the end of the string. -function find_start_brace(s::AbstractString; c_start='(', c_end=')') - r = reverse(s) - i = firstindex(r) - braces = in_comment = 0 - in_single_quotes = in_double_quotes = in_back_ticks = false - num_single_quotes_in_string = count('\'', s) - while i <= ncodeunits(r) - c, i = iterate(r, i) - if c == '#' && i <= ncodeunits(r) && iterate(r, i)[1] == '=' - c, i = iterate(r, i) # consume '=' - new_comments = 1 - # handle #=#=#=#, by counting =# pairs - while i <= ncodeunits(r) && iterate(r, i)[1] == '#' - c, i = iterate(r, i) # consume '#' - iterate(r, i)[1] == '=' || break - c, i = iterate(r, i) # consume '=' - new_comments += 1 - end - if c == '=' - in_comment += new_comments - else - in_comment -= new_comments - end - elseif !in_single_quotes && !in_double_quotes && !in_back_ticks && in_comment == 0 - if c == c_start - braces += 1 - elseif c == c_end - braces -= 1 - elseif c == '\'' && num_single_quotes_in_string % 2 == 0 - # ' can be a transpose too, so check if there are even number of 's in the string - # TODO: This probably needs to be more robust - in_single_quotes = true - elseif c == '"' - in_double_quotes = true - elseif c == '`' - in_back_ticks = true - end - else - if in_single_quotes && - c == '\'' && i <= ncodeunits(r) && iterate(r, i)[1] != '\\' - in_single_quotes = false - elseif in_double_quotes && - c == '"' && i <= ncodeunits(r) && iterate(r, i)[1] != '\\' - in_double_quotes = false - elseif in_back_ticks && - c == '`' && i <= ncodeunits(r) && iterate(r, i)[1] != '\\' - in_back_ticks = false - elseif in_comment > 0 && - c == '=' && i <= ncodeunits(r) && iterate(r, i)[1] == '#' - # handle =#=#=#=, by counting #= pairs - c, i = iterate(r, i) # consume '#' - old_comments = 1 - while i <= ncodeunits(r) && iterate(r, i)[1] == '=' - c, i = iterate(r, i) # consume '=' - iterate(r, i)[1] == '#' || break - c, i = iterate(r, i) # consume '#' - old_comments += 1 - end - if c == '#' - in_comment -= old_comments - else - in_comment += old_comments - end - end - end - braces == 1 && break - end - braces != 1 && return 0:-1, -1 - method_name_end = reverseind(s, i) - startind = nextind(s, something(findprev(in(non_identifier_chars), s, method_name_end), 0))::Int - return (startind:lastindex(s), method_name_end) -end - struct REPLCacheToken end struct REPLInterpreter <: CC.AbstractInterpreter @@ -631,18 +571,18 @@ CC.bail_out_toplevel_call(::REPLInterpreter, ::CC.InferenceLoopState, ::CC.Infer # be employed, for instance, by `typeinf_ext_toplevel`. is_repl_frame(sv::CC.InferenceState) = sv.linfo.def isa Module && sv.cache_mode === CC.CACHE_MODE_NULL -function is_call_graph_uncached(sv::CC.InferenceState) +function is_call_stack_uncached(sv::CC.InferenceState) CC.is_cached(sv) && return false parent = CC.frame_parent(sv) parent === nothing && return true - return is_call_graph_uncached(parent::CC.InferenceState) + return is_call_stack_uncached(parent::CC.InferenceState) end # aggressive global binding resolution within `repl_frame` function CC.abstract_eval_globalref(interp::REPLInterpreter, g::GlobalRef, bailed::Bool, sv::CC.InferenceState) # Ignore saw_latestworld - if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_graph_uncached(sv)) + if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_stack_uncached(sv)) partition = CC.abstract_eval_binding_partition!(interp, g, sv) if CC.is_defined_const_binding(CC.binding_kind(partition)) return CC.RTEffects(Const(CC.partition_restriction(partition)), Union{}, CC.EFFECTS_TOTAL) @@ -666,33 +606,11 @@ function is_repl_frame_getproperty(sv::CC.InferenceState) return is_repl_frame(CC.frame_parent(sv)) end -# aggressive global binding resolution for `getproperty(::Module, ::Symbol)` calls within `repl_frame` -function CC.builtin_tfunction(interp::REPLInterpreter, @nospecialize(f), - argtypes::Vector{Any}, sv::CC.InferenceState) - if f === Core.getglobal && (interp.limit_aggressive_inference ? is_repl_frame_getproperty(sv) : is_call_graph_uncached(sv)) - if length(argtypes) == 2 - a1, a2 = argtypes - if isa(a1, Const) && isa(a2, Const) - a1val, a2val = a1.val, a2.val - if isa(a1val, Module) && isa(a2val, Symbol) - g = GlobalRef(a1val, a2val) - if isdefined_globalref(g) - return Const(ccall(:jl_get_globalref_value, Any, (Any,), g)) - end - return Union{} - end - end - end - end - return @invoke CC.builtin_tfunction(interp::CC.AbstractInterpreter, f::Any, - argtypes::Vector{Any}, sv::CC.InferenceState) -end - # aggressive concrete evaluation for `:inconsistent` frames within `repl_frame` function CC.concrete_eval_eligible(interp::REPLInterpreter, @nospecialize(f), result::CC.MethodCallResult, arginfo::CC.ArgInfo, sv::CC.InferenceState) - if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_graph_uncached(sv)) + if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_stack_uncached(sv)) neweffects = CC.Effects(result.effects; consistent=CC.ALWAYS_TRUE) result = CC.MethodCallResult(result.rt, result.exct, neweffects, result.edge, result.edgecycle, result.edgelimited, result.volatile_inf_result) @@ -729,6 +647,7 @@ end # lower `ex` and run type inference on the resulting top-level expression function repl_eval_ex(@nospecialize(ex), context_module::Module; limit_aggressive_inference::Bool=false) + expr_has_error(ex) && return nothing if (isexpr(ex, :toplevel) || isexpr(ex, :tuple)) && !isempty(ex.args) # get the inference result for the last expression ex = ex.args[end] @@ -778,6 +697,16 @@ code_typed(CC.typeinf, (REPLInterpreter, CC.InferenceState)) # Method completion on function call expression that look like :(max(1)) MAX_METHOD_COMPLETIONS::Int = 40 function _complete_methods(ex_org::Expr, context_module::Module, shift::Bool) + isempty(ex_org.args) && return 2, nothing, [], Set{Symbol}() + # Desugar do block call into call with lambda + if ex_org.head === :do && length(ex_org.args) >= 2 + ex_call = ex_org.args[1] + ex_args = [x for x in ex_call.args if !(x isa Expr && x.head === :parameters)] + ex_params = findfirst(x -> x isa Expr && x.head === :parameters, ex_call.args) + new_args = [ex_args[1], ex_org.args[end], ex_args[2:end]...] + ex_params !== nothing && push!(new_args, ex_call.args[ex_params]) + ex_org = Expr(:call, new_args...) + end funct = repl_eval_ex(ex_org.args[1], context_module) funct === nothing && return 2, nothing, [], Set{Symbol}() funct = CC.widenconst(funct) @@ -785,9 +714,12 @@ function _complete_methods(ex_org::Expr, context_module::Module, shift::Bool) return kwargs_flag, funct, args_ex, kwargs_ex end -function complete_methods(ex_org::Expr, context_module::Module=Main, shift::Bool=false) +# cursor_pos: either :positional (complete either kwargs or positional) or :kwargs (beyond semicolon) +function complete_methods(ex_org::Expr, context_module::Module=Main, shift::Bool=false, cursor_pos::Symbol=:positional) kwargs_flag, funct, args_ex, kwargs_ex = _complete_methods(ex_org, context_module, shift)::Tuple{Int, Any, Vector{Any}, Set{Symbol}} out = Completion[] + # Allow more arguments when cursor before semicolon, even if kwargs are present + cursor_pos == :positional && kwargs_flag == 1 && (kwargs_flag = 0) kwargs_flag == 2 && return out # one of the kwargs is invalid kwargs_flag == 0 && push!(args_ex, Vararg{Any}) # allow more arguments if there is no semicolon complete_methods!(out, funct, args_ex, kwargs_ex, shift ? -2 : MAX_METHOD_COMPLETIONS, kwargs_flag == 1) @@ -795,24 +727,12 @@ function complete_methods(ex_org::Expr, context_module::Module=Main, shift::Bool end MAX_ANY_METHOD_COMPLETIONS::Int = 10 -function recursive_explore_names!(seen::IdSet, callee_module::Module, initial_module::Module, exploredmodules::IdSet{Module}=IdSet{Module}()) - push!(exploredmodules, callee_module) - for name in names(callee_module; all=true, imported=true) - if !Base.isdeprecated(callee_module, name) && !startswith(string(name), '#') && isdefined(initial_module, name) - func = getfield(callee_module, name) - if !isa(func, Module) - funct = Core.Typeof(func) - push!(seen, funct) - elseif isa(func, Module) && func ∉ exploredmodules - recursive_explore_names!(seen, func, initial_module, exploredmodules) - end - end - end -end -function recursive_explore_names(callee_module::Module, initial_module::Module) - seen = IdSet{Any}() - recursive_explore_names!(seen, callee_module, initial_module) - seen + +function accessible(mod::Module, private::Bool) + bindings = IdSet{Any}(Core.Typeof(getglobal(mod, s)) for s in names(mod; all=private, imported=private, usings=private) + if !Base.isdeprecated(mod, s) && !startswith(string(s), '#') && !startswith(string(s), '@') && isdefined(mod, s)) + delete!(bindings, Module) + return collect(bindings) end function complete_any_methods(ex_org::Expr, callee_module::Module, context_module::Module, moreargs::Bool, shift::Bool) @@ -830,7 +750,7 @@ function complete_any_methods(ex_org::Expr, callee_module::Module, context_modul # semicolon for the ".?(" syntax moreargs && push!(args_ex, Vararg{Any}) - for seen_name in recursive_explore_names(callee_module, callee_module) + for seen_name in accessible(callee_module, callee_module === context_module) complete_methods!(out, seen_name, args_ex, kwargs_ex, MAX_ANY_METHOD_COMPLETIONS, false) end @@ -935,50 +855,6 @@ const subscript_regex = Regex("^\\\\_[" * join(isdigit(k) || isletter(k) ? "$k" const superscripts = Dict(k[3]=>v[1] for (k,v) in latex_symbols if startswith(k, "\\^") && length(k)==3) const superscript_regex = Regex("^\\\\\\^[" * join(isdigit(k) || isletter(k) ? "$k" : "\\$k" for k in keys(superscripts)) * "]+\\z") -# Aux function to detect whether we're right after a using or import keyword -function get_import_mode(s::String) - # allow all of these to start with leading whitespace and macros like @eval and @eval( - # ^\s*(?:@\w+\s*(?:\(\s*)?)? - - # match simple cases like `using |` and `import |` - mod_import_match_simple = match(r"^\s*(?:@\w+\s*(?:\(\s*)?)?\b(using|import)\s*$", s) - if mod_import_match_simple !== nothing - if mod_import_match_simple[1] == "using" - return :using_module - else - return :import_module - end - end - # match module import statements like `using Foo|`, `import Foo, Bar|` and `using Foo.Bar, Baz, |` - mod_import_match = match(r"^\s*(?:@\w+\s*(?:\(\s*)?)?\b(using|import)\s+([\w\.]+(?:\s*,\s*[\w\.]+)*),?\s*$", s) - if mod_import_match !== nothing - if mod_import_match.captures[1] == "using" - return :using_module - else - return :import_module - end - end - # now match explicit name import statements like `using Foo: |` and `import Foo: bar, baz|` - name_import_match = match(r"^\s*(?:@\w+\s*(?:\(\s*)?)?\b(using|import)\s+([\w\.]+)\s*:\s*([\w@!\s,]+)$", s) - if name_import_match !== nothing - if name_import_match[1] == "using" - return :using_name - else - return :import_name - end - end - return nothing -end - -function close_path_completion(dir, path, str, pos) - path = unescape_string(replace(path, "\\\$"=>"\$")) - path = joinpath(dir, path) - # ...except if it's a directory... - Base.isaccessibledir(path) && return false - # ...and except if there's already a " at the cursor. - return lastindex(str) <= pos || str[nextind(str, pos)] != '"' -end - function bslash_completions(string::String, pos::Int, hint::Bool=false) slashpos = something(findprev(isequal('\\'), string, pos), 0) if (something(findprev(in(bslash_separators), string, pos), 0) < slashpos && @@ -1006,29 +882,7 @@ function bslash_completions(string::String, pos::Int, hint::Bool=false) completions = Completion[BslashCompletion(name, "$(symbol_dict[name]) $name") for name in sort!(collect(namelist))] return (true, (completions, slashpos:pos, true)) end - return (false, (Completion[], 0:-1, false)) -end - -function dict_identifier_key(str::String, tag::Symbol, context_module::Module=Main) - if tag === :string - str_close = str*"\"" - elseif tag === :cmd - str_close = str*"`" - else - str_close = str - end - frange, end_of_identifier = find_start_brace(str_close, c_start='[', c_end=']') - isempty(frange) && return (nothing, nothing, nothing) - objstr = str[1:end_of_identifier] - objex = Meta.parse(objstr, raise=false, depwarn=false) - objt = repl_eval_ex(objex, context_module) - isa(objt, Core.Const) || return (nothing, nothing, nothing) - obj = objt.val - isa(obj, AbstractDict) || return (nothing, nothing, nothing) - (Base.haslength(obj) && length(obj)::Int < 1_000_000) || return (nothing, nothing, nothing) - begin_of_key = something(findnext(!isspace, str, nextind(str, end_of_identifier) + 1), # +1 for [ - lastindex(str)+1) - return (obj, str[begin_of_key:end], begin_of_key) + return (false, (Completion[], 1:0, false)) end # This needs to be a separate non-inlined function, see #19441 @@ -1041,48 +895,23 @@ end return matches end -# Identify an argument being completed in a method call. If the argument is empty, method -# suggestions will be provided instead of argument completions. -function identify_possible_method_completion(partial, last_idx) - fail = 0:-1, Expr(:nothing), 0:-1, 0 - - # First, check that the last punctuation is either ',', ';' or '(' - idx_last_punct = something(findprev(x -> ispunct(x) && x != '_' && x != '!', partial, last_idx), 0)::Int - idx_last_punct == 0 && return fail - last_punct = partial[idx_last_punct] - last_punct == ',' || last_punct == ';' || last_punct == '(' || return fail - - # Then, check that `last_punct` is only followed by an identifier or nothing - before_last_word_start = something(findprev(in(non_identifier_chars), partial, last_idx), 0) - before_last_word_start == 0 && return fail - all(isspace, @view partial[nextind(partial, idx_last_punct):before_last_word_start]) || return fail - - # Check that `last_punct` is either the last '(' or placed after a previous '(' - frange, method_name_end = find_start_brace(@view partial[1:idx_last_punct]) - method_name_end ∈ frange || return fail - - # Strip the preceding ! operators, if any, and close the expression with a ')' - s = replace(partial[frange], r"\G\!+([^=\(]+)" => s"\1"; count=1) * ')' - ex = Meta.parse(s, raise=false, depwarn=false) - isa(ex, Expr) || return fail - - # `wordrange` is the position of the last argument to complete - wordrange = nextind(partial, before_last_word_start):last_idx - return frange, ex, wordrange, method_name_end -end - # Provide completion for keyword arguments in function calls -function complete_keyword_argument(partial::String, last_idx::Int, context_module::Module; - shift::Bool=false) - frange, ex, wordrange, = identify_possible_method_completion(partial, last_idx) - fail = Completion[], 0:-1, frange - ex.head === :call || is_broadcasting_expr(ex) || return fail - +# Returns true if the current argument must be a keyword because the cursor is beyond the semicolon +function complete_keyword_argument!(suggestions::Vector{Completion}, + ex::Expr, last_word::String, + context_module::Module, + arg_pos::Symbol; shift::Bool=false) kwargs_flag, funct, args_ex, kwargs_ex = _complete_methods(ex, context_module, true)::Tuple{Int, Any, Vector{Any}, Set{Symbol}} - kwargs_flag == 2 && return fail # one of the previous kwargs is invalid + kwargs_flag == 2 && return false # one of the previous kwargs is invalid methods = Completion[] - complete_methods!(methods, funct, Any[Vararg{Any}], kwargs_ex, shift ? -1 : MAX_METHOD_COMPLETIONS, kwargs_flag == 1) + # Limit kwarg completions to cases when function is concretely known; looking up + # matching methods for abstract functions — particularly `Any` or `Function` — can + # take many seconds to run over the thousands of possible methods. Note that + # isabstracttype would return naively return true for common constructor calls + # like Array, but the REPL's introspection here may know their Type{T}. + isconcretetype(funct) || return false + complete_methods!(methods, funct, Any[Vararg{Any}], kwargs_ex, -1, arg_pos == :kwargs) # TODO: use args_ex instead of Any[Vararg{Any}] and only provide kwarg completion for # method calls compatible with the current arguments. @@ -1091,7 +920,6 @@ function complete_keyword_argument(partial::String, last_idx::Int, context_modul # previously in the expression. The corresponding suggestion is "kwname=". # If the keyword corresponds to an existing name, also include "kwname" as a suggestion # since the syntax "foo(; kwname)" is equivalent to "foo(; kwname=kwname)". - last_word = partial[wordrange] # the word to complete kwargs = Set{String}() for m in methods # if MAX_METHOD_COMPLETIONS is hit a single TextCompletion is return by complete_methods! with an explanation @@ -1102,22 +930,18 @@ function complete_keyword_argument(partial::String, last_idx::Int, context_modul current_kwarg_candidates = String[] for _kw in possible_kwargs kw = String(_kw) - if !endswith(kw, "...") && startswith(kw, last_word) && _kw ∉ kwargs_ex + # HACK: Should consider removing current arg from AST. + if !endswith(kw, "...") && startswith(kw, last_word) && (_kw ∉ kwargs_ex || kw == last_word) push!(current_kwarg_candidates, kw) end end union!(kwargs, current_kwarg_candidates) end - suggestions = Completion[KeywordArgumentCompletion(kwarg) for kwarg in kwargs] - - # Only add these if not in kwarg space. i.e. not in `foo(; ` - if kwargs_flag == 0 - complete_symbol!(suggestions, #=prefix=#nothing, last_word, context_module; shift) - complete_keyval!(suggestions, last_word) + for kwarg in kwargs + push!(suggestions, KeywordArgumentCompletion(kwarg)) end - - return sort!(suggestions, by=named_completion_completion), wordrange + return kwargs_flag != 0 && arg_pos == :kwargs end function get_loading_candidates(pkgstarts::String, project_file::String) @@ -1136,411 +960,372 @@ function get_loading_candidates(pkgstarts::String, project_file::String) return loading_candidates end -function complete_loading_candidates!(suggestions::Vector{Completion}, pkgstarts::String, project_file::String) - for name in get_loading_candidates(pkgstarts, project_file) - push!(suggestions, PackageCompletion(name)) +function complete_loading_candidates!(suggestions::Vector{Completion}, s::String) + for name in ("Core", "Base") + startswith(name, s) && push!(suggestions, PackageCompletion(name)) end - return suggestions -end -function complete_identifiers!(suggestions::Vector{Completion}, - context_module::Module, string::String, name::String, - pos::Int, separatorpos::Int, startpos::Int; - comp_keywords::Bool=false, - complete_modules_only::Bool=false, - shift::Bool=false) - if comp_keywords - complete_keyword!(suggestions, name) - complete_keyval!(suggestions, name) - end - if separatorpos > 1 && (string[separatorpos] == '.' || string[separatorpos] == ':') - s = string[1:prevind(string, separatorpos)] - # First see if the whole string up to `pos` is a valid expression. If so, use it. - prefix = Meta.parse(s, raise=false, depwarn=false) - if isexpr(prefix, :incomplete) - s = string[startpos:pos] - # Heuristic to find the start of the expression. TODO: This would be better - # done with a proper error-recovering parser. - if 0 < startpos <= lastindex(string) && string[startpos] == '.' - i = prevind(string, startpos) - while 0 < i - c = string[i] - if c in (')', ']') - if c == ')' - c_start = '(' - c_end = ')' - elseif c == ']' - c_start = '[' - c_end = ']' - end - frange, end_of_identifier = find_start_brace(string[1:prevind(string, i)], c_start=c_start, c_end=c_end) - isempty(frange) && break # unbalanced parens - startpos = first(frange) - i = prevind(string, startpos) - elseif c in ('\'', '\"', '\`') - s = "$c$c"*string[startpos:pos] - break + # If there's no dot, we're in toplevel, so we should + # also search for packages + for dir in Base.load_path() + if basename(dir) in Base.project_names && isfile(dir) + for name in get_loading_candidates(s, dir) + push!(suggestions, PackageCompletion(name)) + end + end + isdir(dir) || continue + for entry in _readdirx(dir) + pname = entry.name + if pname[1] != '.' && pname != "METADATA" && + pname != "REQUIRE" && startswith(pname, s) + # Valid file paths are + # .jl + # /src/.jl + # .jl/src/.jl + if isfile(entry) + endswith(pname, ".jl") && push!(suggestions, + PackageCompletion(pname[1:prevind(pname, end-2)])) + else + mod_name = if endswith(pname, ".jl") + pname[1:prevind(pname, end-2)] else - break + pname end - s = string[startpos:pos] - end - end - if something(findlast(in(non_identifier_chars), s), 0) < something(findlast(isequal('.'), s), 0) - lookup_name, name = rsplit(s, ".", limit=2) - name = String(name) - prefix = Meta.parse(lookup_name, raise=false, depwarn=false) - end - isexpr(prefix, :incomplete) && (prefix = nothing) - elseif isexpr(prefix, (:using, :import)) - arglast = prefix.args[end] # focus on completion to the last argument - if isexpr(arglast, :.) - # We come here for cases like: - # - `string`: "using Mod1.Mod2.M" - # - `ex`: :(using Mod1.Mod2) - # - `name`: "M" - # Now we transform `ex` to `:(Mod1.Mod2)` to allow `complete_symbol!` to - # complete for inner modules whose name starts with `M`. - # Note that `complete_modules_only=true` is set within `completions` - prefix = nothing - firstdot = true - for arg = arglast.args - if arg === :. - # override `context_module` if multiple `.` accessors are used - if firstdot - firstdot = false - else - context_module = parentmodule(context_module) - end - elseif arg isa Symbol - if prefix === nothing - prefix = arg - else - prefix = Expr(:., prefix, QuoteNode(arg)) - end - else # invalid expression - prefix = nothing - break + if isfile(joinpath(entry, "src", + "$mod_name.jl")) + push!(suggestions, PackageCompletion(mod_name)) end end end - elseif isexpr(prefix, :call) && length(prefix.args) > 1 - isinfix = s[end] != ')' - # A complete call expression that does not finish with ')' is an infix call. - if !isinfix - # Handle infix call argument completion of the form bar + foo(qux). - frange, end_of_identifier = find_start_brace(@view s[1:prevind(s, end)]) - if !isempty(frange) # if find_start_brace fails to find the brace just continue - isinfix = Meta.parse(@view(s[frange[1]:end]), raise=false, depwarn=false) == prefix.args[end] - end - end - if isinfix - prefix = prefix.args[end] - end - elseif isexpr(prefix, :macrocall) && length(prefix.args) > 1 - # allow symbol completions within potentially incomplete macrocalls - if s[end] ≠ '`' && s[end] ≠ ')' - prefix = prefix.args[end] - end end - else - prefix = nothing end - complete_symbol!(suggestions, prefix, name, context_module; complete_modules_only, shift) - return suggestions end function completions(string::String, pos::Int, context_module::Module=Main, shift::Bool=true, hint::Bool=false) - # First parse everything up to the current position - partial = string[1:pos] - inc_tag = Base.incomplete_tag(Meta.parse(partial, raise=false, depwarn=false)) - - if !hint # require a tab press for completion of these - # ?(x, y)TAB lists methods you can call with these objects - # ?(x, y TAB lists methods that take these objects as the first two arguments - # MyModule.?(x, y)TAB restricts the search to names in MyModule - rexm = match(r"(\w+\.|)\?\((.*)$", partial) - if rexm !== nothing - # Get the module scope - if isempty(rexm.captures[1]) - callee_module = context_module - else - modname = Symbol(rexm.captures[1][1:end-1]) - if isdefined(context_module, modname) - callee_module = getfield(context_module, modname) - if !isa(callee_module, Module) - callee_module = context_module - end - else - callee_module = context_module - end - end - moreargs = !endswith(rexm.captures[2], ')') - callstr = "_(" * rexm.captures[2] - if moreargs - callstr *= ')' - end - ex_org = Meta.parse(callstr, raise=false, depwarn=false) - if isa(ex_org, Expr) - return complete_any_methods(ex_org, callee_module::Module, context_module, moreargs, shift), (0:length(rexm.captures[1])+1) .+ rexm.offset, false - end - end - end + # filename needs to be string so macro can be evaluated + node = parseall(CursorNode, string, ignore_errors=true, keep_parens=true, filename="none") + cur = @something seek_pos(node, pos) node - # if completing a key in a Dict - identifier, partial_key, loc = dict_identifier_key(partial, inc_tag, context_module) - if identifier !== nothing - matches = find_dict_matches(identifier, partial_key) - length(matches)==1 && (lastindex(string) <= pos || string[nextind(string,pos)] != ']') && (matches[1]*=']') - length(matches)>0 && return Completion[DictCompletion(identifier, match) for match in sort!(matches)], loc::Int:pos, true - end + # Back up before whitespace to get a more useful AST node. + pos_not_ws = findprev(!isspace, string, pos) + cur_not_ws = something(seek_pos(node, pos_not_ws), node) suggestions = Completion[] + sort_suggestions() = sort!(unique!(named_completion, suggestions), by=named_completion_completion) + + # Search for methods (requires tab press): + # ?(x, y)TAB lists methods you can call with these objects + # ?(x, y TAB lists methods that take these objects as the first two arguments + # MyModule.?(x, y)TAB restricts the search to names in MyModule + if !hint + cs = method_search(view(string, 1:pos), context_module, shift) + cs !== nothing && return cs + end - # Check if this is a var"" string macro that should be completed like - # an identifier rather than a string. - # TODO: It would be nice for the parser to give us more information here - # so that we can lookup the macro by identity rather than pattern matching - # its invocation. - varrange = findprev("var\"", string, pos) - - expanded = nothing - was_expanded = false - - if varrange !== nothing - ok, ret = bslash_completions(string, pos) - ok && return ret - startpos = first(varrange) + 4 - separatorpos = something(findprev(isequal('.'), string, first(varrange)-1), 0) - name = string[startpos:pos] - complete_identifiers!(suggestions, context_module, string, name, - pos, separatorpos, startpos; - shift) - return sort!(unique!(named_completion, suggestions), by=named_completion_completion), (separatorpos+1):pos, true - elseif inc_tag === :cmd - # TODO: should this call shell_completions instead of partially reimplementing it? - let m = match(r"[\t\n\r\"`><=*?|]| (?!\\)", reverse(partial)) # fuzzy shell_parse in reverse - startpos = nextind(partial, reverseind(partial, m.offset)) - r = startpos:pos - scs::String = string[r] - - expanded = complete_expanduser(scs, r) - was_expanded = expanded[3] - if was_expanded - scs = (only(expanded[1])::PathCompletion).path - # If tab press, ispath and user expansion available, return it now - # otherwise see if we can complete the path further before returning with expanded ~ - !hint && ispath(scs) && return expanded::Completions - end - - path::String = replace(scs, r"(\\+)\g1(\\?)`" => "\1\2`") # fuzzy unescape_raw_string: match an even number of \ before ` and replace with half as many - # This expansion with "\\ "=>' ' replacement and shell_escape=true - # assumes the path isn't further quoted within the cmd backticks. - path = replace(path, r"\\ " => " ", r"\$" => "\$") # fuzzy shell_parse (reversed by shell_escape_posixly) - paths, dir, success = complete_path(path, shell_escape=true, raw_escape=true) - - if success && !isempty(dir) - let dir = do_raw_escape(do_shell_escape(dir)) - # if escaping of dir matches scs prefix, remove that from the completions - # otherwise make it the whole completion - if endswith(dir, "/") && startswith(scs, dir) - r = (startpos + sizeof(dir)):pos - elseif startswith(scs, dir * "/") - r = nextind(string, startpos + sizeof(dir)):pos - else - map!(paths, paths) do c::PathCompletion - p = dir * "/" * c.path - was_expanded && (p = contractuser(p)) - return PathCompletion(p) - end - end - end - end - if isempty(paths) && !hint && was_expanded - # if not able to provide completions, not hinting, and ~ expansion was possible, return ~ expansion - return expanded::Completions - else - return sort!(paths, by=p->p.path), r::UnitRange{Int}, success + # Complete keys in a Dict: + # my_dict[ TAB + n, key, closed = find_ref_key(cur_not_ws, pos) + if n !== nothing + key::UnitRange{Int} + obj = dict_eval(Expr(n), context_module) + if obj !== nothing + # Skip leading whitespace inside brackets. + i = @something findnext(!isspace, string, first(key)) nextind(string, last(key)) + key = i:last(key) + s = string[intersect(key, 1:pos)] + matches = find_dict_matches(obj, s) + length(matches) == 1 && !closed && (matches[1] *= ']') + if length(matches) > 0 + ret = Completion[DictCompletion(obj, match) for match in sort!(matches)] + return ret, key, true end end - elseif inc_tag === :string - # Find first non-escaped quote - let m = match(r"\"(?!\\)", reverse(partial)) - startpos = nextind(partial, reverseind(partial, m.offset)) - r = startpos:pos - scs::String = string[r] - - expanded = complete_expanduser(scs, r) - was_expanded = expanded[3] - if was_expanded - scs = (only(expanded[1])::PathCompletion).path - # If tab press, ispath and user expansion available, return it now - # otherwise see if we can complete the path further before returning with expanded ~ - !hint && ispath(scs) && return expanded::Completions - end - - path = try - unescape_string(replace(scs, "\\\$"=>"\$")) - catch ex - ex isa ArgumentError || rethrow() - nothing - end - if !isnothing(path) - paths, dir, success = complete_path(path::String, string_escape=true) - - if length(paths) == 1 - p = (paths[1]::PathCompletion).path - hint && was_expanded && (p = contractuser(p)) - if close_path_completion(dir, p, path, pos) - paths[1] = PathCompletion(p * "\"") - end - end + end - if success && !isempty(dir) - let dir = do_string_escape(dir) - # if escaping of dir matches scs prefix, remove that from the completions - # otherwise make it the whole completion - if endswith(dir, "/") && startswith(scs, dir) - r = (startpos + sizeof(dir)):pos - elseif startswith(scs, dir * "/") && dir != dirname(homedir()) - was_expanded && (dir = contractuser(dir)) - r = nextind(string, startpos + sizeof(dir)):pos - else - map!(paths, paths) do c::PathCompletion - p = dir * "/" * c.path - hint && was_expanded && (p = contractuser(p)) - return PathCompletion(p) - end - end - end - end + # Complete Cmd strings: + # `fil TAB => `file + # `file ~/exa TAB => `file ~/example.txt + # `file ~/example.txt TAB => `file /home/user/example.txt + if (n = find_parent(cur, K"CmdString")) !== nothing + off = n.position - 1 + ret, r, success = shell_completions(string[char_range(n)], pos - off, hint, cmd_escape=true) + success && return ret, r .+ off, success + end - # Fallthrough allowed so that Latex symbols can be completed in strings - if success - return sort!(paths, by=p->p.path), r::UnitRange{Int}, success - elseif !hint && was_expanded - # if not able to provide completions, not hinting, and ~ expansion was possible, return ~ expansion - return expanded::Completions - end - end + # Complete ordinary strings: + # "~/exa TAB => "~/example.txt" + # "~/example.txt TAB => "/home/user/example.txt" + r, closed = find_str(cur) + if r !== nothing + s = do_string_unescape(string[r]) + ret, success = complete_path_string(s, hint; string_escape=true, + dirsep=Sys.iswindows() ? '\\' : '/') + if length(ret) == 1 && !closed && close_path_completion(ret[1].path) + ret[1] = PathCompletion(ret[1].path * '"') end + success && return ret, r, success end - # if path has ~ and we didn't find any paths to complete just return the expanded path - was_expanded && return expanded::Completions + # Backlash symbols: + # \pi => π + # Comes after string completion so backslash escapes are not misinterpreted. ok, ret = bslash_completions(string, pos) ok && return ret - # Make sure that only bslash_completions is working on strings - inc_tag === :string && return Completion[], 0:-1, false - if inc_tag === :other - frange, ex, wordrange, method_name_end = identify_possible_method_completion(partial, pos) - if last(frange) != -1 && all(isspace, @view partial[wordrange]) # no last argument to complete - if ex.head === :call - return complete_methods(ex, context_module, shift), first(frange):method_name_end, false - elseif is_broadcasting_expr(ex) - return complete_methods(ex, context_module, shift), first(frange):(method_name_end - 1), false - end + # Don't fall back to symbol completion inside strings or comments. + inside_cmdstr = find_parent(cur, K"cmdstring") !== nothing + (kind(cur) in KSet"String Comment ErrorEofMultiComment" || inside_cmdstr) && + return Completion[], 1:0, false + + n, arg_pos = find_prefix_call(cur_not_ws) + if n !== nothing + func = first(children_nt(n)) + e = Expr(n) + # Remove arguments past the first parse error (allows unclosed parens) + if is_broadcasting_expr(e) + i = findfirst(x -> x isa Expr && x.head == :error, e.args[2].args) + i !== nothing && deleteat!(e.args[2].args, i:lastindex(e.args[2].args)) + else + i = findfirst(x -> x isa Expr && x.head == :error, e.args) + i !== nothing && deleteat!(e.args, i:lastindex(e.args)) end - elseif inc_tag === :comment - return Completion[], 0:-1, false - end - # Check whether we can complete a keyword argument in a function call - kwarg_completion, wordrange = complete_keyword_argument(partial, pos, context_module; shift) - isempty(wordrange) || return kwarg_completion, wordrange, !isempty(kwarg_completion) + # Method completion: + # foo( TAB => list of method signatures for foo + # foo(x, TAB => list of methods signatures for foo with x as first argument + if kind(cur_not_ws) in KSet"( , ;" + # Don't provide method completions unless the cursor is after: '(' ',' ';' + return complete_methods(e, context_module, shift, arg_pos), char_range(func), false + + # Keyword argument completion: + # foo(ar TAB => keyword arguments like `arg1=` + elseif kind(cur) == K"Identifier" + r = char_range(cur) + s = string[intersect(r, 1:pos)] + # Return without adding more suggestions if kwargs only + complete_keyword_argument!(suggestions, e, s, context_module, arg_pos; shift) && + return sort_suggestions(), r, true + end + end - startpos = nextind(string, something(findprev(in(non_identifier_chars), string, pos), 0)) - # strip preceding ! operator - if (m = match(r"\G\!+", partial, startpos)) isa RegexMatch - startpos += length(m.match) + # Symbol completion + # TODO: Should completions replace the identifier at the cursor? + looks_like_ident = Base.isidentifier(@view string[intersect(char_range(cur), 1:pos)]) + if cur.parent !== nothing && kind(cur.parent) == K"var" + # Replace the entire var"foo", but search using only "foo". + r = intersect(char_range(cur.parent), 1:pos) + r2 = char_range(children_nt(cur.parent)[1]) + s = string[intersect(r2, 1:pos)] + elseif kind(cur) == K"MacroName" + # Include the `@` + r = intersect(prevind(string, cur.position):char_last(cur), 1:pos) + s = string[r] + elseif looks_like_ident || kind(cur) in KSet"Bool Identifier @" + r = intersect(char_range(cur), 1:pos) + s = string[r] + else + r = nextind(string, pos):pos + s = "" end - separatorpos = something(findprev(isequal('.'), string, pos), 0) - namepos = max(startpos, separatorpos+1) - name = string[namepos:pos] - import_mode = get_import_mode(string) - if import_mode === :using_module || import_mode === :import_module + complete_modules_only = false + prefix = node_prefix(cur, context_module) + comp_keywords = prefix === nothing && !isempty(s) + + # Complete loadable module names: + # import Mod TAB + # import Mod1, Mod2 TAB + # using Mod TAB + if (n = find_parent(cur, K"importpath")) !== nothing # Given input lines like `using Foo|`, `import Foo, Bar|` and `using Foo.Bar, Baz, |`: # Let's look only for packages and modules we can reach from here + if prefix == nothing + complete_loading_candidates!(suggestions, s) + return sort_suggestions(), r, true + end - # If there's no dot, we're in toplevel, so we should - # also search for packages - s = string[startpos:pos] - if separatorpos <= startpos - for dir in Base.load_path() - if basename(dir) in Base.project_names && isfile(dir) - complete_loading_candidates!(suggestions, s, dir) - end - isdir(dir) || continue - for entry in _readdirx(dir) - pname = entry.name - if pname[1] != '.' && pname != "METADATA" && - pname != "REQUIRE" && startswith(pname, s) - # Valid file paths are - # .jl - # /src/.jl - # .jl/src/.jl - if isfile(entry) - endswith(pname, ".jl") && push!(suggestions, - PackageCompletion(pname[1:prevind(pname, end-2)])) - else - mod_name = if endswith(pname, ".jl") - pname[1:prevind(pname, end-2)] - else - pname - end - if isfile(joinpath(entry, "src", - "$mod_name.jl")) - push!(suggestions, PackageCompletion(mod_name)) - end - end + # Allow completion for `import Mod.name` (where `name` is not a module) + complete_modules_only = prefix == nothing || kind(n.parent) == K"using" + comp_keywords = false + end + + if comp_keywords + complete_keyword!(suggestions, s) + complete_keyval!(suggestions, s) + end + + complete_symbol!(suggestions, prefix, s, context_module; complete_modules_only, shift) + return sort_suggestions(), r, true +end + +function close_path_completion(path) + path = expanduser(path) + path = do_string_unescape(path) + !Base.isaccessibledir(path) +end + +# Lowering can misbehave with nested error expressions. +function expr_has_error(@nospecialize(e)) + e isa Expr || return false + e.head === :error && return true + any(expr_has_error, e.args) +end + +# Is the cursor inside the square brackets of a ref expression? If so, returns: +# - The ref node +# - The range of characters for the brackets +# - A flag indicating if the closing bracket is present +function find_ref_key(cur::CursorNode, pos::Int) + n = find_parent(cur, K"ref") + n !== nothing || return nothing, nothing, nothing + key, closed = find_delim(n, K"[", K"]") + first(key) - 1 <= pos <= last(key) || return nothing, nothing, nothing + n, key, closed +end + +# If the cursor is in a literal string, return the contents and char range +# inside the quotes. Ignores triple strings. +function find_str(cur::CursorNode) + n = find_parent(cur, K"string") + n !== nothing || return nothing, nothing + find_delim(n, K"\"", K"\"") +end + +# Is the cursor directly inside of the arguments of a prefix call (no nested +# expressions)? If so, return: +# - The call node +# - Either :positional or :kwargs, if the cursor is before or after the `;` +function find_prefix_call(cur::CursorNode) + n = cur.parent + n !== nothing || return nothing, nothing + is_call(n) = kind(n) in KSet"call dotcall" && is_prefix_call(n) + if kind(n) == K"parameters" + is_call(n.parent) || return nothing, nothing + n.parent, :kwargs + else + # Check that we are beyond the function name. + is_call(n) && cur.index > children_nt(n)[1].index || return nothing, nothing + n, :positional + end +end + +# If node is the field in a getfield-like expression, return the value +# complete_symbol! should use as the prefix. +function node_prefix(node::CursorNode, context_module::Module) + node.parent !== nothing || return nothing + p = node.parent + # In x.var"y", the parent is the "var" when the cursor is on "y". + kind(p) == K"var" && (p = p.parent) + + # expr.node => expr + if kind(p) == K"." + n = children_nt(p)[1] + # Don't use prefix if we are the value + n !== node || return nothing + return Expr(n) + end + + if kind(p) == K"importpath" + if p.parent !== nothing && kind(p.parent) == K":" && p.index_nt > 1 + # import A.B: C.node + chain = children_nt(children_nt(p.parent)[1]) + append!(chain, children_nt(p)[1:end-1]) + else + # import A.node + # import A.node: ... + chain = children_nt(p)[1:node.index_nt] + # Don't include the node under cursor in prefix unless it is `.` + kind(chain[end]) != K"." && deleteat!(chain, lastindex(chain)) + end + length(chain) > 0 || return nothing + + # (:importpath :x :y :z) => (:. (:. :x :y) :z) + # (:importpath :. :. :z) => (:. (parentmodule context_module) :z) + if (i = findlast(x -> kind(x) == K".", chain)) !== nothing + init = context_module + for j in 2:i + init = parentmodule(init) + end + deleteat!(chain, 1:i) + else + # No leading `.`, init is the first element of the path + init = chain[1].val + deleteat!(chain, 1) + end + + # Convert the "chain" into nested (. a b) expressions. + all(x -> kind(x) == K"Identifier", chain) || return nothing + return foldl((x, y) -> Expr(:., x, Expr(:quote, y.val)), chain; init) + end + + nothing +end + +function dict_eval(@nospecialize(e), context_module::Module=Main) + objt = repl_eval_ex(e.args[1], context_module) + isa(objt, Core.Const) || return nothing + obj = objt.val + isa(obj, AbstractDict) || return nothing + (Base.haslength(obj) && length(obj)::Int < 1_000_000) || return nothing + return obj +end + +function method_search(partial::AbstractString, context_module::Module, shift::Bool) + rexm = match(r"([\w.]+.)?\?\((.*)$", partial) + if rexm !== nothing + # Get the module scope + callee_module = context_module + if !isnothing(rexm.captures[1]) + modnames = map(Symbol, split(something(rexm.captures[1]), '.')) + for m in modnames + if isdefined(callee_module, m) + callee_module = getfield(callee_module, m) + if !isa(callee_module, Module) + callee_module = context_module + break end end end end - comp_keywords = false - complete_modules_only = import_mode === :using_module # allow completion for `import Mod.name` (where `name` is not a module) - elseif import_mode === :using_name || import_mode === :import_name - # `using Foo: |` and `import Foo: bar, baz|` - separatorpos = findprev(isequal(':'), string, pos)::Int - comp_keywords = false - complete_modules_only = false - else - comp_keywords = !isempty(name) && startpos > separatorpos - complete_modules_only = false + moreargs = !endswith(rexm.captures[2], ')') + callstr = "_(" * rexm.captures[2] + if moreargs + callstr *= ')' + end + ex_org = Meta.parse(callstr, raise=false, depwarn=false) + if isa(ex_org, Expr) + pos_q = isnothing(rexm.captures[1]) ? 1 : sizeof(something(rexm.captures[1]))+1 # position after ? + return complete_any_methods(ex_org, callee_module::Module, context_module, moreargs, shift), (0:pos_q) .+ rexm.offset, false + end end - - complete_identifiers!(suggestions, context_module, string, name, - pos, separatorpos, startpos; - comp_keywords, complete_modules_only, shift) - return sort!(unique!(named_completion, suggestions), by=named_completion_completion), namepos:pos, true end -function shell_completions(string, pos, hint::Bool=false) +function shell_completions(str, pos, hint::Bool=false; cmd_escape::Bool=false) # First parse everything up to the current position - scs = string[1:pos] + scs = str[1:pos] args, last_arg_start = try Base.shell_parse(scs, true)::Tuple{Expr,Int} catch ex ex isa ArgumentError || ex isa ErrorException || rethrow() - return Completion[], 0:-1, false + return Completion[], 1:0, false end ex = args.args[end]::Expr # Now look at the last thing we parsed - isempty(ex.args) && return Completion[], 0:-1, false - lastarg = ex.args[end] + isempty(ex.args) && return Completion[], 1:0, false + # Concatenate every string fragment so dir\file completes correctly. + lastarg = all(x -> x isa String, ex.args) ? string(ex.args...) : ex.args[end] + # As Base.shell_parse throws away trailing spaces (unless they are escaped), # we need to special case here. # If the last char was a space, but shell_parse ignored it search on "". if isexpr(lastarg, :incomplete) || isexpr(lastarg, :error) - partial = string[last_arg_start:pos] + partial = str[last_arg_start:pos] ret, range = completions(partial, lastindex(partial), Main, true, hint) range = range .+ (last_arg_start - 1) return ret, range, true elseif endswith(scs, ' ') && !endswith(scs, "\\ ") r = pos+1:pos - paths, dir, success = complete_path("", use_envpath=false, shell_escape=true) + paths, dir, success = complete_path(""; use_envpath=false, shell_escape=!cmd_escape, cmd_escape, dirsep='/') return paths, r, success elseif all(@nospecialize(arg) -> arg isa AbstractString, ex.args) # Join these and treat this as a path @@ -1550,44 +1335,63 @@ function shell_completions(string, pos, hint::Bool=false) # Also try looking into the env path if the user wants to complete the first argument use_envpath = length(args.args) < 2 - expanded = complete_expanduser(path, r) - was_expanded = expanded[3] - if was_expanded - path = (only(expanded[1])::PathCompletion).path - # If tab press, ispath and user expansion available, return it now - # otherwise see if we can complete the path further before returning with expanded ~ - !hint && ispath(path) && return expanded::Completions - end + paths, success = complete_path_string(path, hint; use_envpath, shell_escape=!cmd_escape, cmd_escape, dirsep='/') + return paths, r, success + end + return Completion[], 1:0, false +end - paths, dir, success = complete_path(path, use_envpath=use_envpath, shell_escape=true, contract_user=was_expanded) - - if success && !isempty(dir) - let dir = do_shell_escape(dir) - # if escaping of dir matches scs prefix, remove that from the completions - # otherwise make it the whole completion - partial = string[last_arg_start:pos] - if endswith(dir, "/") && startswith(partial, dir) - r = (last_arg_start + sizeof(dir)):pos - elseif startswith(partial, dir * "/") - r = nextind(string, last_arg_start + sizeof(dir)):pos - else - map!(paths, paths) do c::PathCompletion - return PathCompletion(dir * "/" * c.path) - end - end - end +function complete_path_string(path, hint::Bool=false; + shell_escape::Bool=false, + cmd_escape::Bool=false, + string_escape::Bool=false, + dirsep='/', + kws...) + # Expand "~" and remember if we expanded it. + local expanded + try + let p = expanduser(path) + expanded = path != p + path = p end - # if ~ was expanded earlier and the incomplete string isn't a path - # return the path with contracted user to match what the hint shows. Otherwise expand ~ - # i.e. require two tab presses to expand user - if was_expanded && !ispath(path) - map!(paths, paths) do c::PathCompletion - PathCompletion(contractuser(c.path)) - end + catch e + e isa ArgumentError || rethrow() + expanded = false + end + + function escape(p) + shell_escape && (p = do_shell_escape(p)) + string_escape && (p = do_string_escape(p)) + cmd_escape && (p = do_cmd_escape(p)) + p + end + + paths, dir, success = complete_path(path; dirsep, kws...) + + # Expand '~' if the user hits TAB after exhausting completions (either + # because we have found an existing file, or there is no such file). + full_path = try + ispath(path) || isempty(paths) + catch err + # access(2) errors unhandled by ispath: EACCES, EIO, ELOOP, ENAMETOOLONG + if err isa Base.IOError + false + elseif err isa Base.ArgumentError && occursin("embedded NULs", err.msg) + false + else + rethrow() end - return paths, r, success end - return Completion[], 0:-1, false + expanded && !hint && full_path && return Completion[PathCompletion(escape(path))], true + + # Expand '~' if the user hits TAB on a path ending in '/'. + expanded && (hint || path != dir * "/") && (dir = contractuser(dir)) + + map!(paths) do c::PathCompletion + p = joinpath_withsep(dir, c.path; dirsep) + PathCompletion(escape(p)) + end + return sort!(paths, by=p->p.path), success end function __init__() diff --git a/stdlib/REPL/src/SyntaxUtil.jl b/stdlib/REPL/src/SyntaxUtil.jl new file mode 100644 index 0000000000000..6b455aa16dc9f --- /dev/null +++ b/stdlib/REPL/src/SyntaxUtil.jl @@ -0,0 +1,111 @@ +module SyntaxUtil + +import Base.JuliaSyntax: build_tree +using Base.JuliaSyntax: + AbstractSyntaxData, GreenNode, Kind, ParseStream, SourceFile, SyntaxHead, SyntaxNode, TreeNode, + byte_range, children, first_byte, head, is_leaf, is_trivia, kind, parse_julia_literal, span, + @K_str, _unsafe_wrap_substring + +export CursorNode, char_range, char_last, children_nt, find_delim, seek_pos + +# Like SyntaxNode, but keeps trivia, and tracks each child's index in its parent. +# Extracted from JuliaSyntax/src/syntax_tree.jl +# TODO: don't duplicate so much code? +struct CursorData <: AbstractSyntaxData + source::SourceFile + raw::GreenNode{SyntaxHead} + position::Int + index::Int + index_nt::Int # nth non-trivia in parent + val::Any +end + +const CursorNode = TreeNode{CursorData} + +function CursorNode(source::SourceFile, raw::GreenNode{SyntaxHead}; + position::Integer=1) + GC.@preserve source begin + raw_offset, txtbuf = _unsafe_wrap_substring(source.code) + offset = raw_offset - source.byte_offset + _to_CursorNode(source, txtbuf, offset, raw, convert(Int, position)) + end +end + +function _to_CursorNode(source::SourceFile, txtbuf::Vector{UInt8}, offset::Int, + raw::GreenNode{SyntaxHead}, + position::Int, index::Int=-1, index_nt::Int=-1) + if is_leaf(raw) + valrange = position:position + span(raw) - 1 + val = parse_julia_literal(txtbuf, head(raw), valrange .+ offset) + return CursorNode(nothing, nothing, CursorData(source, raw, position, index, index_nt, val)) + else + cs = CursorNode[] + pos = position + i_nt = 1 + for (i,rawchild) in enumerate(children(raw)) + push!(cs, _to_CursorNode(source, txtbuf, offset, rawchild, pos, i, i_nt)) + pos += Int(rawchild.span) + i_nt += !is_trivia(rawchild) + end + node = CursorNode(nothing, cs, CursorData(source, raw, position, index, index_nt, nothing)) + for c in cs + c.parent = node + end + return node + end +end + +function build_tree(::Type{CursorNode}, stream::ParseStream; + filename=nothing, first_line=1, kws...) + green_tree = build_tree(GreenNode, stream; kws...) + source = SourceFile(stream, filename=filename, first_line=first_line) + CursorNode(source, green_tree, position=first_byte(stream)) +end + +Base.show(io::IO, node::CursorNode) = show(io, MIME("text/plain"), node.raw) +Base.show(io::IO, mime::MIME{Symbol("text/plain")}, node::CursorNode) = show(io, mime, node.raw) + +function Base.Expr(node::CursorNode) + (; filename, first_line) = node.source + src = SourceFile(node.source[byte_range(node)]; filename, first_line) + Expr(SyntaxNode(src, node.raw)) +end + +char_range(node) = node.position:char_last(node) +char_last(node) = thisind(node.source, node.position + span(node) - 1) + +children_nt(node) = [n for n in children(node) if !is_trivia(n)] + +function seek_pos(node, pos) + pos in byte_range(node) || return nothing + (cs = children(node)) === nothing && return node + for n in cs + c = seek_pos(n, pos) + c === nothing || return c + end + node +end + +find_parent(node, k::Kind) = find_parent(node, n -> kind(n) == k) +function find_parent(node, f::Function) + while node !== nothing && !f(node) + node = node.parent + end + node +end + +# Return the character range between left_kind and right_kind in node. The left +# delimiter must be present, while the range will extend to the rest of the node +# if the right delimiter is missing. +function find_delim(node, left_kind, right_kind) + cs = children(node) + left = findfirst(c -> kind(c) == left_kind, cs) + left !== nothing || return nothing, nothing + right = findlast(c -> kind(c) == right_kind, cs) + closed = right !== nothing && right != left + right = closed ? thisind(node.source, cs[right].position - 1) : char_last(node) + left = nextind(node.source, char_last(cs[left])) + return left:right, closed +end + +end diff --git a/stdlib/REPL/src/Terminals.jl b/stdlib/REPL/src/Terminals.jl index aba6bff73a607..14ea6dd3dff77 100644 --- a/stdlib/REPL/src/Terminals.jl +++ b/stdlib/REPL/src/Terminals.jl @@ -146,7 +146,7 @@ end @eval clear_line(t::UnixTerminal) = write(t.out_stream, $"\r$(CSI)0K") beep(t::UnixTerminal) = write(t.err_stream,"\x7") -Base.displaysize(t::UnixTerminal) = displaysize(t.out_stream) +Base.displaysize(t::UnixTerminal) = displaysize(t.out_stream)::Tuple{Int,Int} hascolor(t::TTYTerminal) = get(t.out_stream, :color, false)::Bool diff --git a/stdlib/REPL/src/latex_symbols.jl b/stdlib/REPL/src/latex_symbols.jl index 9f5b7e3e864ed..b739accc72499 100644 --- a/stdlib/REPL/src/latex_symbols.jl +++ b/stdlib/REPL/src/latex_symbols.jl @@ -517,6 +517,7 @@ const latex_symbols = Dict( "\\mapsto" => "↦", "\\hookleftarrow" => "↩", "\\hookrightarrow" => "↪", + "\\hookunderrightarrow" => "🢲", "\\looparrowleft" => "↫", "\\looparrowright" => "↬", "\\leftrightsquigarrow" => "↭", diff --git a/stdlib/REPL/src/precompile.jl b/stdlib/REPL/src/precompile.jl index 4f3b3a6eae083..3b69484ee3996 100644 --- a/stdlib/REPL/src/precompile.jl +++ b/stdlib/REPL/src/precompile.jl @@ -24,7 +24,8 @@ function repl_workload() function check_errors(out) str = String(out) if occursin("ERROR:", str) && !any(occursin(e, str) for e in allowed_errors) - @error "Unexpected error (Review REPL precompilation with debug_output on):\n$str" + @error "Unexpected error (Review REPL precompilation with debug_output on):\n$str" exception=( + Base.PrecompilableError(), Base.backtrace()) exit(1) end end @@ -40,6 +41,7 @@ function repl_workload() # This is notified as soon as the first prompt appears repl_init_event = Base.Event() + repl_init_done_event = Base.Event() atreplinit() do repl # Main is closed so we can't evaluate in it, but atreplinit runs at @@ -48,6 +50,7 @@ function repl_workload() t = @async begin wait(repl_init_event) REPL.activate(REPL.Precompile; interactive_utils=false) + notify(repl_init_done_event) end Base.errormonitor(t) end @@ -58,6 +61,7 @@ function repl_workload() printstyled("a", "b") display([1]) display([1 2; 3 4]) + display("a string") foo(x) = 1 @time @eval foo(1) ; pwd @@ -79,6 +83,9 @@ function repl_workload() """ JULIA_PROMPT = "julia> " + # The help text for `reinterpret` has example `julia>` prompts in it, + # so use the longer prompt to avoid desychronization. + ACTIVATED_JULIA_PROMPT = "(REPL.Precompile) julia> " PKG_PROMPT = "pkg> " SHELL_PROMPT = "shell> " HELP_PROMPT = "help?> " @@ -141,18 +148,24 @@ function repl_workload() end schedule(repltask) # wait for the definitive prompt before start writing to the TTY - check_errors(readuntil(output_copy, JULIA_PROMPT)) + check_errors(readuntil(output_copy, JULIA_PROMPT, keep=true)) + + # Switch to the activated prompt + notify(repl_init_event) + wait(repl_init_done_event) + write(ptm, "\n") + # The prompt prints twice - once for the restatement of the input, once + # to indicate ready for the new prompt. + check_errors(readuntil(output_copy, ACTIVATED_JULIA_PROMPT, keep=true)) + check_errors(readuntil(output_copy, ACTIVATED_JULIA_PROMPT, keep=true)) + write(debug_output, "\n#### REPL STARTED ####\n") - sleep(0.01) - check_errors(readavailable(output_copy)) # Input our script precompile_lines = split(repl_script::String, '\n'; keepempty=false) curr = 0 for l in precompile_lines sleep(0.01) # try to let a bit of output accumulate before reading again curr += 1 - # consume any other output - bytesavailable(output_copy) > 0 && check_errors(readavailable(output_copy)) # push our input write(debug_output, "\n#### inputting statement: ####\n$(repr(l))\n####\n") # If the line ends with a CTRL_C, don't write an extra newline, which would @@ -165,13 +178,12 @@ function repl_workload() strbuf = "" while !eof(output_copy) strbuf *= String(readavailable(output_copy)) - occursin(JULIA_PROMPT, strbuf) && break + occursin(ACTIVATED_JULIA_PROMPT, strbuf) && break occursin(PKG_PROMPT, strbuf) && break occursin(SHELL_PROMPT, strbuf) && break occursin(HELP_PROMPT, strbuf) && break sleep(0.01) # try to let a bit of output accumulate before reading again end - notify(repl_init_event) check_errors(strbuf) end write(debug_output, "\n#### COMPLETED - Closing REPL ####\n") diff --git a/stdlib/REPL/test/precompilation.jl b/stdlib/REPL/test/precompilation.jl index 7efcf0b5e8282..fcd6f9aa1ae2b 100644 --- a/stdlib/REPL/test/precompilation.jl +++ b/stdlib/REPL/test/precompilation.jl @@ -33,7 +33,8 @@ if !Sys.iswindows() # given this test checks that startup is snappy, it's best to add workloads to # contrib/generate_precompile.jl rather than increase this number. But if that's not # possible, it'd be helpful to add a comment with the statement and a reason below - expected_precompiles = 0 + expected_precompiles = 1 + # Jameson told me to bump this n_precompiles = count(r"precompile\(", tracecompile_out) diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index d1233ee00da1b..5fcef91e2e0a8 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -244,8 +244,9 @@ fake_repl(options = REPL.Options(confirm_exit=false,hascolor=true)) do stdin_wri @test occursin("shell> ", s) # check for the echo of the prompt @test occursin("'", s) # check for the echo of the input s = readuntil(stdout_read, "\n\n") - @test(startswith(s, "\e[0mERROR: unterminated single quote\nStacktrace:\n [1] ") || - startswith(s, "\e[0m\e[1m\e[91mERROR: \e[39m\e[22m\e[91munterminated single quote\e[39m\nStacktrace:\n [1] ")) + @test(startswith(s, "\e[0mERROR: unterminated single quote\nStacktrace:\n [1] ") || + startswith(s, "\e[0m\e[1m\e[91mERROR: \e[39m\e[22m\e[91munterminated single quote\e[39m\nStacktrace:\n [1] "), + skip = Sys.iswindows() && Sys.WORD_SIZE == 32) write(stdin_write, "\b") wait(t) end @@ -983,6 +984,13 @@ let ends_with_semicolon = REPL.ends_with_semicolon @test ends_with_semicolon("f()= 1;") # the next result does not matter because this is not legal syntax @test_nowarn ends_with_semicolon("1; #=# 2") + + # #46189 - adjoint operator with comment + @test ends_with_semicolon("W';") == true + @test ends_with_semicolon("W'; # comment") + @test !ends_with_semicolon("W'") + @test !ends_with_semicolon("x'") + @test !ends_with_semicolon("'a'") end # PR #20794, TTYTerminal with other kinds of streams @@ -1550,59 +1558,61 @@ end @testset "Install missing packages via hooks" begin @testset "Parse AST for packages" begin - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Foo")) + test_find_packages(e) = + REPL.modules_to_be_loaded(Meta.lower(@__MODULE__, e)) + test_find_packages(s::String) = + REPL.modules_to_be_loaded(Meta.lower(@__MODULE__, Meta.parse(s))) + + mods = test_find_packages("using Foo") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("import Foo")) + mods = test_find_packages("import Foo") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Foo, Bar")) + mods = test_find_packages("using Foo, Bar") @test mods == [:Foo, :Bar] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("import Foo, Bar")) + mods = test_find_packages("import Foo, Bar") @test mods == [:Foo, :Bar] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Foo.bar, Foo.baz")) + mods = test_find_packages("using Foo.bar, Foo.baz") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("if false using Foo end")) + mods = test_find_packages("if false using Foo end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("if false if false using Foo end end")) + mods = test_find_packages("if false if false using Foo end end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("if false using Foo, Bar end")) + mods = test_find_packages("if false using Foo, Bar end") @test mods == [:Foo, :Bar] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("if false using Foo: bar end")) + mods = test_find_packages("if false using Foo: bar end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("import Foo.bar as baz")) + mods = test_find_packages("import Foo.bar as baz") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using .Foo")) + mods = test_find_packages("using .Foo") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Base")) + mods = test_find_packages("using Base") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Base: nope")) + mods = test_find_packages("using Base: nope") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Main")) + mods = test_find_packages("using Main") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("using Core")) + mods = test_find_packages("using Core") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line(":(using Foo)")) + mods = test_find_packages(":(using Foo)") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("ex = :(using Foo)")) + mods = test_find_packages("ex = :(using Foo)") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("Foo")) + mods = test_find_packages("@eval using Foo") @test isempty(mods) - - mods = REPL.modules_to_be_loaded(Base.parse_input_line("@eval using Foo")) - @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("begin using Foo; @eval using Bar end")) + mods = test_find_packages("begin using Foo; @eval using Bar end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(Base.parse_input_line("Core.eval(Main,\"using Foo\")")) + mods = test_find_packages("Core.eval(Main,\"using Foo\")") @test isempty(mods) - mods = REPL.modules_to_be_loaded(Base.parse_input_line("begin using Foo; Core.eval(Main,\"using Foo\") end")) + mods = test_find_packages("begin using Foo; Core.eval(Main,\"using Foo\") end") @test mods == [:Foo] - mods = REPL.modules_to_be_loaded(:(import .Foo: a)) + mods = test_find_packages(:(import .Foo: a)) @test isempty(mods) - mods = REPL.modules_to_be_loaded(:(using .Foo: a)) + mods = test_find_packages(:(using .Foo: a)) @test isempty(mods) end end @@ -1948,11 +1958,18 @@ end @test output == "…[printing stopped after displaying 0 bytes; $hint]" @test sprint(io -> show(REPL.LimitIO(io, 5), "abc")) == "\"abc\"" @test_throws REPL.LimitIOException(1) sprint(io -> show(REPL.LimitIO(io, 1), "abc")) + + # displaying objects at the REPL sometimes needs access to displaysize, like Dict + @test displaysize(IOContext(REPL.LimitIO(stdout, 100), stdout)) == displaysize(stdout) finally REPL.SHOW_MAXIMUM_BYTES = previous end end +@testset "`displaysize` return type inference" begin + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{REPL.Terminals.UnixTerminal}) +end + @testset "Dummy Pkg prompt" begin # do this in an empty depot to test default for new users withenv("JULIA_DEPOT_PATH" => mktempdir() * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do @@ -1983,3 +2000,32 @@ end write(proj_file, "name = \"Bar\"\n") @test get_prompt("--project=$proj_file") == "(Bar) pkg> " end + +# Issue #58158 add alias for Char display in REPL +@testset "REPL show_repl Char alias" begin + # Test character with a known emoji alias + output = sprint(REPL.show_repl, MIME("text/plain"), '😼'; context=(:color => true)) + # Check for base info and the specific alias + @test occursin("'😼': Unicode U+1F63C (category So: Symbol, other)", output) + @test occursin(", input as ", output) # Check for the prefix text + @test occursin("\\:smirk_cat:", output) # Check for the alias text (may be colored) + + # Test character with a known LaTeX alias + output = sprint(REPL.show_repl, MIME("text/plain"), 'α'; context=(:color => true)) + # Check for base info and the specific alias + @test occursin("'α': Unicode U+03B1 (category Ll: Letter, lowercase)", output) + @test occursin(", input as ", output) # Check for the prefix text + @test occursin("\\alpha", output) # Check for the alias text (may be colored) + + # Test character without an alias + output = sprint(REPL.show_repl, MIME("text/plain"), 'X'; context=(:color => true)) + # Check for base info only + @test occursin("'X': ASCII/Unicode U+0058 (category Lu: Letter, uppercase)", output) + # Ensure alias part is *not* printed + @test !occursin(", input as ", output) + + # Test another character without an alias (symbol) + output = sprint(REPL.show_repl, MIME("text/plain"), '+'; context=(:color => true)) + @test occursin("'+': ASCII/Unicode U+002B (category Sm: Symbol, math)", output) + @test !occursin(", input as ", output) +end diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index d9aa11cf609dd..1b6dad2d17a00 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -17,6 +17,16 @@ let ex = module CompletionFoo using Random import Test + # make everything public, so that nothing gets hidden unintentionally from completions + public Test_y, Text_x, type_test, unicode_αΒγ, CompletionFoo2, bar, + foo, @foobar, @barfoo, @error_expanding, + @error_lowering_conditional, @error_throwing, NonStruct, x, + CustomDict, NoLengthDict, test, test1, test2, test3, test4, test5, + test6, test7, test8, test9, test10, test11, a, test!12, kwtest, + kwtest2, kwtest3, kwtest4, kwtest5, named, fmsoebelkv, array, + varfloat, tuple, test_y_array, test_dict, test_customdict, + @teststr_str, @tϵsτstρ_str, @testcmd_cmd, @tϵsτcmδ_cmd, + var"complicated symbol with spaces", WeirdNames, @ignoremacro mutable struct Test_y yy @@ -129,6 +139,7 @@ let ex = kwtest4(a::SubString; x23, _something) = pass kwtest5(a::Int, b, x...; somekwarg, somekotherkwarg) = pass kwtest5(a::Char, b; xyz) = pass + kwtest6(f::Function, arg1; somekwarg) = pass const named = (; len2=3) const fmsoebelkv = (; len2=3) @@ -158,6 +169,10 @@ let ex = export exported_symbol exported_symbol(::WeirdNames) = nothing + macro ignoremacro(e...) + nothing + end + end # module CompletionFoo test_repl_comp_dict = CompletionFoo.test_dict test_repl_comp_customdict = CompletionFoo.test_customdict @@ -180,9 +195,13 @@ end test_complete(s) = map_completion_text(@inferred(completions(s, lastindex(s)))) test_scomplete(s) = map_completion_text(@inferred(shell_completions(s, lastindex(s)))) +# | is reserved in test_complete_pos +test_complete_pos(s) = map_completion_text(@inferred(completions(replace(s, '|' => ""), findfirst('|', s)-1))) test_complete_context(s, m=@__MODULE__; shift::Bool=true) = map_completion_text(@inferred(completions(s,lastindex(s), m, shift))) -test_complete_foo(s) = test_complete_context(s, Main.CompletionFoo) +test_complete_context_pos(s, m=@__MODULE__; shift::Bool=true) = + map_completion_text(@inferred(completions(replace(s, '|' => ""), findfirst('|', s)-1, m, shift))) +test_complete_foo(s; shift::Bool=true) = test_complete_context(s, Main.CompletionFoo; shift) test_complete_noshift(s) = map_completion_text(@inferred(completions(s, lastindex(s), Main, false))) test_bslashcomplete(s) = map_named_completion(@inferred(bslash_completions(s, lastindex(s)))[2]) @@ -290,10 +309,11 @@ let s = "Main.CompletionFoo.bar.no_val_available" @test length(c)==0 end -#cannot do dot completion on infix operator -let s = "+." - c, r = test_complete(s) - @test length(c)==0 +#cannot do dot completion on infix operator (get default completions) +let s1 = "", s2 = "+." + c1, r1 = test_complete(s1) + c2, r2 = test_complete(s2) + @test length(c1)==length(c2) end # To complete on a variable of a type, the type T of the variable @@ -458,13 +478,13 @@ let c, r, res = test_complete(s) @test !res @test all(m -> string(m) in c, methods(isnothing)) - @test s[r] == s[1:end-1] + @test s[r] == s[2:end-1] s = "!!isnothing(" c, r, res = test_complete(s) @test !res @test all(m -> string(m) in c, methods(isnothing)) - @test s[r] == s[1:end-1] + @test s[r] == s[3:end-1] end # Test completion of methods with input concrete args and args where typeinference determine their type @@ -1052,10 +1072,43 @@ end let c, r, res c, r, res = test_scomplete("\$a") @test c == String[] - @test r === 0:-1 + @test r === 1:0 @test res === false end +# A pair of utility function for the REPL completions test to test PATH_cache +# dependent completions, which ordinarily happen asynchronously. +# Only to be used from the test suite +function test_only_arm_cache_refresh() + @lock REPL.REPLCompletions.PATH_cache_lock begin + @assert REPL.REPLCompletions.PATH_cache_condition === nothing + + # Arm a condition we can wait on + REPL.REPLCompletions.PATH_cache_condition = Threads.Condition(REPL.REPLCompletions.PATH_cache_lock) + + # Check if the previous update is still running - if so, wait for it to finish + while REPL.REPLCompletions.PATH_cache_task !== nothing + @assert !istaskdone(REPL.REPLCompletions.PATH_cache_task) + wait(REPL.REPLCompletions.PATH_cache_condition) + end + + # force the next cache update to happen immediately + REPL.REPLCompletions.next_cache_update = 0 + end + return nothing +end + +function test_only_wait_cache_path_done() + @lock REPL.REPLCompletions.PATH_cache_lock begin + @assert REPL.REPLCompletions.PATH_cache_condition !== nothing + + while REPL.REPLCompletions.next_cache_update == 0. + wait(REPL.REPLCompletions.PATH_cache_condition) + end + REPL.REPLCompletions.PATH_cache_condition = nothing + end +end + if Sys.isunix() let s, c, r #Assume that we can rely on the existence and accessibility of /tmp @@ -1064,38 +1117,38 @@ let s, c, r # Issue #8047 s = "@show \"/dev/nul" c,r = test_complete(s) - @test "null\"" in c - @test r == 13:15 - @test s[r] == "nul" + @test "/dev/null\"" in c + @test r == 8:15 + @test s[r] == "/dev/nul" # Tests path in Julia code and not closing " if it's a directory # Issue #8047 s = "@show \"/tm" c,r = test_complete(s) - @test "tmp/" in c - @test r == 9:10 - @test s[r] == "tm" + @test "/tmp/" in c + @test r == 8:10 + @test s[r] == "/tm" # Tests path in Julia code and not double-closing " # Issue #8047 s = "@show \"/dev/nul\"" c,r = completions(s, 15) c = map(named_completion, c) - @test "null\"" in [_c.completion for _c in c] - @test r == 13:15 - @test s[r] == "nul" + @test "/dev/null" in [_c.completion for _c in c] + @test r == 8:15 + @test s[r] == "/dev/nul" s = "/t" c,r = test_scomplete(s) - @test "tmp/" in c - @test r == 2:2 - @test s[r] == "t" + @test "/tmp/" in c + @test r == 1:2 + @test s[r] == "/t" s = "/tmp" c,r = test_scomplete(s) - @test "tmp/" in c - @test r == 2:4 - @test s[r] == "tmp" + @test "/tmp/" in c + @test r == 1:4 + @test s[r] == "/tmp" # This should match things that are inside the tmp directory s = tempdir() @@ -1106,7 +1159,7 @@ let s, c, r c,r = test_scomplete(s) @test !("tmp/" in c) @test !("$s/tmp/" in c) - @test r === (sizeof(s) + 1):sizeof(s) + @test r === 1:sizeof(s) end s = "cd \$(Iter" @@ -1131,8 +1184,8 @@ let s, c, r touch(file) s = string(tempdir(), "/repl\\ ") c,r = test_scomplete(s) - @test ["'repl completions'"] == c - @test s[r] == "repl\\ " + @test [Base.shell_escape_posixly(joinpath(tempdir(), "repl completions"))] == c + @test s[r] == string(tempdir(), "/repl\\ ") rm(file) end @@ -1144,12 +1197,19 @@ let s, c, r mkdir(dir) s = "\"" * path * "/tmpfoob" c,r = test_complete(s) - @test "tmpfoobar/" in c - l = 3 + length(path) - @test r == l:l+6 - @test s[r] == "tmpfoob" + @test string(dir, "/") in c + @test r == 2:sizeof(s) + @test s[r] == joinpath(path, "tmpfoob") + + # Homedir expansion inside Cmd string (#57624) + s = "`ls " * path * "/tmpfoob" + c,r = test_complete(s) + @test string(dir, "/") in c + @test r == 5:sizeof(s) + @test s[r] == joinpath(path, "tmpfoob") + s = "\"~" - @test "tmpfoobar/" in c + @test joinpath(path, "tmpfoobar/") in c c,r = test_complete(s) s = "\"~user" c, r = test_complete(s) @@ -1180,12 +1240,9 @@ let s, c, r # Files reachable by PATH are cached async when PATH is seen to have been changed by `complete_path` # so changes are unlikely to appear in the first complete. For testing purposes we can wait for # caching to finish - @lock REPL.REPLCompletions.PATH_cache_lock begin - # force the next cache update to happen immediately - REPL.REPLCompletions.next_cache_update = 0 - end + test_only_arm_cache_refresh() c,r = test_scomplete(s) - timedwait(()->REPL.REPLCompletions.next_cache_update != 0, 5) # wait for caching to complete + test_only_wait_cache_path_done() c,r = test_scomplete(s) @test "tmp-executable" in c @test r == 1:9 @@ -1214,12 +1271,9 @@ let s, c, r withenv("PATH" => string(tempdir(), ":", dir)) do s = string("repl-completio") - @lock REPL.REPLCompletions.PATH_cache_lock begin - # force the next cache update to happen immediately - REPL.REPLCompletions.next_cache_update = 0 - end + test_only_arm_cache_refresh() c,r = test_scomplete(s) - timedwait(()->REPL.REPLCompletions.next_cache_update != 0, 5) # wait for caching to complete + test_only_wait_cache_path_done() c,r = test_scomplete(s) @test ["repl-completion"] == c @test s[r] == "repl-completio" @@ -1248,7 +1302,7 @@ let current_dir, forbidden e isa Base.IOError && occursin("ELOOP", e.msg) end c, r = test_complete("\"$(escape_string(path))/selfsym") - @test c == ["selfsymlink\""] + @test c == [escape_string(joinpath(path, "selfsymlink")) * "\""] end end @@ -1285,47 +1339,47 @@ mktempdir() do path s = Sys.iswindows() ? "cd $dir_space\\\\space" : "cd $dir_space/space" c, r = test_scomplete(s) @test s[r] == (Sys.iswindows() ? "$dir_space\\\\space" : "$dir_space/space") - @test "'$space_folder'/'space .file'" in c + @test "'$space_folder/space .file'" in c # Also use shell escape rules within cmd backticks s = "`$s" c, r = test_scomplete(s) @test s[r] == (Sys.iswindows() ? "$dir_space\\\\space" : "$dir_space/space") - @test "'$space_folder'/'space .file'" in c + @test "'$space_folder/space .file'" in c # escape string according to Julia escaping rules julia_esc(str) = REPL.REPLCompletions.do_string_escape(str) # For normal strings the string should be properly escaped according to # the usual rules for Julia strings. - s = "cd(\"" * julia_esc(joinpath(path, space_folder) * "/space") + s = "cd(\"" * julia_esc(joinpath(path, space_folder, "space")) c, r = test_complete(s) - @test s[r] == "space" - @test "space .file\"" in c + @test s[r] == julia_esc(joinpath(path, space_folder, "space")) + @test julia_esc(joinpath(path, space_folder, "space .file")) * "\"" in c # '$' is the only character which can appear in a windows filename and # which needs to be escaped in Julia strings (on unix we could do this # test with all sorts of special chars) touch(joinpath(space_folder, "needs_escape\$.file")) - escpath = julia_esc(joinpath(path, space_folder) * "/needs_escape\$") + escpath = julia_esc(joinpath(path, space_folder, "needs_escape\$")) s = "cd(\"$escpath" c, r = test_complete(s) - @test s[r] == "needs_escape\\\$" - @test "needs_escape\\\$.file\"" in c + @test s[r] == julia_esc(joinpath(path, space_folder, "needs_escape\$")) + @test julia_esc(joinpath(path, space_folder, "needs_escape\$.file")) * "\"" in c if !Sys.iswindows() touch(joinpath(space_folder, "needs_escape2\n\".file")) escpath = julia_esc(joinpath(path, space_folder, "needs_escape2\n\"")) s = "cd(\"$escpath" c, r = test_complete(s) - @test s[r] == "needs_escape2\\n\\\"" - @test "needs_escape2\\n\\\".file\"" in c + @test s[r] == joinpath(path, space_folder, "needs_escape2\\n\\\"") + @test joinpath(path, space_folder, "needs_escape2\\n\\\".file\"") in c touch(joinpath(space_folder, "needs_escape3\\.file")) escpath = julia_esc(joinpath(path, space_folder, "needs_escape3\\")) s = "cd(\"$escpath" c, r = test_complete(s) - @test s[r] == "needs_escape3\\\\" - @test "needs_escape3\\\\.file\"" in c + @test s[r] == joinpath(path, space_folder, "needs_escape3\\\\") + @test joinpath(path, space_folder, "needs_escape3\\\\.file\"") in c end # Test for issue #10324 @@ -1339,16 +1393,17 @@ mktempdir() do path test_dir = "test$(c)test" mkdir(joinpath(path, test_dir)) try - if !(c in ['\'','$']) # As these characters hold special meaning + # TODO: test on Windows when backslash-paths fixed + if !Sys.iswindows() && !(c in ['\'','$']) # As these characters hold special meaning # in shell commands the shell path completion cannot complete # paths with these characters c, r, res = test_scomplete(test_dir) - @test c[1] == "'$test_dir/'" + @test c[1] == "'$(joinpath(test_dir, ""))'" @test res end escdir = julia_esc(test_dir) c, r, res = test_complete("\""*escdir) - @test c[1] == escdir * "/" + @test c[1] == julia_esc(joinpath(test_dir, "")) @test res finally rm(joinpath(path, test_dir), recursive=true) @@ -1361,7 +1416,7 @@ end # Test tilde path completion let (c, r, res) = test_complete("\"~/ka8w5rsz") if !Sys.iswindows() - @test res && c == String[homedir() * "/ka8w5rsz"] + @test res && c == String[homedir() * "/ka8w5rsz\""] else @test !res end @@ -1376,7 +1431,7 @@ if !Sys.iswindows() try let (c, r, res) = test_complete("\"~/Zx6Wa0GkC") @test res - @test c == String["Zx6Wa0GkC0/"] + @test c == String["~/Zx6Wa0GkC0/"] end let (c, r, res) = test_complete("\"~/Zx6Wa0GkC0") @test res @@ -1384,11 +1439,11 @@ if !Sys.iswindows() end let (c, r, res) = test_complete("\"~/Zx6Wa0GkC0/my_") @test res - @test c == String["my_file\""] + @test c == String["~/Zx6Wa0GkC0/my_file\""] end let (c, r, res) = test_complete("\"~/Zx6Wa0GkC0/my_file") @test res - @test c == String[homedir() * "/Zx6Wa0GkC0/my_file"] + @test c == String[homedir() * "/Zx6Wa0GkC0/my_file\""] end finally rm(path, recursive=true) @@ -1414,8 +1469,8 @@ if Sys.iswindows() s = "cd ../" c,r = test_scomplete(s) - @test r == lastindex(s)+1:lastindex(s) - @test "$temp_name/" in c + @test r == 4:6 + @test "../$temp_name/" in c s = "ls $(file[1:2])" c,r = test_scomplete(s) @@ -1425,12 +1480,12 @@ if Sys.iswindows() s = "cd(\"..\\\\" c,r = test_complete(s) @test r == lastindex(s)-3:lastindex(s) - @test "../$temp_name/" in c + @test "..\\\\$temp_name\\\\" in c s = "cd(\"../" c,r = test_complete(s) - @test r == lastindex(s)+1:lastindex(s) - @test "$temp_name/" in c + @test r == 5:7 + @test "..\\\\$temp_name\\\\" in c s = "cd(\"$(file[1:2])" c,r = test_complete(s) @@ -1485,10 +1540,10 @@ function test_dict_completion(dict_name) s = "$dict_name[ \"abcd" # leading whitespace c, r = test_complete(s) @test c == Any["\"abcd\"]"] - s = "$dict_name[\"abcd]" # trailing close bracket + s = "$dict_name[Bas]" # trailing close bracket c, r = completions(s, lastindex(s) - 1) c = map(x -> named_completion(x).completion, c) - @test c == Any["\"abcd\""] + @test c == Any["Base"] s = "$dict_name[:b" c, r = test_complete(s) @test c == Any[":bar", ":bar2"] @@ -1542,9 +1597,13 @@ test_dict_completion("test_repl_comp_customdict") @testset "dict_identifier_key" begin # Issue #23004: this should not throw: - @test REPLCompletions.dict_identifier_key("test_dict_ℂ[\\", :other) isa Tuple + let s = "test_dict_ℂ[\\" + @test REPLCompletions.completions(s, sizeof(s), Main.CompletionFoo) isa Tuple + end # Issue #55931: neither should this: - @test REPLCompletions.dict_identifier_key("test_dict_no_length[", :other) isa NTuple{3,Nothing} + let s = "test_dict_no_length[" + @test REPLCompletions.completions(s, sizeof(s), Main.CompletionFoo) isa Tuple + end end @testset "completion of string/cmd macros (#22577)" begin @@ -2485,8 +2544,211 @@ let (c, r, res) = test_complete_context("global xxx::Number = Base.", Main) @test "pi" ∈ c end +# #55842 +let (c, r) = test_complete_pos("@tim| using Date") + @test "@time" in c + @test r == 1:4 +end + +# #56389 +let s = "begin\n using Ran" + c, r = test_complete(s) + @test "Random" in c + @test r == 15:17 + @test s[r] == "Ran" +end +let s = "using .CompletionFoo: bar, type_" + c, r = test_complete(s) + @test "type_test" in c + @test r == 28:32 + @test s[r] == "type_" +end + +# #55518 +let s = "CompletionFoo.@barfoo nothi" + c, r = test_complete(s) + @test "nothing" in c + @test r == 23:27 +end +let s = "CompletionFoo.@barfoo kwtest" + c, r = test_complete(s) + @test isempty(c) +end +let s = "CompletionFoo.kwtest(x=type" + c, r = test_complete(s) + @test "typeof" in c + @test !("type_test" in c) + @test r == 24:27 +end +let s = "CompletionFoo.bar; nothi" + c, r = test_complete(s) + @test "nothing" in c + @test r == 20:24 +end +let s = "CompletionFoo.bar; @ti" + c, r = test_complete(s) + @test "@time" in c + @test r == 20:22 +end +let s = "x = sin.([1]); y = ex" + c, r = test_complete(s) + @test "exp" in c + @test r == 20:21 +end + +# #57611 +let s = "x = Base.BinaryPlatforms.ar" + c, r = test_complete(s) + @test "arch" in c + @test r == 26:27 +end + +# #55520 +let s = "@ignoremacro A .= A setup=(A=ident" + c, r = test_complete(s) + @test "identity"in c + @test r == 30:34 +end + +# #57307 +let s = "unicode_αβγ.yy = named.len" + c, r = test_complete_foo(s) + @test "len2" in c + @test r == 27:29 +end + +# #55429 +let s = "@time @eval CompletionFoo.Compl" + c, r = test_complete(s) + @test "CompletionFoo2" in c + @test r == 27:31 +end + +# #55420 +let s = "CompletionFoo.test(iden" + c, r = test_complete(s) + @test "identity" in c + @test r == 20:23 +end + +# #57772 +let s = "sum(!ismis" + c, r = test_complete(s) + @test "ismissing" in c + @test r == 6:10 +end +let s = "sum(!!ismis" + c, r = test_complete(s) + @test "ismissing" in c + @test r == 7:11 +end + +# Don't trigger complete_methods! when the cursor is on the function name. +let s = "prin|(\"hello\")" + c, r = test_complete_pos(s) + @test "print" in c + @test r == 1:4 +end + +# Don't crash when tab-completing paths that cause ispath() to throw +let s = "include(\"" * repeat("a", 5000) # ENAMETOOLONG + c, r = test_complete(s) + @test isempty(c) +end + # JuliaLang/julia#57780 const issue57780 = ["a", "b", "c"] const issue57780_orig = copy(issue57780) test_complete_context("empty!(issue57780).", Main) @test issue57780 == issue57780_orig + +function g54131 end +for i in 1:498 + @eval g54131(::Val{$i}) = i +end +g54131(::Val{499}; kwarg=true) = 499*kwarg +struct F54131; end +Base.getproperty(::F54131, ::Symbol) = Any[cos, sin, g54131][rand(1:3)] +f54131 = F54131() +@testset "performance of kwarg completion with large method tables" begin + # The goal here is to simply ensure we aren't hitting catestrophically bad + # behaviors when shift isn't pressed. The difference between good and bad + # is on the order of tens of milliseconds vs tens of seconds; using 1 sec as + # a very rough canary that is hopefully robust even in the noisy CI coalmines + s = "g54131(kwa" + a, b, c = completions(s, lastindex(s), @__MODULE__, #= shift =# false) + @test REPLCompletions.KeywordArgumentCompletion("kwarg") in a + @test (@elapsed completions(s, lastindex(s), @__MODULE__, false)) < 1 + + s = "f54131.x(" + a, b, c = completions(s, lastindex(s), @__MODULE__, false) + @test only(a) isa REPLCompletions.TextCompletion + @test (@elapsed completions(s, lastindex(s), @__MODULE__, false)) < 1 + + s = "f54131.x(kwa" + a, b, c = completions(s, lastindex(s), @__MODULE__, false) + @test_broken REPLCompletions.KeywordArgumentCompletion("kwarg") in a + @test (@elapsed completions(s, lastindex(s), @__MODULE__, false)) < 1 +end + +# Completion inside string interpolation +let s = "\"example: \$varflo" + c, r = test_complete_foo(s) + @test "varfloat" in c + @test r == 12:17 +end + +let s = "\"example: \$(3 + findfir" + c, r = test_complete(s) + @test "findfirst" in c + @test r == 17:23 +end + +let s = "\"example: \$(named.len" + c, r = test_complete_foo(s) + @test "len2" in c + @test r == 19:21 +end + +# #58296 - complete positional arguments before semicolon +let s = "string(findfi|; base=16)" + c, r = test_complete_pos(s) + @test "findfirst" in c + @test r == 8:13 +end + +# Unknown functions should not cause completions to fail +let s = "foo58296(findfi" + c, r = test_complete(s) + @test "findfirst" in c + @test r == 10:15 +end + +# #58931 - only show local names when completing the empty string +let s = "" + c, r = test_complete_foo(s) + @test "test" in c + @test !("rand" in c) +end + +# #58309, #58832 - don't show every name when completing after a full keyword +let s = "true" # bool is a little different (Base.isidentifier special case) + c, r = test_complete(s) + @test "trues" in c + @test "true" in c + @test !("rand" in c) +end + +let s = "for" + c, r = test_complete(s) + @test "for" in c + @test "foreach" in c + @test !("rand" in c) +end + +# #58833 - Autocompletion of keyword arguments with do-blocks is broken +let s = "kwtest6(123; som|) do x; x + 3 end" + c, r = test_complete_context_pos(s, Main.CompletionFoo) + @test "somekwarg=" in c + @test r == 14:16 +end diff --git a/stdlib/Random/src/DSFMT.jl b/stdlib/Random/src/DSFMT.jl index 25155b4e8575d..9d4b2575a9556 100644 --- a/stdlib/Random/src/DSFMT.jl +++ b/stdlib/Random/src/DSFMT.jl @@ -126,15 +126,17 @@ function mulxmod!(f::GF2X, m::GF2X, deg=degree(m))::GF2X end # cache for X^(2i) mod m -const _squares = Dict{GF2X, Vector{GF2X}}() +const _squares = Base.Lockable(Dict{GF2X, Vector{GF2X}}()) # compute f^2 mod m function sqrmod!(f::GF2X, m::GF2X)::GF2X d = degree(m)-1 0 <= degree(f) <= d || throw(DomainError("f must satisfy 0 <= degree(f) <= degree(m)-1")) - sqrs = get!(_squares, m) do + sqrs = @lock _squares get(_squares[], m, nothing) + if sqrs === nothing x2i = GF2X(1) - GF2X[copy(mulxmod!(mulxmod!(x2i, m, d+1), m, d+1)) for i=1:d] + sqrs = GF2X[copy(mulxmod!(mulxmod!(x2i, m, d+1), m, d+1)) for i=1:d] + @lock _squares get!(_squares[], m, sqrs) end foldl(filter(i->coeff(f, i), 0:degree(f)); init=GF2X(0)) do g, i i <= d÷2 ? # optimization for "simple" squares @@ -154,16 +156,10 @@ function powxmod(e::BigInt, m::GF2X)::GF2X end "Cached jump polynomials for `MersenneTwister`." -const JumpPolys = Dict{BigInt,GF2X}() +const JumpPolys = Base.Lockable(Dict{BigInt,GF2X}()) -const CharPoly_ref = Ref{GF2X}() -# Ref because it can not be initialized at load time -function CharPoly() - if !isassigned(CharPoly_ref) - CharPoly_ref[] = GF2X(Poly19937) - end - return CharPoly_ref[] -end +# OncePerProcess because it can not be initialized at load time +const CharPoly = OncePerProcess{GF2X}(() -> GF2X(Poly19937)) """ calc_jump(steps::Integer) @@ -175,12 +171,17 @@ less than the period (e.g. ``steps ≪ 2^19937-1``). function calc_jump(steps::Integer, charpoly::GF2X=CharPoly())::GF2X steps < 0 && throw(DomainError("jump steps must be >= 0 (got $steps)")) - if isempty(JumpPolys) - JumpPolys[big(10)^20] = GF2X(JPOLY1e20) + poly = @lock JumpPolys begin + if isempty(JumpPolys[]) + JumpPolys[][big(10)^20] = GF2X(JPOLY1e20) + end + get(JumpPolys[], steps, nothing) end - get!(JumpPolys, steps) do - powxmod(big(steps), charpoly) + if poly === nothing + poly = powxmod(big(steps), charpoly) + @lock JumpPolys get!(JumpPolys[], steps, poly) end + poly end diff --git a/stdlib/Random/src/MersenneTwister.jl b/stdlib/Random/src/MersenneTwister.jl new file mode 100644 index 0000000000000..7caa75ddcd0a7 --- /dev/null +++ b/stdlib/Random/src/MersenneTwister.jl @@ -0,0 +1,667 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## MersenneTwister + +const MT_CACHE_F = 501 << 1 # number of Float64 in the cache +const MT_CACHE_I = 501 << 4 # number of bytes in the UInt128 cache + +@assert dsfmt_get_min_array_size() <= MT_CACHE_F + +mutable struct MersenneTwister <: AbstractRNG + seed::Any + state::DSFMT_state + vals::Vector{Float64} + ints::Vector{UInt128} + idxF::Int + idxI::Int + + # counters for show + adv::Int64 # state of advance at the DSFMT_state level + adv_jump::BigInt # number of skipped Float64 values via randjump + adv_vals::Int64 # state of advance when vals is filled-up + adv_ints::Int64 # state of advance when ints is filled-up + + function MersenneTwister(seed, state, vals, ints, idxF, idxI, + adv, adv_jump, adv_vals, adv_ints) + length(vals) == MT_CACHE_F && 0 <= idxF <= MT_CACHE_F || + throw(DomainError((length(vals), idxF), + "`length(vals)` and `idxF` must be consistent with $MT_CACHE_F")) + length(ints) == MT_CACHE_I >> 4 && 0 <= idxI <= MT_CACHE_I || + throw(DomainError((length(ints), idxI), + "`length(ints)` and `idxI` must be consistent with $MT_CACHE_I")) + new(seed, state, vals, ints, idxF, idxI, + adv, adv_jump, adv_vals, adv_ints) + end +end + +MersenneTwister(seed, state::DSFMT_state) = + MersenneTwister(seed, state, + Vector{Float64}(undef, MT_CACHE_F), + Vector{UInt128}(undef, MT_CACHE_I >> 4), + MT_CACHE_F, 0, 0, 0, -1, -1) + +""" + MersenneTwister(seed) + MersenneTwister() + +Create a `MersenneTwister` RNG object. Different RNG objects can have +their own seeds, which may be useful for generating different streams +of random numbers. +The `seed` may be an integer, a string, or a vector of `UInt32` integers. +If no seed is provided, a randomly generated one is created (using entropy from the system). +See the [`seed!`](@ref) function for reseeding an already existing `MersenneTwister` object. + +!!! compat "Julia 1.11" + Passing a negative integer seed requires at least Julia 1.11. + +# Examples +```jldoctest +julia> rng = MersenneTwister(123); + +julia> x1 = rand(rng, 2) +2-element Vector{Float64}: + 0.37453777969575874 + 0.8735343642013971 + +julia> x2 = rand(MersenneTwister(123), 2) +2-element Vector{Float64}: + 0.37453777969575874 + 0.8735343642013971 + +julia> x1 == x2 +true +``` +""" +MersenneTwister(seed=nothing) = + seed!(MersenneTwister(Vector{UInt32}(), DSFMT_state()), seed) + + +function copy!(dst::MersenneTwister, src::MersenneTwister) + dst.seed = src.seed + copy!(dst.state, src.state) + copyto!(dst.vals, src.vals) + copyto!(dst.ints, src.ints) + dst.idxF = src.idxF + dst.idxI = src.idxI + dst.adv = src.adv + dst.adv_jump = src.adv_jump + dst.adv_vals = src.adv_vals + dst.adv_ints = src.adv_ints + dst +end + +copy(src::MersenneTwister) = + MersenneTwister(src.seed, copy(src.state), copy(src.vals), copy(src.ints), + src.idxF, src.idxI, src.adv, src.adv_jump, src.adv_vals, src.adv_ints) + + +==(r1::MersenneTwister, r2::MersenneTwister) = + r1.seed == r2.seed && r1.state == r2.state && + isequal(r1.vals, r2.vals) && + isequal(r1.ints, r2.ints) && + r1.idxF == r2.idxF && r1.idxI == r2.idxI + +hash(r::MersenneTwister, h::UInt) = + foldr(hash, (r.seed, r.state, r.vals, r.ints, r.idxF, r.idxI); init=h) + +function show(io::IO, rng::MersenneTwister) + # seed + if rng.adv_jump == 0 && rng.adv == 0 + return print(io, MersenneTwister, "(", repr(rng.seed), ")") + end + print(io, MersenneTwister, "(", repr(rng.seed), ", (") + # state + sep = ", " + show(io, rng.adv_jump) + print(io, sep) + show(io, rng.adv) + if rng.adv_vals != -1 || rng.adv_ints != -1 + # "(0, 0)" is nicer on the eyes than (-1, 1002) + s = rng.adv_vals != -1 + print(io, sep) + show(io, s ? rng.adv_vals : zero(rng.adv_vals)) + print(io, sep) + show(io, s ? rng.idxF : zero(rng.idxF)) + end + if rng.adv_ints != -1 + idxI = (length(rng.ints)*16 - rng.idxI) / 8 # 8 represents one Int64 + idxI = Int(idxI) # idxI should always be an integer when using public APIs + print(io, sep) + show(io, rng.adv_ints) + print(io, sep) + show(io, idxI) + end + print(io, "))") +end + +### low level API + +function reset_caches!(r::MersenneTwister) + # zeroing the caches makes comparing two MersenneTwister RNGs easier + fill!(r.vals, 0.0) + fill!(r.ints, zero(UInt128)) + mt_setempty!(r) + mt_setempty!(r, UInt128) + r.adv_vals = -1 + r.adv_ints = -1 + r +end + +#### floats + +mt_avail(r::MersenneTwister) = MT_CACHE_F - r.idxF +mt_empty(r::MersenneTwister) = r.idxF == MT_CACHE_F +mt_setfull!(r::MersenneTwister) = r.idxF = 0 +mt_setempty!(r::MersenneTwister) = r.idxF = MT_CACHE_F +mt_pop!(r::MersenneTwister) = @inbounds return r.vals[r.idxF+=1] + +@noinline function gen_rand(r::MersenneTwister) + r.adv_vals = r.adv + GC.@preserve r fill_array!(r, pointer(r.vals), length(r.vals), CloseOpen12()) + mt_setfull!(r) +end + +reserve_1(r::MersenneTwister) = (mt_empty(r) && gen_rand(r); nothing) +# `reserve` allows one to call `rand_inbounds` n times +# precondition: n <= MT_CACHE_F +reserve(r::MersenneTwister, n::Int) = (mt_avail(r) < n && gen_rand(r); nothing) + +#### ints + +logsizeof(::Type{<:Union{Bool,Int8,UInt8}}) = 0 +logsizeof(::Type{<:Union{Int16,UInt16}}) = 1 +logsizeof(::Type{<:Union{Int32,UInt32}}) = 2 +logsizeof(::Type{<:Union{Int64,UInt64}}) = 3 +logsizeof(::Type{<:Union{Int128,UInt128}}) = 4 + +idxmask(::Type{<:Union{Bool,Int8,UInt8}}) = 15 +idxmask(::Type{<:Union{Int16,UInt16}}) = 7 +idxmask(::Type{<:Union{Int32,UInt32}}) = 3 +idxmask(::Type{<:Union{Int64,UInt64}}) = 1 +idxmask(::Type{<:Union{Int128,UInt128}}) = 0 + + +mt_avail(r::MersenneTwister, ::Type{T}) where {T<:BitInteger} = + r.idxI >> logsizeof(T) + +function mt_setfull!(r::MersenneTwister, ::Type{<:BitInteger}) + r.adv_ints = r.adv + ints = r.ints + + @assert length(ints) == 501 + # dSFMT natively randomizes 52 out of 64 bits of each UInt64 words, + # i.e. 12 bits are missing; + # by generating 5 words == 5*52 == 260 bits, we can fully + # randomize 4 UInt64 = 256 bits; IOW, at the array level, we must + # randomize ceil(501*1.25) = 627 UInt128 words (with 2*52 bits each), + # which we then condense into fully randomized 501 UInt128 words + + len = 501 + 126 # 126 == ceil(501 / 4) + resize!(ints, len) + p = pointer(ints) # must be *after* resize! + GC.@preserve r fill_array!(r, Ptr{Float64}(p), len*2, CloseOpen12_64()) + + k = 501 + n = 0 + @inbounds while n != 500 + u = ints[k+=1] + ints[n+=1] ⊻= u << 48 + ints[n+=1] ⊻= u << 36 + ints[n+=1] ⊻= u << 24 + ints[n+=1] ⊻= u << 12 + end + @assert k == len - 1 + @inbounds ints[501] ⊻= ints[len] << 48 + resize!(ints, 501) + r.idxI = MT_CACHE_I +end + +mt_setempty!(r::MersenneTwister, ::Type{<:BitInteger}) = r.idxI = 0 + +function reserve1(r::MersenneTwister, ::Type{T}) where T<:BitInteger + r.idxI < sizeof(T) && mt_setfull!(r, T) + nothing +end + +function mt_pop!(r::MersenneTwister, ::Type{T}) where T<:BitInteger + reserve1(r, T) + r.idxI -= sizeof(T) + i = r.idxI + @inbounds x128 = r.ints[1 + i >> 4] + i128 = (i >> logsizeof(T)) & idxmask(T) # 0-based "indice" in x128 + (x128 >> (i128 * (sizeof(T) << 3))) % T +end + +function mt_pop!(r::MersenneTwister, ::Type{T}) where {T<:Union{Int128,UInt128}} + reserve1(r, T) + idx = r.idxI >> 4 + r.idxI = idx << 4 - 16 + @inbounds r.ints[idx] % T +end + + +#### seed!() + +function initstate!(r::MersenneTwister, data::StridedVector, seed) + # we deepcopy `seed` because the caller might mutate it, and it's useful + # to keep it constant inside `MersenneTwister`; but multiple instances + # can share the same seed without any problem (e.g. in `copy`) + r.seed = deepcopy(seed) + dsfmt_init_by_array(r.state, reinterpret(UInt32, data)) + reset_caches!(r) + r.adv = 0 + r.adv_jump = 0 + return r +end + +# When a seed is not provided, we generate one via `RandomDevice()` rather +# than calling directly `initstate!` with `rand(RandomDevice(), UInt32, 8)` because the +# seed is printed in `show(::MersenneTwister)`, so we need one; the cost of `hash_seed` is a +# small overhead compared to `initstate!`. +# A random seed with 128 bits is a good compromise for almost surely getting distinct +# seeds, while having them printed reasonably tersely. +seed!(r::MersenneTwister, seeder::AbstractRNG) = seed!(r, rand(seeder, UInt128)) +seed!(r::MersenneTwister, ::Nothing) = seed!(r, RandomDevice()) +seed!(r::MersenneTwister, seed) = initstate!(r, rand(SeedHasher(seed), UInt32, 8), seed) + + +### generation + +# MersenneTwister produces natively Float64 +rng_native_52(::MersenneTwister) = Float64 + +#### helper functions + +# precondition: !mt_empty(r) +rand_inbounds(r::MersenneTwister, ::CloseOpen12_64) = mt_pop!(r) +rand_inbounds(r::MersenneTwister, ::CloseOpen01_64=CloseOpen01()) = + rand_inbounds(r, CloseOpen12()) - 1.0 + +rand_inbounds(r::MersenneTwister, ::UInt52Raw{T}) where {T<:BitInteger} = + reinterpret(UInt64, rand_inbounds(r, CloseOpen12())) % T + +function rand(r::MersenneTwister, x::SamplerTrivial{UInt52Raw{UInt64}}) + reserve_1(r) + rand_inbounds(r, x[]) +end + +function rand(r::MersenneTwister, ::SamplerTrivial{UInt2x52Raw{UInt128}}) + reserve(r, 2) + rand_inbounds(r, UInt52Raw(UInt128)) << 64 | rand_inbounds(r, UInt52Raw(UInt128)) +end + +function rand(r::MersenneTwister, ::SamplerTrivial{UInt104Raw{UInt128}}) + reserve(r, 2) + rand_inbounds(r, UInt52Raw(UInt128)) << 52 ⊻ rand_inbounds(r, UInt52Raw(UInt128)) +end + +#### floats + +rand(r::MersenneTwister, sp::SamplerTrivial{CloseOpen12_64}) = + (reserve_1(r); rand_inbounds(r, sp[])) + +#### integers + +rand(r::MersenneTwister, T::SamplerUnion(Int64, UInt64, Int128, UInt128)) = + mt_pop!(r, T[]) + +rand(r::MersenneTwister, T::SamplerUnion(Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32)) = + rand(r, UInt52Raw()) % T[] + +#### arrays of floats + +##### AbstractArray + +function rand!(r::MersenneTwister, A::AbstractArray{Float64}, + I::SamplerTrivial{<:FloatInterval_64}) + region = LinearIndices(A) + # what follows is equivalent to this simple loop but more efficient: + # for i=region + # @inbounds A[i] = rand(r, I[]) + # end + m = Base.checked_sub(first(region), 1) + n = last(region) + while m < n + s = mt_avail(r) + if s == 0 + gen_rand(r) + s = mt_avail(r) + end + m2 = min(n, m+s) + for i=m+1:m2 + @inbounds A[i] = rand_inbounds(r, I[]) + end + m = m2 + end + A +end + + +##### Array : internal functions + +# this is essentially equivalent to rand!(r, ::AbstractArray{Float64}, I) above, but due to +# optimizations which can't be done currently when working with pointers, we have to re-order +# manually the computation flow to get the performance +# (see https://discourse.julialang.org/t/unsafe-store-sometimes-slower-than-arrays-setindex) +function _rand_max383!(r::MersenneTwister, A::UnsafeView{Float64}, I::FloatInterval_64) + n = length(A) + @assert n <= dsfmt_get_min_array_size()+1 # == 383 + mt_avail(r) == 0 && gen_rand(r) + # from now on, at most one call to gen_rand(r) will be necessary + m = min(n, mt_avail(r)) + GC.@preserve r unsafe_copyto!(A.ptr, pointer(r.vals, r.idxF+1), m) + if m == n + r.idxF += m + else # m < n + gen_rand(r) + GC.@preserve r unsafe_copyto!(A.ptr+m*sizeof(Float64), pointer(r.vals), n-m) + r.idxF = n-m + end + if I isa CloseOpen01 + for i=1:n + A[i] -= 1.0 + end + end + A +end + +function fill_array!(rng::MersenneTwister, A::Ptr{Float64}, n::Int, I) + rng.adv += n + fill_array!(rng.state, A, n, I) +end + +fill_array!(s::DSFMT_state, A::Ptr{Float64}, n::Int, ::CloseOpen01_64) = + dsfmt_fill_array_close_open!(s, A, n) + +fill_array!(s::DSFMT_state, A::Ptr{Float64}, n::Int, ::CloseOpen12_64) = + dsfmt_fill_array_close1_open2!(s, A, n) + + +function rand!(r::MersenneTwister, A::UnsafeView{Float64}, + I::SamplerTrivial{<:FloatInterval_64}) + # depending on the alignment of A, the data written by fill_array! may have + # to be left-shifted by up to 15 bytes (cf. unsafe_copyto! below) for + # reproducibility purposes; + # so, even for well aligned arrays, fill_array! is used to generate only + # the n-2 first values (or n-3 if n is odd), and the remaining values are + # generated by the scalar version of rand + n = length(A) + n2 = (n-2) ÷ 2 * 2 + n2 < dsfmt_get_min_array_size() && return _rand_max383!(r, A, I[]) + + pA = A.ptr + align = Csize_t(pA) % 16 + if align > 0 + pA2 = pA + 16 - align + fill_array!(r, pA2, n2, I[]) # generate the data in-place, but shifted + unsafe_copyto!(pA, pA2, n2) # move the data to the beginning of the array + else + fill_array!(r, pA, n2, I[]) + end + for i=n2+1:n + A[i] = rand(r, I[]) + end + A +end + +# fills up A reinterpreted as an array of Float64 with n64 values +function _rand!(r::MersenneTwister, A::Array{T}, n64::Int, I::FloatInterval_64) where T + # n64 is the length in terms of `Float64` of the target + @assert sizeof(Float64)*n64 <= sizeof(T)*length(A) && isbitstype(T) + GC.@preserve A rand!(r, UnsafeView{Float64}(pointer(A), n64), SamplerTrivial(I)) + A +end + +##### Array: Float64, Float16, Float32 + +rand!(r::MersenneTwister, A::Array{Float64}, I::SamplerTrivial{<:FloatInterval_64}) = + _rand!(r, A, length(A), I[]) + +mask128(u::UInt128, ::Type{Float16}) = + (u & 0x03ff03ff03ff03ff03ff03ff03ff03ff) | 0x3c003c003c003c003c003c003c003c00 + +mask128(u::UInt128, ::Type{Float32}) = + (u & 0x007fffff007fffff007fffff007fffff) | 0x3f8000003f8000003f8000003f800000 + +for T in (Float16, Float32) + @eval function rand!(r::MersenneTwister, A::Array{$T}, ::SamplerTrivial{CloseOpen12{$T}}) + n = length(A) + n128 = n * sizeof($T) ÷ 16 + _rand!(r, A, 2*n128, CloseOpen12()) + GC.@preserve A begin + A128 = UnsafeView{UInt128}(pointer(A), n128) + for i in 1:n128 + u = A128[i] + u ⊻= u << 26 + # at this point, the 64 low bits of u, "k" being the k-th bit of A128[i] and "+" + # the bit xor, are: + # [..., 58+32,..., 53+27, 52+26, ..., 33+7, 32+6, ..., 27+1, 26, ..., 1] + # the bits needing to be random are + # [1:10, 17:26, 33:42, 49:58] (for Float16) + # [1:23, 33:55] (for Float32) + # this is obviously satisfied on the 32 low bits side, and on the high side, + # the entropy comes from bits 33:52 of A128[i] and then from bits 27:32 + # (which are discarded on the low side) + # this is similar for the 64 high bits of u + A128[i] = mask128(u, $T) + end + end + for i in 16*n128÷sizeof($T)+1:n + @inbounds A[i] = rand(r, $T) + one($T) + end + A + end + + @eval function rand!(r::MersenneTwister, A::Array{$T}, ::SamplerTrivial{CloseOpen01{$T}}) + rand!(r, A, CloseOpen12($T)) + I32 = one(Float32) + for i in eachindex(A) + @inbounds A[i] = Float32(A[i])-I32 # faster than "A[i] -= one(T)" for T==Float16 + end + A + end +end + +#### arrays of integers + +function rand!(r::MersenneTwister, A::UnsafeView{UInt128}, ::SamplerType{UInt128}) + n::Int=length(A) + i = n + while true + rand!(r, UnsafeView{Float64}(A.ptr, 2i), CloseOpen12()) + n < 5 && break + i = 0 + while n-i >= 5 + u = A[i+=1] + A[n] ⊻= u << 48 + A[n-=1] ⊻= u << 36 + A[n-=1] ⊻= u << 24 + A[n-=1] ⊻= u << 12 + n-=1 + end + end + if n > 0 + u = rand(r, UInt2x52Raw()) + for i = 1:n + A[i] ⊻= u << (12*i) + end + end + A +end + +for T in BitInteger_types + @eval function rand!(r::MersenneTwister, A::Array{$T}, sp::SamplerType{$T}) + GC.@preserve A rand!(r, UnsafeView(pointer(A), length(A)), sp) + A + end + + T == UInt128 && continue + + @eval function rand!(r::MersenneTwister, A::UnsafeView{$T}, ::SamplerType{$T}) + n = length(A) + n128 = n * sizeof($T) ÷ 16 + rand!(r, UnsafeView{UInt128}(pointer(A), n128)) + for i = 16*n128÷sizeof($T)+1:n + @inbounds A[i] = rand(r, $T) + end + A + end +end + + +#### arrays of Bool + +# similar to Array{UInt8}, but we need to mask the result so that only the LSB +# in each byte can be non-zero + +function rand!(r::MersenneTwister, A1::Array{Bool}, sp::SamplerType{Bool}) + n1 = length(A1) + n128 = n1 ÷ 16 + + if n128 == 0 + bits = rand(r, UInt52Raw()) + else + GC.@preserve A1 begin + A = UnsafeView{UInt128}(pointer(A1), n128) + rand!(r, UnsafeView{Float64}(A.ptr, 2*n128), CloseOpen12()) + # without masking, non-zero bits could be observed in other + # positions than the LSB of each byte + mask = 0x01010101010101010101010101010101 + # we need up to 15 bits of entropy in `bits` for the final loop, + # which we will extract from x = A[1] % UInt64; + # let y = x % UInt32; y contains 32 bits of entropy, but 4 + # of them will be used for A[1] itself (the first of + # each byte). To compensate, we xor with (y >> 17), + # which gets the entropy from the second bit of each byte + # of the upper-half of y, and sets it in the first bit + # of each byte of the lower half; the first two bytes + # now contain 16 usable random bits + x = A[1] % UInt64 + bits = x ⊻ x >> 17 + for i = 1:n128 + # << 5 to randomize the first bit of the 8th & 16th byte + # (i.e. we move bit 52 (resp. 52 + 64), which is unused, + # to position 57 (resp. 57 + 64)) + A[i] = (A[i] ⊻ A[i] << 5) & mask + end + end + end + for i = 16*n128+1:n1 + @inbounds A1[i] = bits % Bool + bits >>= 1 + end + A1 +end + + +### randjump + +# Old randjump methods are deprecated, the scalar version is in the Future module. + +function _randjump(r::MersenneTwister, jumppoly::DSFMT.GF2X) + adv = r.adv + adv_jump = r.adv_jump + s = MersenneTwister(r.seed, DSFMT.dsfmt_jump(r.state, jumppoly)) + reset_caches!(s) + s.adv = adv + s.adv_jump = adv_jump + s +end + +# NON-PUBLIC +function jump(r::MersenneTwister, steps::Integer) + iseven(steps) || throw(DomainError(steps, "steps must be even")) + # steps >= 0 checked in calc_jump (`steps >> 1 < 0` if `steps < 0`) + j = _randjump(r, Random.DSFMT.calc_jump(steps >> 1)) + j.adv_jump += steps + j +end + +# NON-PUBLIC +jump!(r::MersenneTwister, steps::Integer) = copy!(r, jump(r, steps)) + + +### constructors matching show (EXPERIMENTAL) + +# parameters in the tuples are: +# 1: .adv_jump (jump steps) +# 2: .adv (number of generated floats at the DSFMT_state level since seeding, besides jumps) +# 3, 4: .adv_vals, .idxF (counters to reconstruct the float cache, optional if 5-6 not shown)) +# 5, 6: .adv_ints, .idxI (counters to reconstruct the integer cache, optional) + +Random.MersenneTwister(seed, advance::NTuple{6,Integer}) = + advance!(MersenneTwister(seed), advance...) + +Random.MersenneTwister(seed, advance::NTuple{4,Integer}) = + MersenneTwister(seed, (advance..., 0, 0)) + +Random.MersenneTwister(seed, advance::NTuple{2,Integer}) = + MersenneTwister(seed, (advance..., 0, 0, 0, 0)) + +# advances raw state (per fill_array!) of r by n steps (Float64 values) +function _advance_n!(r::MersenneTwister, n::Int64, work::Vector{Float64}) + n == 0 && return + n < 0 && throw(DomainError(n, "can't advance $r to the specified state")) + ms = dsfmt_get_min_array_size() % Int64 + @assert n >= ms + lw = ms + n % ms + resize!(work, lw) + GC.@preserve work fill_array!(r, pointer(work), lw, CloseOpen12()) + c::Int64 = lw + GC.@preserve work while n > c + fill_array!(r, pointer(work), ms, CloseOpen12()) + c += ms + end + @assert n == c +end + +function _advance_to!(r::MersenneTwister, adv::Int64, work) + _advance_n!(r, adv - r.adv, work) + @assert r.adv == adv +end + +function _advance_F!(r::MersenneTwister, adv_vals, idxF, work) + _advance_to!(r, adv_vals, work) + gen_rand(r) + @assert r.adv_vals == adv_vals + r.idxF = idxF +end + +function _advance_I!(r::MersenneTwister, adv_ints, idxI, work) + _advance_to!(r, adv_ints, work) + mt_setfull!(r, Int) # sets r.adv_ints + @assert r.adv_ints == adv_ints + r.idxI = 16*length(r.ints) - 8*idxI +end + +function advance!(r::MersenneTwister, adv_jump, adv, adv_vals, idxF, adv_ints, idxI) + adv_jump = BigInt(adv_jump) + adv, adv_vals, adv_ints = Int64.((adv, adv_vals, adv_ints)) + idxF, idxI = Int.((idxF, idxI)) + + ms = dsfmt_get_min_array_size() % Int + work = sizehint!(Vector{Float64}(), 2ms) + + adv_jump != 0 && jump!(r, adv_jump) + advF = (adv_vals, idxF) != (0, 0) + advI = (adv_ints, idxI) != (0, 0) + + if advI && advF + @assert adv_vals != adv_ints + if adv_vals < adv_ints + _advance_F!(r, adv_vals, idxF, work) + _advance_I!(r, adv_ints, idxI, work) + else + _advance_I!(r, adv_ints, idxI, work) + _advance_F!(r, adv_vals, idxF, work) + end + elseif advF + _advance_F!(r, adv_vals, idxF, work) + elseif advI + _advance_I!(r, adv_ints, idxI, work) + else + @assert adv == 0 + end + _advance_to!(r, adv, work) + r +end diff --git a/stdlib/Random/src/RNGs.jl b/stdlib/Random/src/RNGs.jl index 0ae479e129b3b..bd3df8e54f194 100644 --- a/stdlib/Random/src/RNGs.jl +++ b/stdlib/Random/src/RNGs.jl @@ -1,379 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -## RandomDevice - - -""" - RandomDevice() - -Create a `RandomDevice` RNG object. -Two such objects will always generate different streams of random numbers. -The entropy is obtained from the operating system. -""" -struct RandomDevice <: AbstractRNG; end -RandomDevice(seed::Nothing) = RandomDevice() -seed!(rng::RandomDevice, ::Nothing) = rng - -rand(rd::RandomDevice, sp::SamplerBoolBitInteger) = Libc.getrandom!(Ref{sp[]}())[] -rand(rd::RandomDevice, ::SamplerType{Bool}) = rand(rd, UInt8) % Bool -function rand!(rd::RandomDevice, A::Array{Bool}, ::SamplerType{Bool}) - Libc.getrandom!(A) - # we need to mask the result so that only the LSB in each byte can be non-zero - GC.@preserve A begin - p = Ptr{UInt8}(pointer(A)) - for i = 1:length(A) - unsafe_store!(p, unsafe_load(p) & 0x1) - p += 1 - end - end - return A -end -for T in BitInteger_types - @eval rand!(rd::RandomDevice, A::Array{$T}, ::SamplerType{$T}) = Libc.getrandom!(A) -end - -# RandomDevice produces natively UInt64 -rng_native_52(::RandomDevice) = UInt64 - - -## MersenneTwister - -const MT_CACHE_F = 501 << 1 # number of Float64 in the cache -const MT_CACHE_I = 501 << 4 # number of bytes in the UInt128 cache - -@assert dsfmt_get_min_array_size() <= MT_CACHE_F - -mutable struct MersenneTwister <: AbstractRNG - seed::Any - state::DSFMT_state - vals::Vector{Float64} - ints::Vector{UInt128} - idxF::Int - idxI::Int - - # counters for show - adv::Int64 # state of advance at the DSFMT_state level - adv_jump::BigInt # number of skipped Float64 values via randjump - adv_vals::Int64 # state of advance when vals is filled-up - adv_ints::Int64 # state of advance when ints is filled-up - - function MersenneTwister(seed, state, vals, ints, idxF, idxI, - adv, adv_jump, adv_vals, adv_ints) - length(vals) == MT_CACHE_F && 0 <= idxF <= MT_CACHE_F || - throw(DomainError((length(vals), idxF), - "`length(vals)` and `idxF` must be consistent with $MT_CACHE_F")) - length(ints) == MT_CACHE_I >> 4 && 0 <= idxI <= MT_CACHE_I || - throw(DomainError((length(ints), idxI), - "`length(ints)` and `idxI` must be consistent with $MT_CACHE_I")) - new(seed, state, vals, ints, idxF, idxI, - adv, adv_jump, adv_vals, adv_ints) - end -end - -MersenneTwister(seed, state::DSFMT_state) = - MersenneTwister(seed, state, - Vector{Float64}(undef, MT_CACHE_F), - Vector{UInt128}(undef, MT_CACHE_I >> 4), - MT_CACHE_F, 0, 0, 0, -1, -1) - -""" - MersenneTwister(seed) - MersenneTwister() - -Create a `MersenneTwister` RNG object. Different RNG objects can have -their own seeds, which may be useful for generating different streams -of random numbers. -The `seed` may be an integer, a string, or a vector of `UInt32` integers. -If no seed is provided, a randomly generated one is created (using entropy from the system). -See the [`seed!`](@ref) function for reseeding an already existing `MersenneTwister` object. - -!!! compat "Julia 1.11" - Passing a negative integer seed requires at least Julia 1.11. - -# Examples -```jldoctest -julia> rng = MersenneTwister(123); - -julia> x1 = rand(rng, 2) -2-element Vector{Float64}: - 0.37453777969575874 - 0.8735343642013971 - -julia> x2 = rand(MersenneTwister(123), 2) -2-element Vector{Float64}: - 0.37453777969575874 - 0.8735343642013971 - -julia> x1 == x2 -true -``` -""" -MersenneTwister(seed=nothing) = - seed!(MersenneTwister(Vector{UInt32}(), DSFMT_state()), seed) - - -function copy!(dst::MersenneTwister, src::MersenneTwister) - dst.seed = src.seed - copy!(dst.state, src.state) - copyto!(dst.vals, src.vals) - copyto!(dst.ints, src.ints) - dst.idxF = src.idxF - dst.idxI = src.idxI - dst.adv = src.adv - dst.adv_jump = src.adv_jump - dst.adv_vals = src.adv_vals - dst.adv_ints = src.adv_ints - dst -end - -copy(src::MersenneTwister) = - MersenneTwister(src.seed, copy(src.state), copy(src.vals), copy(src.ints), - src.idxF, src.idxI, src.adv, src.adv_jump, src.adv_vals, src.adv_ints) - - -==(r1::MersenneTwister, r2::MersenneTwister) = - r1.seed == r2.seed && r1.state == r2.state && - isequal(r1.vals, r2.vals) && - isequal(r1.ints, r2.ints) && - r1.idxF == r2.idxF && r1.idxI == r2.idxI - -hash(r::MersenneTwister, h::UInt) = - foldr(hash, (r.seed, r.state, r.vals, r.ints, r.idxF, r.idxI); init=h) - -function show(io::IO, rng::MersenneTwister) - # seed - if rng.adv_jump == 0 && rng.adv == 0 - return print(io, MersenneTwister, "(", repr(rng.seed), ")") - end - print(io, MersenneTwister, "(", repr(rng.seed), ", (") - # state - adv = Integer[rng.adv_jump, rng.adv] - if rng.adv_vals != -1 || rng.adv_ints != -1 - if rng.adv_vals == -1 - @assert rng.idxF == MT_CACHE_F - push!(adv, 0, 0) # "(0, 0)" is nicer on the eyes than (-1, 1002) - else - push!(adv, rng.adv_vals, rng.idxF) - end - end - if rng.adv_ints != -1 - idxI = (length(rng.ints)*16 - rng.idxI) / 8 # 8 represents one Int64 - idxI = Int(idxI) # idxI should always be an integer when using public APIs - push!(adv, rng.adv_ints, idxI) - end - join(io, adv, ", ") - print(io, "))") -end - -### low level API - -function reset_caches!(r::MersenneTwister) - # zeroing the caches makes comparing two MersenneTwister RNGs easier - fill!(r.vals, 0.0) - fill!(r.ints, zero(UInt128)) - mt_setempty!(r) - mt_setempty!(r, UInt128) - r.adv_vals = -1 - r.adv_ints = -1 - r -end - -#### floats - -mt_avail(r::MersenneTwister) = MT_CACHE_F - r.idxF -mt_empty(r::MersenneTwister) = r.idxF == MT_CACHE_F -mt_setfull!(r::MersenneTwister) = r.idxF = 0 -mt_setempty!(r::MersenneTwister) = r.idxF = MT_CACHE_F -mt_pop!(r::MersenneTwister) = @inbounds return r.vals[r.idxF+=1] - -@noinline function gen_rand(r::MersenneTwister) - r.adv_vals = r.adv - GC.@preserve r fill_array!(r, pointer(r.vals), length(r.vals), CloseOpen12()) - mt_setfull!(r) -end - -reserve_1(r::MersenneTwister) = (mt_empty(r) && gen_rand(r); nothing) -# `reserve` allows one to call `rand_inbounds` n times -# precondition: n <= MT_CACHE_F -reserve(r::MersenneTwister, n::Int) = (mt_avail(r) < n && gen_rand(r); nothing) - -#### ints - -logsizeof(::Type{<:Union{Bool,Int8,UInt8}}) = 0 -logsizeof(::Type{<:Union{Int16,UInt16}}) = 1 -logsizeof(::Type{<:Union{Int32,UInt32}}) = 2 -logsizeof(::Type{<:Union{Int64,UInt64}}) = 3 -logsizeof(::Type{<:Union{Int128,UInt128}}) = 4 - -idxmask(::Type{<:Union{Bool,Int8,UInt8}}) = 15 -idxmask(::Type{<:Union{Int16,UInt16}}) = 7 -idxmask(::Type{<:Union{Int32,UInt32}}) = 3 -idxmask(::Type{<:Union{Int64,UInt64}}) = 1 -idxmask(::Type{<:Union{Int128,UInt128}}) = 0 - - -mt_avail(r::MersenneTwister, ::Type{T}) where {T<:BitInteger} = - r.idxI >> logsizeof(T) - -function mt_setfull!(r::MersenneTwister, ::Type{<:BitInteger}) - r.adv_ints = r.adv - ints = r.ints - - @assert length(ints) == 501 - # dSFMT natively randomizes 52 out of 64 bits of each UInt64 words, - # i.e. 12 bits are missing; - # by generating 5 words == 5*52 == 260 bits, we can fully - # randomize 4 UInt64 = 256 bits; IOW, at the array level, we must - # randomize ceil(501*1.25) = 627 UInt128 words (with 2*52 bits each), - # which we then condense into fully randomized 501 UInt128 words - - len = 501 + 126 # 126 == ceil(501 / 4) - resize!(ints, len) - p = pointer(ints) # must be *after* resize! - GC.@preserve r fill_array!(r, Ptr{Float64}(p), len*2, CloseOpen12_64()) - - k = 501 - n = 0 - @inbounds while n != 500 - u = ints[k+=1] - ints[n+=1] ⊻= u << 48 - ints[n+=1] ⊻= u << 36 - ints[n+=1] ⊻= u << 24 - ints[n+=1] ⊻= u << 12 - end - @assert k == len - 1 - @inbounds ints[501] ⊻= ints[len] << 48 - resize!(ints, 501) - r.idxI = MT_CACHE_I -end - -mt_setempty!(r::MersenneTwister, ::Type{<:BitInteger}) = r.idxI = 0 - -function reserve1(r::MersenneTwister, ::Type{T}) where T<:BitInteger - r.idxI < sizeof(T) && mt_setfull!(r, T) - nothing -end - -function mt_pop!(r::MersenneTwister, ::Type{T}) where T<:BitInteger - reserve1(r, T) - r.idxI -= sizeof(T) - i = r.idxI - @inbounds x128 = r.ints[1 + i >> 4] - i128 = (i >> logsizeof(T)) & idxmask(T) # 0-based "indice" in x128 - (x128 >> (i128 * (sizeof(T) << 3))) % T -end - -function mt_pop!(r::MersenneTwister, ::Type{T}) where {T<:Union{Int128,UInt128}} - reserve1(r, T) - idx = r.idxI >> 4 - r.idxI = idx << 4 - 16 - @inbounds r.ints[idx] % T -end - - -### seeding - -#### random_seed() & hash_seed() - -# random_seed tries to produce a random seed of type UInt128 from system entropy -function random_seed() - try - # as MersenneTwister prints its seed when `show`ed, 128 bits is a good compromise for - # almost surely always getting distinct seeds, while having them printed reasonably tersely - return rand(RandomDevice(), UInt128) - catch ex - ex isa IOError || rethrow() - @warn "Entropy pool not available to seed RNG; using ad-hoc entropy sources." - return Libc.rand() - end -end - -function hash_seed(seed::Integer) - ctx = SHA.SHA2_256_CTX() - neg = signbit(seed) - if neg - seed = ~seed - end - @assert seed >= 0 - while true - word = (seed % UInt32) & 0xffffffff - seed >>>= 32 - SHA.update!(ctx, reinterpret(NTuple{4, UInt8}, word)) - iszero(seed) && break - end - # make sure the hash of negative numbers is different from the hash of positive numbers - neg && SHA.update!(ctx, (0x01,)) - SHA.digest!(ctx) -end - -function hash_seed(seed::Union{AbstractArray{UInt32}, AbstractArray{UInt64}}) - ctx = SHA.SHA2_256_CTX() - for xx in seed - SHA.update!(ctx, reinterpret(NTuple{8, UInt8}, UInt64(xx))) - end - # discriminate from hash_seed(::Integer) - SHA.update!(ctx, (0x10,)) - SHA.digest!(ctx) -end - -function hash_seed(str::AbstractString) - ctx = SHA.SHA2_256_CTX() - # convert to String such that `codeunits(str)` below is consistent between equal - # strings of different types - str = String(str) - SHA.update!(ctx, codeunits(str)) - # signature for strings: so far, all hash_seed functions end-up hashing a multiple - # of 4 bytes of data, and add the signature (1 byte) at the end; so hash as many - # bytes as necessary to have a total number of hashed bytes equal to 0 mod 4 (padding), - # and then hash the signature 0x05; in order for strings of different lengths to have - # different hashes, padding bytes are set equal to the number of padding bytes - pad = 4 - mod(ncodeunits(str), 4) - for _=1:pad - SHA.update!(ctx, (pad % UInt8,)) - end - SHA.update!(ctx, (0x05,)) - SHA.digest!(ctx) -end - - -""" - hash_seed(seed)::AbstractVector{UInt8} - -Return a cryptographic hash of `seed` of size 256 bits (32 bytes). -`seed` can currently be of type -`Union{Integer, AbstractString, AbstractArray{UInt32}, AbstractArray{UInt64}}`, -but modules can extend this function for types they own. - -`hash_seed` is "injective" : if `n != m`, then `hash_seed(n) != `hash_seed(m)`. -Moreover, if `n == m`, then `hash_seed(n) == hash_seed(m)`. - -This is an internal function subject to change. -""" -hash_seed - -#### seed!() - -function initstate!(r::MersenneTwister, data::StridedVector, seed) - # we deepcopy `seed` because the caller might mutate it, and it's useful - # to keep it constant inside `MersenneTwister`; but multiple instances - # can share the same seed without any problem (e.g. in `copy`) - r.seed = deepcopy(seed) - dsfmt_init_by_array(r.state, reinterpret(UInt32, data)) - reset_caches!(r) - r.adv = 0 - r.adv_jump = 0 - return r -end - -# when a seed is not provided, we generate one via `RandomDevice()` in `random_seed()` rather -# than calling directly `initstate!` with `rand(RandomDevice(), UInt32, whatever)` because the -# seed is printed in `show(::MersenneTwister)`, so we need one; the cost of `hash_seed` is a -# small overhead compared to `initstate!`, so this simple solution is fine -seed!(r::MersenneTwister, ::Nothing) = seed!(r, random_seed()) -seed!(r::MersenneTwister, seed) = initstate!(r, hash_seed(seed), seed) - - -### Global RNG +## default RNG """ Random.default_rng() -> rng @@ -433,416 +60,249 @@ function __init__() end -### generation - -# MersenneTwister produces natively Float64 -rng_native_52(::MersenneTwister) = Float64 +## RandomDevice -#### helper functions +""" + RandomDevice() -# precondition: !mt_empty(r) -rand_inbounds(r::MersenneTwister, ::CloseOpen12_64) = mt_pop!(r) -rand_inbounds(r::MersenneTwister, ::CloseOpen01_64=CloseOpen01()) = - rand_inbounds(r, CloseOpen12()) - 1.0 +Create a `RandomDevice` RNG object. +Two such objects will always generate different streams of random numbers. +The entropy is obtained from the operating system. +""" +struct RandomDevice <: AbstractRNG; end +RandomDevice(seed::Nothing) = RandomDevice() +seed!(rng::RandomDevice, ::Nothing) = rng -rand_inbounds(r::MersenneTwister, ::UInt52Raw{T}) where {T<:BitInteger} = - reinterpret(UInt64, rand_inbounds(r, CloseOpen12())) % T +rand(rd::RandomDevice, sp::SamplerBoolBitInteger) = Libc.getrandom!(Ref{sp[]}())[] +rand(rd::RandomDevice, ::SamplerType{Bool}) = rand(rd, UInt8) % Bool -function rand(r::MersenneTwister, x::SamplerTrivial{UInt52Raw{UInt64}}) - reserve_1(r) - rand_inbounds(r, x[]) -end +# specialization for homogeneous tuple types of builtin integers, to avoid +# repeated system calls +rand(rd::RandomDevice, sp::SamplerTag{Ref{Tuple{Vararg{T, N}}}, Tuple{S}} + ) where {T, N, S <: SamplerUnion(Base.BitInteger_types...)} = + Libc.getrandom!(Ref{gentype(sp)}())[] -function rand(r::MersenneTwister, ::SamplerTrivial{UInt2x52Raw{UInt128}}) - reserve(r, 2) - rand_inbounds(r, UInt52Raw(UInt128)) << 64 | rand_inbounds(r, UInt52Raw(UInt128)) +function rand!(rd::RandomDevice, A::Array{Bool}, ::SamplerType{Bool}) + Libc.getrandom!(A) + # we need to mask the result so that only the LSB in each byte can be non-zero + GC.@preserve A begin + p = Ptr{UInt8}(pointer(A)) + for i = 1:length(A) + unsafe_store!(p, unsafe_load(p) & 0x1) + p += 1 + end + end + return A end - -function rand(r::MersenneTwister, ::SamplerTrivial{UInt104Raw{UInt128}}) - reserve(r, 2) - rand_inbounds(r, UInt52Raw(UInt128)) << 52 ⊻ rand_inbounds(r, UInt52Raw(UInt128)) +for T in BitInteger_types + @eval rand!(rd::RandomDevice, A::Array{$T}, ::SamplerType{$T}) = Libc.getrandom!(A) end -#### floats +# RandomDevice produces natively UInt64 +rng_native_52(::RandomDevice) = UInt64 -rand(r::MersenneTwister, sp::SamplerTrivial{CloseOpen12_64}) = - (reserve_1(r); rand_inbounds(r, sp[])) -#### integers +## SeedHasher -rand(r::MersenneTwister, T::SamplerUnion(Int64, UInt64, Int128, UInt128)) = - mt_pop!(r, T[]) +""" + Random.SeedHasher(seed=nothing) -rand(r::MersenneTwister, T::SamplerUnion(Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32)) = - rand(r, UInt52Raw()) % T[] +Create a `Random.SeedHasher` RNG object, which generates random bytes with the help +of a cryptographic hash function (SHA2), via calls to [`Random.hash_seed`](@ref). -#### arrays of floats +Given two seeds `s1` and `s2`, the random streams generated by +`SeedHasher(s1)` and `SeedHasher(s2)` should be distinct if and only if +`s1` and `s2` are distinct. -##### AbstractArray +This RNG is used by default in `Random.seed!(::AbstractRNG, seed::Any)`, such that +RNGs usually need only to implement `seed!(rng, ::AbstractRNG)`. -function rand!(r::MersenneTwister, A::AbstractArray{Float64}, - I::SamplerTrivial{<:FloatInterval_64}) - region = LinearIndices(A) - # what follows is equivalent to this simple loop but more efficient: - # for i=region - # @inbounds A[i] = rand(r, I[]) - # end - m = Base.checked_sub(first(region), 1) - n = last(region) - while m < n - s = mt_avail(r) - if s == 0 - gen_rand(r) - s = mt_avail(r) - end - m2 = min(n, m+s) - for i=m+1:m2 - @inbounds A[i] = rand_inbounds(r, I[]) +This is an internal type, subject to change. +""" +mutable struct SeedHasher <: AbstractRNG + bytes::Vector{UInt8} + idx::Int + cnt::Int64 + + SeedHasher(seed=nothing) = seed!(new(), seed) +end + +seed!(rng::SeedHasher, seeder::AbstractRNG) = seed!(rng, rand(seeder, UInt64, 4)) +seed!(rng::SeedHasher, ::Nothing) = seed!(rng, RandomDevice()) + +function seed!(rng::SeedHasher, seed) + # typically, no more than 256 bits will be needed, so use + # SHA2_256 because it's faster + ctx = SHA2_256_CTX() + hash_seed(seed, ctx) + rng.bytes = SHA.digest!(ctx)::Vector{UInt8} + rng.idx = 0 + rng.cnt = 0 + rng +end + +@noinline function rehash!(rng::SeedHasher) + # more random bytes are necessary, from now on use SHA2_512 to generate + # more bytes at once + ctx = SHA2_512_CTX() + SHA.update!(ctx, rng.bytes) + # also hash the counter, just for the extremely unlikely case where the hash of + # rng.bytes is equal to rng.bytes (i.e. rng.bytes is a "fixed point"), or more generally + # if there is a small cycle + SHA.update!(ctx, reinterpret(NTuple{8, UInt8}, rng.cnt += 1)) + rng.bytes = SHA.digest!(ctx) + rng.idx = 0 + rng +end + +function rand(rng::SeedHasher, ::SamplerType{UInt8}) + rng.idx < length(rng.bytes) || rehash!(rng) + rng.bytes[rng.idx += 1] +end + +for TT = Base.BitInteger_types + TT === UInt8 && continue + @eval function rand(rng::SeedHasher, ::SamplerType{$TT}) + xx = zero($TT) + for ii = 0:sizeof($TT)-1 + xx |= (rand(rng, UInt8) % $TT) << (8 * ii) end - m = m2 + xx end - A end +rand(rng::SeedHasher, ::SamplerType{Bool}) = rand(rng, UInt8) % Bool -##### Array : internal functions +rng_native_52(::SeedHasher) = UInt64 -# internal array-like type to circumvent the lack of flexibility with reinterpret -struct UnsafeView{T} <: DenseArray{T,1} - ptr::Ptr{T} - len::Int -end -Base.length(a::UnsafeView) = a.len -Base.getindex(a::UnsafeView, i::Int) = unsafe_load(a.ptr, i) -Base.setindex!(a::UnsafeView, x, i::Int) = unsafe_store!(a.ptr, x, i) -Base.pointer(a::UnsafeView) = a.ptr -Base.size(a::UnsafeView) = (a.len,) -Base.elsize(::Type{UnsafeView{T}}) where {T} = sizeof(T) - -# this is essentially equivalent to rand!(r, ::AbstractArray{Float64}, I) above, but due to -# optimizations which can't be done currently when working with pointers, we have to re-order -# manually the computation flow to get the performance -# (see https://discourse.julialang.org/t/unsafe-store-sometimes-slower-than-arrays-setindex) -function _rand_max383!(r::MersenneTwister, A::UnsafeView{Float64}, I::FloatInterval_64) - n = length(A) - @assert n <= dsfmt_get_min_array_size()+1 # == 383 - mt_avail(r) == 0 && gen_rand(r) - # from now on, at most one call to gen_rand(r) will be necessary - m = min(n, mt_avail(r)) - GC.@preserve r unsafe_copyto!(A.ptr, pointer(r.vals, r.idxF+1), m) - if m == n - r.idxF += m - else # m < n - gen_rand(r) - GC.@preserve r unsafe_copyto!(A.ptr+m*sizeof(Float64), pointer(r.vals), n-m) - r.idxF = n-m - end - if I isa CloseOpen01 - for i=1:n - A[i] -= 1.0 - end - end - A -end +## seeding -function fill_array!(rng::MersenneTwister, A::Ptr{Float64}, n::Int, I) - rng.adv += n - fill_array!(rng.state, A, n, I) -end +""" + seed!([rng=default_rng()], seed) -> rng + seed!([rng=default_rng()]) -> rng -fill_array!(s::DSFMT_state, A::Ptr{Float64}, n::Int, ::CloseOpen01_64) = - dsfmt_fill_array_close_open!(s, A, n) - -fill_array!(s::DSFMT_state, A::Ptr{Float64}, n::Int, ::CloseOpen12_64) = - dsfmt_fill_array_close1_open2!(s, A, n) - - -function rand!(r::MersenneTwister, A::UnsafeView{Float64}, - I::SamplerTrivial{<:FloatInterval_64}) - # depending on the alignment of A, the data written by fill_array! may have - # to be left-shifted by up to 15 bytes (cf. unsafe_copyto! below) for - # reproducibility purposes; - # so, even for well aligned arrays, fill_array! is used to generate only - # the n-2 first values (or n-3 if n is odd), and the remaining values are - # generated by the scalar version of rand - n = length(A) - n2 = (n-2) ÷ 2 * 2 - n2 < dsfmt_get_min_array_size() && return _rand_max383!(r, A, I[]) - - pA = A.ptr - align = Csize_t(pA) % 16 - if align > 0 - pA2 = pA + 16 - align - fill_array!(r, pA2, n2, I[]) # generate the data in-place, but shifted - unsafe_copyto!(pA, pA2, n2) # move the data to the beginning of the array - else - fill_array!(r, pA, n2, I[]) - end - for i=n2+1:n - A[i] = rand(r, I[]) - end - A -end +Reseed the random number generator: `rng` will give a reproducible +sequence of numbers if and only if a `seed` is provided. Some RNGs +don't accept a seed, like `RandomDevice`. +After the call to `seed!`, `rng` is equivalent to a newly created +object initialized with the same seed. -# fills up A reinterpreted as an array of Float64 with n64 values -function _rand!(r::MersenneTwister, A::Array{T}, n64::Int, I::FloatInterval_64) where T - # n64 is the length in terms of `Float64` of the target - @assert sizeof(Float64)*n64 <= sizeof(T)*length(A) && isbitstype(T) - GC.@preserve A rand!(r, UnsafeView{Float64}(pointer(A), n64), SamplerTrivial(I)) - A -end +The types of accepted seeds depend on the type of `rng`, but in general, +integer seeds should work. Providing `nothing` as the seed should be +equivalent to not providing one. -##### Array: Float64, Float16, Float32 - -rand!(r::MersenneTwister, A::Array{Float64}, I::SamplerTrivial{<:FloatInterval_64}) = - _rand!(r, A, length(A), I[]) - -mask128(u::UInt128, ::Type{Float16}) = - (u & 0x03ff03ff03ff03ff03ff03ff03ff03ff) | 0x3c003c003c003c003c003c003c003c00 - -mask128(u::UInt128, ::Type{Float32}) = - (u & 0x007fffff007fffff007fffff007fffff) | 0x3f8000003f8000003f8000003f800000 - -for T in (Float16, Float32) - @eval function rand!(r::MersenneTwister, A::Array{$T}, ::SamplerTrivial{CloseOpen12{$T}}) - n = length(A) - n128 = n * sizeof($T) ÷ 16 - _rand!(r, A, 2*n128, CloseOpen12()) - GC.@preserve A begin - A128 = UnsafeView{UInt128}(pointer(A), n128) - for i in 1:n128 - u = A128[i] - u ⊻= u << 26 - # at this point, the 64 low bits of u, "k" being the k-th bit of A128[i] and "+" - # the bit xor, are: - # [..., 58+32,..., 53+27, 52+26, ..., 33+7, 32+6, ..., 27+1, 26, ..., 1] - # the bits needing to be random are - # [1:10, 17:26, 33:42, 49:58] (for Float16) - # [1:23, 33:55] (for Float32) - # this is obviously satisfied on the 32 low bits side, and on the high side, - # the entropy comes from bits 33:52 of A128[i] and then from bits 27:32 - # (which are discarded on the low side) - # this is similar for the 64 high bits of u - A128[i] = mask128(u, $T) - end - end - for i in 16*n128÷sizeof($T)+1:n - @inbounds A[i] = rand(r, $T) + one($T) - end - A - end +If `rng` is not specified, it defaults to seeding the state of the +shared task-local generator. - @eval function rand!(r::MersenneTwister, A::Array{$T}, ::SamplerTrivial{CloseOpen01{$T}}) - rand!(r, A, CloseOpen12($T)) - I32 = one(Float32) - for i in eachindex(A) - @inbounds A[i] = Float32(A[i])-I32 # faster than "A[i] -= one(T)" for T==Float16 - end - A - end -end +# Examples +```jldoctest; filter = r"(true|false)" +julia> Random.seed!(1234); -#### arrays of integers +julia> x1 = rand(2) +2-element Vector{Float64}: + 0.32597672886359486 + 0.5490511363155669 -function rand!(r::MersenneTwister, A::UnsafeView{UInt128}, ::SamplerType{UInt128}) - n::Int=length(A) - i = n - while true - rand!(r, UnsafeView{Float64}(A.ptr, 2i), CloseOpen12()) - n < 5 && break - i = 0 - while n-i >= 5 - u = A[i+=1] - A[n] ⊻= u << 48 - A[n-=1] ⊻= u << 36 - A[n-=1] ⊻= u << 24 - A[n-=1] ⊻= u << 12 - n-=1 - end - end - if n > 0 - u = rand(r, UInt2x52Raw()) - for i = 1:n - A[i] ⊻= u << (12*i) - end - end - A -end +julia> Random.seed!(1234); -for T in BitInteger_types - @eval function rand!(r::MersenneTwister, A::Array{$T}, sp::SamplerType{$T}) - GC.@preserve A rand!(r, UnsafeView(pointer(A), length(A)), sp) - A - end - - T == UInt128 && continue +julia> x2 = rand(2) +2-element Vector{Float64}: + 0.32597672886359486 + 0.5490511363155669 - @eval function rand!(r::MersenneTwister, A::UnsafeView{$T}, ::SamplerType{$T}) - n = length(A) - n128 = n * sizeof($T) ÷ 16 - rand!(r, UnsafeView{UInt128}(pointer(A), n128)) - for i = 16*n128÷sizeof($T)+1:n - @inbounds A[i] = rand(r, $T) - end - A - end -end +julia> x1 == x2 +true +julia> rng = Xoshiro(1234); rand(rng, 2) == x1 +true -#### arrays of Bool +julia> Xoshiro(1) == Random.seed!(rng, 1) +true -# similar to Array{UInt8}, but we need to mask the result so that only the LSB -# in each byte can be non-zero +julia> rand(Random.seed!(rng), Bool) # not reproducible +true -function rand!(r::MersenneTwister, A1::Array{Bool}, sp::SamplerType{Bool}) - n1 = length(A1) - n128 = n1 ÷ 16 +julia> rand(Random.seed!(rng), Bool) # not reproducible either +false - if n128 == 0 - bits = rand(r, UInt52Raw()) +julia> rand(Xoshiro(), Bool) # not reproducible either +true +``` +""" +seed! + +function seed!(rng::AbstractRNG, seed::Any=nothing) + if seed === nothing + seed!(rng, RandomDevice()) + elseif seed isa AbstractRNG + # avoid getting into an infinite recursive call from the other branches + throw(MethodError(seed!, (rng, seed))) else - GC.@preserve A1 begin - A = UnsafeView{UInt128}(pointer(A1), n128) - rand!(r, UnsafeView{Float64}(A.ptr, 2*n128), CloseOpen12()) - # without masking, non-zero bits could be observed in other - # positions than the LSB of each byte - mask = 0x01010101010101010101010101010101 - # we need up to 15 bits of entropy in `bits` for the final loop, - # which we will extract from x = A[1] % UInt64; - # let y = x % UInt32; y contains 32 bits of entropy, but 4 - # of them will be used for A[1] itself (the first of - # each byte). To compensate, we xor with (y >> 17), - # which gets the entropy from the second bit of each byte - # of the upper-half of y, and sets it in the first bit - # of each byte of the lower half; the first two bytes - # now contain 16 usable random bits - x = A[1] % UInt64 - bits = x ⊻ x >> 17 - for i = 1:n128 - # << 5 to randomize the first bit of the 8th & 16th byte - # (i.e. we move bit 52 (resp. 52 + 64), which is unused, - # to position 57 (resp. 57 + 64)) - A[i] = (A[i] ⊻ A[i] << 5) & mask - end - end + seed!(rng, SeedHasher(seed)) end - for i = 16*n128+1:n1 - @inbounds A1[i] = bits % Bool - bits >>= 1 - end - A1 end -### randjump - -# Old randjump methods are deprecated, the scalar version is in the Future module. +### hash_seed() -function _randjump(r::MersenneTwister, jumppoly::DSFMT.GF2X) - adv = r.adv - adv_jump = r.adv_jump - s = MersenneTwister(r.seed, DSFMT.dsfmt_jump(r.state, jumppoly)) - reset_caches!(s) - s.adv = adv - s.adv_jump = adv_jump - s -end - -# NON-PUBLIC -function jump(r::MersenneTwister, steps::Integer) - iseven(steps) || throw(DomainError(steps, "steps must be even")) - # steps >= 0 checked in calc_jump (`steps >> 1 < 0` if `steps < 0`) - j = _randjump(r, Random.DSFMT.calc_jump(steps >> 1)) - j.adv_jump += steps - j -end - -# NON-PUBLIC -jump!(r::MersenneTwister, steps::Integer) = copy!(r, jump(r, steps)) - - -### constructors matching show (EXPERIMENTAL) - -# parameters in the tuples are: -# 1: .adv_jump (jump steps) -# 2: .adv (number of generated floats at the DSFMT_state level since seeding, besides jumps) -# 3, 4: .adv_vals, .idxF (counters to reconstruct the float cache, optional if 5-6 not shown)) -# 5, 6: .adv_ints, .idxI (counters to reconstruct the integer cache, optional) +""" + Random.hash_seed(seed, ctx::SHA_CTX)::AbstractVector{UInt8} -Random.MersenneTwister(seed, advance::NTuple{6,Integer}) = - advance!(MersenneTwister(seed), advance...) +Update `ctx` via `SHA.update!` with the content of `seed`. +This function is used by the [`SeedHasher`](@ref) RNG to produce +random bytes. -Random.MersenneTwister(seed, advance::NTuple{4,Integer}) = - MersenneTwister(seed, (advance..., 0, 0)) +`seed` can currently be of type +`Union{Integer, AbstractString, AbstractArray{UInt32}, AbstractArray{UInt64}}`, +but modules can extend this function for types they own. -Random.MersenneTwister(seed, advance::NTuple{2,Integer}) = - MersenneTwister(seed, (advance..., 0, 0, 0, 0)) +`hash_seed` is "injective" : for two equivalent context objects `cn` and `cm`, +if `n != m`, then `cn` and `cm` will be distinct after calling +`hash_seed(n, cn); hash_seed(m, cm)`. +Moreover, if `n == m`, then `cn` and `cm` remain equivalent after calling +`hash_seed(n, cn); hash_seed(m, cm)`. +""" +function hash_seed end -# advances raw state (per fill_array!) of r by n steps (Float64 values) -function _advance_n!(r::MersenneTwister, n::Int64, work::Vector{Float64}) - n == 0 && return - n < 0 && throw(DomainError(n, "can't advance $r to the specified state")) - ms = dsfmt_get_min_array_size() % Int64 - @assert n >= ms - lw = ms + n % ms - resize!(work, lw) - GC.@preserve work fill_array!(r, pointer(work), lw, CloseOpen12()) - c::Int64 = lw - GC.@preserve work while n > c - fill_array!(r, pointer(work), ms, CloseOpen12()) - c += ms +function hash_seed(seed::Integer, ctx::SHA_CTX) + neg = signbit(seed) + if neg + seed = ~seed end - @assert n == c -end - -function _advance_to!(r::MersenneTwister, adv::Int64, work) - _advance_n!(r, adv - r.adv, work) - @assert r.adv == adv -end - -function _advance_F!(r::MersenneTwister, adv_vals, idxF, work) - _advance_to!(r, adv_vals, work) - gen_rand(r) - @assert r.adv_vals == adv_vals - r.idxF = idxF + @assert seed >= 0 + while true + word = (seed % UInt32) & 0xffffffff + seed >>>= 32 + SHA.update!(ctx, reinterpret(NTuple{4, UInt8}, word)) + iszero(seed) && break + end + # make sure the hash of negative numbers is different from the hash of positive numbers + neg && SHA.update!(ctx, (0x01,)) + nothing end -function _advance_I!(r::MersenneTwister, adv_ints, idxI, work) - _advance_to!(r, adv_ints, work) - mt_setfull!(r, Int) # sets r.adv_ints - @assert r.adv_ints == adv_ints - r.idxI = 16*length(r.ints) - 8*idxI +function hash_seed(seed::Union{AbstractArray{UInt32}, AbstractArray{UInt64}}, ctx::SHA_CTX) + for xx in seed + SHA.update!(ctx, reinterpret(NTuple{8, UInt8}, UInt64(xx))) + end + # discriminate from hash_seed(::Integer) + SHA.update!(ctx, (0x10,)) end -function advance!(r::MersenneTwister, adv_jump, adv, adv_vals, idxF, adv_ints, idxI) - adv_jump = BigInt(adv_jump) - adv, adv_vals, adv_ints = Int64.((adv, adv_vals, adv_ints)) - idxF, idxI = Int.((idxF, idxI)) - - ms = dsfmt_get_min_array_size() % Int - work = sizehint!(Vector{Float64}(), 2ms) - - adv_jump != 0 && jump!(r, adv_jump) - advF = (adv_vals, idxF) != (0, 0) - advI = (adv_ints, idxI) != (0, 0) - - if advI && advF - @assert adv_vals != adv_ints - if adv_vals < adv_ints - _advance_F!(r, adv_vals, idxF, work) - _advance_I!(r, adv_ints, idxI, work) - else - _advance_I!(r, adv_ints, idxI, work) - _advance_F!(r, adv_vals, idxF, work) - end - elseif advF - _advance_F!(r, adv_vals, idxF, work) - elseif advI - _advance_I!(r, adv_ints, idxI, work) - else - @assert adv == 0 +function hash_seed(str::AbstractString, ctx::SHA_CTX) + # convert to String such that `codeunits(str)` below is consistent between equal + # strings of different types + str = String(str) + SHA.update!(ctx, codeunits(str)) + # signature for strings: so far, all hash_seed functions end-up hashing a multiple + # of 4 bytes of data, and add the signature (1 byte) at the end; so hash as many + # bytes as necessary to have a total number of hashed bytes equal to 0 mod 4 (padding), + # and then hash the signature 0x05; in order for strings of different lengths to have + # different hashes, padding bytes are set equal to the number of padding bytes + pad = 4 - mod(ncodeunits(str), 4) + for _=1:pad + SHA.update!(ctx, (pad % UInt8,)) end - _advance_to!(r, adv, work) - r + SHA.update!(ctx, (0x05,)) end diff --git a/stdlib/Random/src/Random.jl b/stdlib/Random/src/Random.jl index 2d75f49480a7b..9e05475b48c20 100644 --- a/stdlib/Random/src/Random.jl +++ b/stdlib/Random/src/Random.jl @@ -13,9 +13,11 @@ include("DSFMT.jl") using .DSFMT using Base.GMP.MPZ using Base.GMP: Limb -import SHA +using SHA: SHA, SHA2_256_CTX, SHA2_512_CTX, SHA_CTX + +using Base: BitInteger, BitInteger_types, BitUnsigned, require_one_based_indexing, + _throw_argerror -using Base: BitInteger, BitInteger_types, BitUnsigned, require_one_based_indexing import Base: copymutable, copy, copy!, ==, hash, convert, rand, randn, show @@ -165,6 +167,11 @@ Sampler(::Type{<:AbstractRNG}, ::Type{T}, ::Repetition) where {T} = SamplerType{ Base.getindex(::SamplerType{T}) where {T} = T +# SamplerUnion(X, Y, ...}) == Union{SamplerType{X}, SamplerType{Y}, ...} +SamplerUnion(U...) = Union{Any[SamplerType{T} for T in U]...} +const SamplerBoolBitInteger = SamplerUnion(Bool, BitInteger_types...) + + struct SamplerTrivial{T,E} <: Sampler{E} self::T end @@ -293,19 +300,24 @@ rand( ::Type{X}, dims::Dims) where {X} = rand(default_rng(), X, d rand(r::AbstractRNG, ::Type{X}, d::Integer, dims::Integer...) where {X} = rand(r, X, Dims((d, dims...))) rand( ::Type{X}, d::Integer, dims::Integer...) where {X} = rand(X, Dims((d, dims...))) -# SamplerUnion(X, Y, ...}) == Union{SamplerType{X}, SamplerType{Y}, ...} -SamplerUnion(U...) = Union{Any[SamplerType{T} for T in U]...} -const SamplerBoolBitInteger = SamplerUnion(Bool, BitInteger_types...) +### UnsafeView +# internal array-like type to circumvent the lack of flexibility with reinterpret + +struct UnsafeView{T} <: DenseArray{T,1} + ptr::Ptr{T} + len::Int +end + +Base.length(a::UnsafeView) = a.len +Base.getindex(a::UnsafeView, i::Int) = unsafe_load(a.ptr, i) +Base.setindex!(a::UnsafeView, x, i::Int) = unsafe_store!(a.ptr, x, i) +Base.pointer(a::UnsafeView) = a.ptr +Base.size(a::UnsafeView) = (a.len,) +Base.elsize(::Type{UnsafeView{T}}) where {T} = sizeof(T) -include("Xoshiro.jl") -include("RNGs.jl") -include("generation.jl") -include("normal.jl") -include("misc.jl") -include("XoshiroSimd.jl") -## rand & rand! & seed! docstrings +## rand & rand! docstrings """ rand([rng=default_rng()], [S], [dims...]) @@ -405,61 +417,13 @@ julia> rand!(Xoshiro(123), zeros(5)) """ rand! -""" - seed!([rng=default_rng()], seed) -> rng - seed!([rng=default_rng()]) -> rng - -Reseed the random number generator: `rng` will give a reproducible -sequence of numbers if and only if a `seed` is provided. Some RNGs -don't accept a seed, like `RandomDevice`. -After the call to `seed!`, `rng` is equivalent to a newly created -object initialized with the same seed. -The types of accepted seeds depend on the type of `rng`, but in general, -integer seeds should work. -If `rng` is not specified, it defaults to seeding the state of the -shared task-local generator. - -# Examples -```julia-repl -julia> Random.seed!(1234); - -julia> x1 = rand(2) -2-element Vector{Float64}: - 0.32597672886359486 - 0.5490511363155669 - -julia> Random.seed!(1234); - -julia> x2 = rand(2) -2-element Vector{Float64}: - 0.32597672886359486 - 0.5490511363155669 - -julia> x1 == x2 -true - -julia> rng = Xoshiro(1234); rand(rng, 2) == x1 -true - -julia> Xoshiro(1) == Random.seed!(rng, 1) -true - -julia> rand(Random.seed!(rng), Bool) # not reproducible -true - -julia> rand(Random.seed!(rng), Bool) # not reproducible either -false - -julia> rand(Xoshiro(), Bool) # not reproducible either -true -``` -""" -seed!(rng::AbstractRNG) = seed!(rng, nothing) -#= -We have this generic definition instead of the alternative option -`seed!(rng::AbstractRNG, ::Nothing) = seed!(rng)` -because it would lead too easily to ambiguities, e.g. when we define `seed!(::Xoshiro, seed)`. -=# +include("Xoshiro.jl") +include("RNGs.jl") +include("MersenneTwister.jl") +include("generation.jl") +include("normal.jl") +include("misc.jl") +include("XoshiroSimd.jl") end # module diff --git a/stdlib/Random/src/Xoshiro.jl b/stdlib/Random/src/Xoshiro.jl index 94c7e1ab24e1d..7d059261b69f5 100644 --- a/stdlib/Random/src/Xoshiro.jl +++ b/stdlib/Random/src/Xoshiro.jl @@ -195,9 +195,6 @@ task creation, simulation results are also independent of the number of availabl threads / CPUs. The random stream should not depend on hardware specifics, up to endianness and possibly word size. -Using or seeding the RNG of any other task than the one returned by `current_task()` -is undefined behavior: it will work most of the time, and may sometimes fail silently. - When seeding `TaskLocalRNG()` with [`seed!`](@ref), the passed seed, if any, may be any integer. @@ -235,7 +232,7 @@ rng_native_52(::TaskLocalRNG) = UInt64 # this variant of setstate! initializes the internal splitmix state, a.k.a. `s4` @inline function initstate!(x::Union{TaskLocalRNG, Xoshiro}, state) length(state) == 4 && eltype(state) == UInt64 || - throw(ArgumentError("initstate! expects a list of 4 `UInt64` values")) + _throw_argerror("initstate! expects a list of 4 `UInt64` values") s0, s1, s2, s3 = state setstate!(x, (s0, s1, s2, s3, 1s0 + 3s1 + 5s2 + 7s3)) end @@ -246,18 +243,8 @@ copy!(dst::Union{TaskLocalRNG, Xoshiro}, src::Union{TaskLocalRNG, Xoshiro}) = se # use a magic (random) number to scramble `h` so that `hash(x)` is distinct from `hash(getstate(x))` hash(x::Union{TaskLocalRNG, Xoshiro}, h::UInt) = hash(getstate(x), h + 0x49a62c2dda6fa9be % UInt) -function seed!(rng::Union{TaskLocalRNG, Xoshiro}, ::Nothing) - # as we get good randomness from RandomDevice, we can skip hashing - rd = RandomDevice() - s0 = rand(rd, UInt64) - s1 = rand(rd, UInt64) - s2 = rand(rd, UInt64) - s3 = rand(rd, UInt64) - initstate!(rng, (s0, s1, s2, s3)) -end - -seed!(rng::Union{TaskLocalRNG, Xoshiro}, seed) = - initstate!(rng, reinterpret(UInt64, hash_seed(seed))) +seed!(rng::Union{TaskLocalRNG, Xoshiro}, seeder::AbstractRNG) = + initstate!(rng, rand(seeder, NTuple{4, UInt64})) @inline function rand(x::Union{TaskLocalRNG, Xoshiro}, ::SamplerType{UInt64}) diff --git a/stdlib/Random/src/generation.jl b/stdlib/Random/src/generation.jl index b605dff9e5d80..1b09c624193f8 100644 --- a/stdlib/Random/src/generation.jl +++ b/stdlib/Random/src/generation.jl @@ -57,7 +57,7 @@ Sampler(::Type{<:AbstractRNG}, I::FloatInterval{BigFloat}, ::Repetition) = SamplerBigFloat{typeof(I)}(precision(BigFloat)) function _rand!(rng::AbstractRNG, z::BigFloat, sp::SamplerBigFloat) - precision(z) == sp.prec || throw(ArgumentError("incompatible BigFloat precision")) + precision(z) == sp.prec || _throw_argerror("incompatible BigFloat precision") limbs = sp.limbs rand!(rng, limbs) @inbounds begin @@ -229,6 +229,9 @@ uint_sup(::Type{<:Base.BitInteger32}) = UInt32 uint_sup(::Type{<:Union{Int64,UInt64}}) = UInt64 uint_sup(::Type{<:Union{Int128,UInt128}}) = UInt128 +@noinline empty_collection_error() = throw(ArgumentError("collection must be non-empty")) + + #### Fast struct SamplerRangeFast{U<:BitUnsigned,T<:BitInteger} <: Sampler{T} @@ -242,7 +245,7 @@ SamplerRangeFast(r::AbstractUnitRange{T}) where T<:BitInteger = SamplerRangeFast(r, uint_sup(T)) function SamplerRangeFast(r::AbstractUnitRange{T}, ::Type{U}) where {T,U} - isempty(r) && throw(ArgumentError("collection must be non-empty")) + isempty(r) && empty_collection_error() m = (last(r) - first(r)) % unsigned(T) % U # % unsigned(T) to not propagate sign bit bw = (Base.top_set_bit(m)) % UInt # bit-width mask = ((1 % U) << bw) - (1 % U) @@ -316,7 +319,7 @@ SamplerRangeInt(r::AbstractUnitRange{T}) where T<:BitInteger = SamplerRangeInt(r, uint_sup(T)) function SamplerRangeInt(r::AbstractUnitRange{T}, ::Type{U}) where {T,U} - isempty(r) && throw(ArgumentError("collection must be non-empty")) + isempty(r) && empty_collection_error() a = first(r) m = (last(r) - first(r)) % unsigned(T) % U k = m + one(U) @@ -362,7 +365,7 @@ struct SamplerRangeNDL{U<:Unsigned,T} <: Sampler{T} end function SamplerRangeNDL(r::AbstractUnitRange{T}) where {T} - isempty(r) && throw(ArgumentError("collection must be non-empty")) + isempty(r) && empty_collection_error() a = first(r) U = uint_sup(T) s = (last(r) - first(r)) % unsigned(T) % U + one(U) # overflow ok @@ -375,16 +378,20 @@ function rand(rng::AbstractRNG, sp::SamplerRangeNDL{U,T}) where {U,T} s = sp.s x = widen(rand(rng, U)) m = x * s - l = m % U - if l < s - t = mod(-s, s) # as s is unsigned, -s is equal to 2^L - s in the paper - while l < t - x = widen(rand(rng, U)) - m = x * s - l = m % U - end + r::T = (m % U) < s ? rand_unlikely(rng, s, m) % T : + iszero(s) ? x % T : + (m >> (8*sizeof(U))) % T + r + sp.a +end + +# similar to `randn_unlikely` : splitting this unlikely path out results in faster code +@noinline function rand_unlikely(rng, s::U, m)::U where {U} + t = mod(-s, s) # as s is unsigned, -s is equal to 2^L - s in the paper + while (m % U) < t + x = widen(rand(rng, U)) + m = x * s end - (s == 0 ? x : m >> (8*sizeof(U))) % T + sp.a + (m >> (8*sizeof(U))) % U end @@ -401,7 +408,7 @@ end function SamplerBigInt(::Type{RNG}, r::AbstractUnitRange{BigInt}, N::Repetition=Val(Inf) ) where {RNG<:AbstractRNG} m = last(r) - first(r) - m.size < 0 && throw(ArgumentError("collection must be non-empty")) + m.size < 0 && empty_collection_error() nlimbs = Int(m.size) hm = nlimbs == 0 ? Limb(0) : GC.@preserve m unsafe_load(m.d, nlimbs) highsp = Sampler(RNG, Limb(0):hm, N) @@ -457,7 +464,7 @@ rand(rng::AbstractRNG, sp::SamplerSimple{<:AbstractArray,<:Sampler}) = ## random values from Dict function Sampler(::Type{RNG}, t::Dict, ::Repetition) where RNG<:AbstractRNG - isempty(t) && throw(ArgumentError("collection must be non-empty")) + isempty(t) && empty_collection_error() # we use Val(Inf) below as rand is called repeatedly internally # even for generating only one random value from t SamplerSimple(t, Sampler(RNG, LinearIndices(t.slots), Val(Inf))) @@ -486,7 +493,7 @@ rand(rng::AbstractRNG, sp::SamplerTag{<:Set,<:Sampler}) = rand(rng, sp.data).fir ## random values from BitSet function Sampler(RNG::Type{<:AbstractRNG}, t::BitSet, n::Repetition) - isempty(t) && throw(ArgumentError("collection must be non-empty")) + isempty(t) && empty_collection_error() SamplerSimple(t, Sampler(RNG, minimum(t):maximum(t), Val(Inf))) end diff --git a/stdlib/Random/src/misc.jl b/stdlib/Random/src/misc.jl index 908776383d45f..0f113e8153f04 100644 --- a/stdlib/Random/src/misc.jl +++ b/stdlib/Random/src/misc.jl @@ -97,7 +97,7 @@ end # size-m subset of A where m is fixed!) function randsubseq!(r::AbstractRNG, S::AbstractArray, A::AbstractArray, p::Real) require_one_based_indexing(S, A) - 0 <= p <= 1 || throw(ArgumentError("probability $p not in [0,1]")) + 0 <= p <= 1 || _throw_argerror(LazyString("probability ", p, " not in [0,1]")) n = length(A) p == 1 && return copyto!(resize!(S, n), A) empty!(S) @@ -297,13 +297,16 @@ randperm(r::AbstractRNG, n::T) where {T <: Integer} = randperm!(r, Vector{T}(und randperm(n::Integer) = randperm(default_rng(), n) """ - randperm!([rng=default_rng(),] A::Array{<:Integer}) + randperm!([rng=default_rng(),] A::AbstractArray{<:Integer}) Construct in `A` a random permutation of length `length(A)`. The optional `rng` argument specifies a random number generator (see [Random Numbers](@ref)). To randomly permute an arbitrary vector, see [`shuffle`](@ref) or [`shuffle!`](@ref). +!!! compat "Julia 1.13" + `A isa Array` was required prior to Julia v1.13. + # Examples ```jldoctest julia> randperm!(Xoshiro(123), Vector{Int}(undef, 4)) @@ -314,8 +317,9 @@ julia> randperm!(Xoshiro(123), Vector{Int}(undef, 4)) 3 ``` """ -function randperm!(r::AbstractRNG, a::Array{<:Integer}) +function randperm!(r::AbstractRNG, a::AbstractArray{<:Integer}) # keep it consistent with `shuffle!` and `randcycle!` if possible + Base.require_one_based_indexing(a) n = length(a) @assert n <= Int64(2)^52 n == 0 && return a @@ -332,7 +336,7 @@ function randperm!(r::AbstractRNG, a::Array{<:Integer}) return a end -randperm!(a::Array{<:Integer}) = randperm!(default_rng(), a) +randperm!(a::AbstractArray{<:Integer}) = randperm!(default_rng(), a) ## randcycle & randcycle! @@ -370,7 +374,7 @@ randcycle(r::AbstractRNG, n::T) where {T <: Integer} = randcycle!(r, Vector{T}(u randcycle(n::Integer) = randcycle(default_rng(), n) """ - randcycle!([rng=default_rng(),] A::Array{<:Integer}) + randcycle!([rng=default_rng(),] A::AbstractArray{<:Integer}) Construct in `A` a random cyclic permutation of length `n = length(A)`. The optional `rng` argument specifies a random number generator, see @@ -382,6 +386,9 @@ which are sampled uniformly. If `A` is empty, `randcycle!` leaves it unchanged. [`randcycle`](@ref) is a variant of this function that allocates a new vector. +!!! compat "Julia 1.13" + `A isa Array` was required prior to Julia v1.13. + # Examples ```jldoctest julia> randcycle!(Xoshiro(123), Vector{Int}(undef, 6)) @@ -394,8 +401,9 @@ julia> randcycle!(Xoshiro(123), Vector{Int}(undef, 6)) 1 ``` """ -function randcycle!(r::AbstractRNG, a::Array{<:Integer}) +function randcycle!(r::AbstractRNG, a::AbstractArray{<:Integer}) # keep it consistent with `shuffle!` and `randperm!` if possible + Base.require_one_based_indexing(a) n = length(a) @assert n <= Int64(2)^52 n == 0 && return a @@ -411,4 +419,4 @@ function randcycle!(r::AbstractRNG, a::Array{<:Integer}) return a end -randcycle!(a::Array{<:Integer}) = randcycle!(default_rng(), a) +randcycle!(a::AbstractArray{<:Integer}) = randcycle!(default_rng(), a) diff --git a/stdlib/Random/test/runtests.jl b/stdlib/Random/test/runtests.jl index a28197bdfe833..e929cd2d8a517 100644 --- a/stdlib/Random/test/runtests.jl +++ b/stdlib/Random/test/runtests.jl @@ -11,8 +11,9 @@ using Random using Random.DSFMT using Random: default_rng, Sampler, SamplerRangeFast, SamplerRangeInt, SamplerRangeNDL, MT_CACHE_F, MT_CACHE_I -using Random: jump_128, jump_192, jump_128!, jump_192! +using Random: jump_128, jump_192, jump_128!, jump_192!, SeedHasher +import SHA import Future # randjump function test_uniform(xs::AbstractArray{T}) where {T<:AbstractFloat} @@ -297,7 +298,7 @@ for f in (:<, :<=, :>, :>=, :(==), :(!=)) end # test all rand APIs -for rng in ([], [MersenneTwister(0)], [RandomDevice()], [Xoshiro()]) +for rng in ([], [MersenneTwister(0)], [RandomDevice()], [Xoshiro(0)], [SeedHasher(0)]) realrng = rng == [] ? default_rng() : only(rng) ftypes = [Float16, Float32, Float64, FakeFloat64, BigFloat] cftypes = [ComplexF16, ComplexF32, ComplexF64, ftypes...] @@ -453,7 +454,8 @@ function hist(X, n) end @testset "uniform distribution of floats" begin - for rng in [MersenneTwister(), RandomDevice(), Xoshiro()], + seed = rand(UInt128) + for rng in [MersenneTwister(seed), RandomDevice(), Xoshiro(seed), SeedHasher(seed)], T in [Float16, Float32, Float64, BigFloat], prec in (T == BigFloat ? [3, 53, 64, 100, 256, 1000] : [256]) @@ -480,7 +482,8 @@ end # but also for 3 linear combinations of positions (for the array version) lcs = unique!.([rand(1:n, 2), rand(1:n, 3), rand(1:n, 5)]) aslcs = zeros(Int, 3) - for rng = (MersenneTwister(), RandomDevice(), Xoshiro()) + seed = rand(UInt128) + for rng = (MersenneTwister(seed), RandomDevice(), Xoshiro(seed), SeedHasher(seed)) for scalar = [false, true] fill!(a, 0) fill!(as, 0) @@ -551,6 +554,14 @@ end @test randcycle!(mta, A) == randcycle!(mtb, B) @test randcycle!(A) === A + @testset "non-`Array` `randperm!` and `randcycle!`" begin + x, y = Memory{Int}(undef, 10), Memory{Int}(undef, 10) + @test randperm!(mta, x) == randperm!(mtb, y) + @test randperm!(x) === x + @test randcycle!(mta, x) == randcycle!(mtb, y) + @test randcycle!(x) === x + end + let p = randcycle(UInt16(10)) @test typeof(p) ≡ Vector{UInt16} @test sort!(p) == 1:10 @@ -662,7 +673,7 @@ end @testset "Random.seed!(rng, ...) returns rng" begin # issue #21248 seed = rand(UInt) - for m = ([MersenneTwister(seed)], [Xoshiro(seed)], []) + for m = ([MersenneTwister(seed)], [Xoshiro(seed)], [SeedHasher(seed)], []) m2 = m == [] ? default_rng() : m[1] @test Random.seed!(m...) === m2 @test Random.seed!(m..., rand(UInt)) === m2 @@ -675,6 +686,7 @@ end @test Random.seed!(m..., typemax(UInt)) === m2 @test Random.seed!(m..., typemax(UInt128)) === m2 @test Random.seed!(m..., "a random seed") === m2 + @test Random.seed!(m..., Random.default_rng()) === m2 end end @@ -707,7 +719,7 @@ end # this shouldn't crash (#22403) @test_throws MethodError rand!(Union{UInt,Int}[1, 2, 3]) -@testset "$RNG() & Random.seed!(rng::$RNG) initializes randomly" for RNG in (MersenneTwister, RandomDevice, Xoshiro) +@testset "$RNG() & Random.seed!(rng::$RNG) initializes randomly" for RNG in (MersenneTwister, RandomDevice, Xoshiro, SeedHasher) m = RNG() a = rand(m, Int) m = RNG() @@ -728,7 +740,7 @@ end @test rand(m, Int) ∉ (a, b, c, d) end -@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister,Xoshiro) +@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister, Xoshiro, SeedHasher) seeds = Any[0, 1, 2, 10000, 10001, rand(UInt32, 8), randstring(), randstring(), rand(UInt128, 3)...] if RNG == Xoshiro push!(seeds, rand(UInt64, rand(1:4))) @@ -739,6 +751,11 @@ end Random.seed!(m, seed) @test a == [rand(m) for _=1:100] end + # rng as a seed + m = RNG(Xoshiro(0)) + a = [rand(m) for _=1:100] + Random.seed!(m, Xoshiro(0)) + @test a == [rand(m) for _=1:100] end @testset "Random.seed!(seed) sets Random.GLOBAL_SEED" begin @@ -763,7 +780,10 @@ struct RandomStruct23964 end @test_throws MethodError rand(RandomStruct23964()) end -@testset "rand(::$(typeof(RNG)), ::UnitRange{$T}" for RNG ∈ (MersenneTwister(rand(UInt128)), RandomDevice(), Xoshiro()), +@testset "rand(::$(typeof(RNG)), ::UnitRange{$T}" for RNG ∈ (MersenneTwister(rand(UInt128)), + RandomDevice(), + Xoshiro(rand(UInt128)), + SeedHasher(rand(UInt128))), T ∈ (Bool, Int8, Int16, Int32, UInt32, Int64, Int128, UInt128) if T === Bool @test rand(RNG, false:true) ∈ (false, true) @@ -824,6 +844,20 @@ end @inferred rand(Tuple{Int32,Int64,Float64}) @inferred rand(NTuple{20,Int}) @test_throws TypeError rand(Tuple{1:2,3:4}) + + @testset "rand(::RandomDevice, ::Type{NTuple{N, Int}})" begin + # RandomDevice has a specialization for homogeneous tuple types of builtin integers + rd = RandomDevice() + @test () == rand(rd, Tuple{}) + xs = rand(rd, Tuple{Int, Int}) + @test xs isa Tuple{Int, Int} && xs[1] != xs[2] + xs = rand(rd, NTuple{2, Int}) + @test xs isa Tuple{Int, Int} && xs[1] != xs[2] + xs = rand(rd, Tuple{Int, UInt}) # not NTuple + @test xs isa Tuple{Int, UInt} && xs[1] != xs[2] + xs = rand(rd, Tuple{Bool}) # not included in the specialization + @test xs isa Tuple{Bool} + end end @testset "GLOBAL_RNG" begin @@ -892,8 +926,11 @@ end @test rand(rng) == rand(GLOBAL_RNG) end -@testset "RNGs broadcast as scalars: T" for T in (MersenneTwister, RandomDevice) - @test length.(rand.(T(), 1:3)) == 1:3 +@testset "RNGs broadcast as scalars: $(typeof(RNG))" for RNG in (MersenneTwister(0), + RandomDevice(), + Xoshiro(0), + SeedHasher(0)) + @test length.(rand.(RNG, 1:3)) == 1:3 end @testset "generated scalar integers do not overlap" begin @@ -1191,7 +1228,14 @@ end end end + @testset "seed! and hash_seed" begin + function hash_seed(seed) + ctx = SHA.SHA2_256_CTX() + Random.hash_seed(seed, ctx) + bytes2hex(SHA.digest!(ctx)) + end + # Test that: # 1) if n == m, then hash_seed(n) == hash_seed(m) # 2) if n != m, then hash_seed(n) != hash_seed(m) @@ -1204,12 +1248,12 @@ end T <: Signed && push!(seeds, T(0), T(1), T(2), T(-1), T(-2)) end - vseeds = Dict{Vector{UInt8}, BigInt}() + vseeds = Dict{String, BigInt}() for seed = seeds bigseed = big(seed) - vseed = Random.hash_seed(bigseed) + vseed = hash_seed(bigseed) # test property 1) above - @test Random.hash_seed(seed) == vseed + @test hash_seed(seed) == vseed # test property 2) above @test bigseed == get!(vseeds, vseed, bigseed) # test that the property 1) is actually inherited by `seed!` @@ -1221,16 +1265,16 @@ end end seed32 = rand(UInt32, rand(1:9)) - hash32 = Random.hash_seed(seed32) - @test Random.hash_seed(map(UInt64, seed32)) == hash32 + hash32 = hash_seed(seed32) + @test hash_seed(map(UInt64, seed32)) == hash32 @test hash32 ∉ keys(vseeds) seed_str = randstring() seed_gstr = GenericString(seed_str) - @test Random.hash_seed(seed_str) == Random.hash_seed(seed_gstr) - string_seeds = Set{Vector{UInt8}}() + @test hash_seed(seed_str) == hash_seed(seed_gstr) + string_seeds = Set{String}() for ch = 'A':'z' - vseed = Random.hash_seed(string(ch)) + vseed = hash_seed(string(ch)) @test vseed ∉ keys(vseeds) @test vseed ∉ string_seeds push!(string_seeds, vseed) diff --git a/stdlib/SHA.version b/stdlib/SHA.version index a5d4372d5798b..507ab1e368f79 100644 --- a/stdlib/SHA.version +++ b/stdlib/SHA.version @@ -1,4 +1,4 @@ SHA_BRANCH = master -SHA_SHA1 = 4451e1362e425bcbc1652ecf55fc0e525b18fb63 +SHA_SHA1 = 169a3369026ee767c454e5b1ca70c62c4db5a933 SHA_GIT_URL := https://github.com/JuliaCrypto/SHA.jl.git SHA_TAR_URL = https://api.github.com/repos/JuliaCrypto/SHA.jl/tarball/$1 diff --git a/stdlib/Serialization/docs/src/index.md b/stdlib/Serialization/docs/src/index.md index 0d00e47ed84ce..77c7558e0306a 100644 --- a/stdlib/Serialization/docs/src/index.md +++ b/stdlib/Serialization/docs/src/index.md @@ -11,3 +11,15 @@ Serialization.serialize Serialization.deserialize Serialization.writeheader ``` + +### Recommended File Extension + +While the Serialization module does not mandate a specific file extension, the Julia community commonly uses the `.jls` extension for serialized Julia files. + +Example: + +```julia +open("model.jls", "w") do io + serialize(io, my_model) +end +``` diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index 3362c9439d385..ee40ebdd4abad 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -8,6 +8,7 @@ Provide serialization of Julia objects via the functions module Serialization import Base: Bottom, unsafe_convert +import Base.ScopedValues: ScopedValue, with import Core: svec, SimpleVector using Base: unaliascopy, unwrap_unionall, require_one_based_indexing, ntupleany using Core.IR @@ -28,6 +29,8 @@ end Serializer(io::IO) = Serializer{typeof(io)}(io) +const current_module = ScopedValue{Union{Nothing,Module}}(nothing) + ## serializing values ## const n_int_literals = 33 @@ -467,6 +470,18 @@ function serialize(s::AbstractSerializer, meth::Method) nothing end +function serialize(s::AbstractSerializer, mt::Core.MethodTable) + serialize_type(s, typeof(mt)) + serialize(s, mt.name) + serialize(s, mt.module) + nothing +end + +function serialize(s::AbstractSerializer, mc::Core.MethodCache) + error("cannot serialize MethodCache objects") +end + + function serialize(s::AbstractSerializer, linfo::Core.MethodInstance) serialize_cycle(s, linfo) && return writetag(s.io, METHODINSTANCE_TAG) @@ -536,11 +551,12 @@ function serialize_typename(s::AbstractSerializer, t::Core.TypeName) serialize(s, t.flags & 0x2 == 0x2) # .mutable serialize(s, Int32(length(primary.types) - t.n_uninitialized)) serialize(s, t.max_methods) - if isdefined(t, :mt) && t.mt !== Symbol.name.mt - serialize(s, t.mt.name) - serialize(s, collect(Base.MethodList(t.mt))) - serialize(s, t.mt.max_args) - kws = collect(methods(Core.kwcall, (Any, t.wrapper, Vararg))) + ms = Base.matches_to_methods(Base._methods_by_ftype(Tuple{t.wrapper, Vararg}, -1, Base.get_world_counter()), t, nothing).ms + if t.singletonname !== t.name || !isempty(ms) + serialize(s, t.singletonname) + serialize(s, ms) + serialize(s, t.max_args) + kws = Base.matches_to_methods(Base._methods_by_ftype(Tuple{typeof(Core.kwcall), Any, t.wrapper, Vararg}, -1, Base.get_world_counter()), t, nothing).ms if isempty(kws) writetag(s.io, UNDEFREF_TAG) else @@ -555,21 +571,17 @@ end # decide whether to send all data for a type (instead of just its name) function should_send_whole_type(s, t::DataType) tn = t.name - if isdefined(tn, :mt) - # TODO improve somehow - # send whole type for anonymous functions in Main - name = tn.mt.name - mod = tn.module - isanonfunction = mod === Main && # only Main - t.super === Function && # only Functions - unsafe_load(unsafe_convert(Ptr{UInt8}, tn.name)) == UInt8('#') && # hidden type - (!isdefined(mod, name) || t != typeof(getglobal(mod, name))) # XXX: 95% accurate test for this being an inner function - # TODO: more accurate test? (tn.name !== "#" name) - #TODO: iskw = startswith(tn.name, "#kw#") && ??? - #TODO: iskw && return send-as-kwftype - return mod === __deserialized_types__ || isanonfunction - end - return false + # TODO improve somehow? + # send whole type for anonymous functions in Main + name = tn.singletonname + mod = tn.module + mod === __deserialized_types__ && return true + isanonfunction = mod === Main && # only Main + t.super === Function && # only Functions + unsafe_load(unsafe_convert(Ptr{UInt8}, tn.name)) == UInt8('#') && # hidden type + (!isdefined(mod, name) || t != typeof(getglobal(mod, name))) # XXX: 95% accurate test for this being an inner function + # TODO: more accurate test? (tn.name !== "#" name) + return isanonfunction end function serialize_type_data(s, @nospecialize(t::DataType)) @@ -1055,7 +1067,10 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) nospecializeinfer = false constprop = 0x00 purity = 0x0000 - template_or_is_opaque = deserialize(s) + local template_or_is_opaque, template + with(current_module => mod) do + template_or_is_opaque = deserialize(s) + end if isa(template_or_is_opaque, Bool) is_for_opaque_closure = template_or_is_opaque if format_version(s) >= 24 @@ -1069,7 +1084,9 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) elseif format_version(s) >= 17 purity = UInt16(deserialize(s)::UInt8) end - template = deserialize(s) + with(current_module => mod) do + template = deserialize(s) + end else template = template_or_is_opaque end @@ -1112,8 +1129,8 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) meth.recursion_relation = recursion_relation end if !is_for_opaque_closure - mt = ccall(:jl_method_table_for, Any, (Any,), sig) - if mt !== nothing && nothing === ccall(:jl_methtable_lookup, Any, (Any, Any, UInt), mt, sig, Base.get_world_counter()) + mt = Core.methodtable + if nothing === ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, Base.get_world_counter()) # XXX: quite sketchy? ccall(:jl_method_table_insert, Cvoid, (Any, Any, Ptr{Cvoid}), mt, meth, C_NULL) end end @@ -1122,6 +1139,12 @@ function deserialize(s::AbstractSerializer, ::Type{Method}) return meth end +function deserialize(s::AbstractSerializer, ::Type{Core.MethodTable}) + name = deserialize(s)::Symbol + mod = deserialize(s)::Module + return getglobal(mod, name)::Core.MethodTable +end + function deserialize(s::AbstractSerializer, ::Type{Core.MethodInstance}) linfo = ccall(:jl_new_method_instance_uninit, Ref{Core.MethodInstance}, (Ptr{Cvoid},), C_NULL) deserialize_cycle(s, linfo) @@ -1167,6 +1190,22 @@ function deserialize(s::AbstractSerializer, ::Type{PhiNode}) return PhiNode(edges, values) end +# v1.12 disallows bare symbols in IR, but older CodeInfos might still have them +function symbol_to_globalref(@nospecialize(x), m::Module) + mapper(@nospecialize(x)) = symbol_to_globalref(x, m) + if x isa Symbol + return GlobalRef(m, x) + elseif x isa Expr + return Expr(x.head, map(mapper, x.args)...) + elseif x isa ReturnNode + return ReturnNode(mapper(x.val)) + elseif x isa GotoIfNot + return GotoIfNot(mapper(x.cond), x.dest) + else + return x + end +end + function deserialize(s::AbstractSerializer, ::Type{CodeInfo}) ci = ccall(:jl_new_code_info_uninit, Ref{CodeInfo}, ()) deserialize_cycle(s, ci) @@ -1185,6 +1224,9 @@ function deserialize(s::AbstractSerializer, ::Type{CodeInfo}) end end end + if current_module[] !== nothing + map!(x->symbol_to_globalref(x, current_module[]), code) + end _x = deserialize(s) have_debuginfo = _x isa Core.DebugInfo if have_debuginfo @@ -1233,6 +1275,9 @@ function deserialize(s::AbstractSerializer, ::Type{CodeInfo}) ci.slottypes = deserialize(s) ci.rettype = deserialize(s) ci.parent = deserialize(s) + if format_version(s) < 29 && ci.parent isa MethodInstance && ci.parent.def isa Method + ci.nargs = ci.parent.def.nargs + end world_or_edges = deserialize(s) pre_13 = isa(world_or_edges, Union{UInt, Int}) if pre_13 @@ -1243,7 +1288,7 @@ function deserialize(s::AbstractSerializer, ::Type{CodeInfo}) ci.min_world = deserialize(s)::UInt ci.max_world = deserialize(s)::UInt end - if format_version(s) >= 26 + if format_version(s) >= 29 ci.method_for_inference_limit_heuristics = deserialize(s) end end @@ -1471,20 +1516,10 @@ function deserialize_typename(s::AbstractSerializer, number) if tag != UNDEFREF_TAG mtname = handle_deserialize(s, tag) defs = deserialize(s) - maxa = deserialize(s)::Int + maxa = deserialize(s)::Union{Int,Int32} if makenew - mt = ccall(:jl_new_method_table, Any, (Any, Any), name, tn.module) - if !isempty(parameters) - mt.offs = 0 - end - mt.name = mtname - setfield!(mt, :max_args, maxa, :monotonic) - ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), tn, Base.fieldindex(Core.TypeName, :mt)-1, mt) - for def in defs - if isdefined(def, :sig) - ccall(:jl_method_table_insert, Cvoid, (Any, Any, Ptr{Cvoid}), mt, def, C_NULL) - end - end + tn.singletonname = mtname + setfield!(tn, :max_args, Int32(maxa), :monotonic) end tag = Int32(read(s.io, UInt8)::UInt8) if tag != UNDEFREF_TAG @@ -1494,9 +1529,6 @@ function deserialize_typename(s::AbstractSerializer, number) @eval Core.kwcall(kwargs::NamedTuple, f::$ty, args...) = $kws(kwargs, f, args...) end end - elseif makenew - mt = Symbol.name.mt - ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), tn, Base.fieldindex(Core.TypeName, :mt)-1, mt) end return tn end diff --git a/stdlib/Serialization/test/runtests.jl b/stdlib/Serialization/test/runtests.jl index f1b83ca947c7e..e341c6e3eb9ec 100644 --- a/stdlib/Serialization/test/runtests.jl +++ b/stdlib/Serialization/test/runtests.jl @@ -1,6 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test, Random, Serialization, Base64 +using Base.ScopedValues: with # Check that serializer hasn't gone out-of-frame @test Serialization.sertag(Symbol) == 1 @@ -661,3 +662,14 @@ end @test_broken isempty(undoc) @test undoc == [:AbstractSerializer, :Serializer] end + +# test method definitions from v1.11 +if Int === Int64 + let f_data = "N0pMGgQAAAAWAQEFdGh1bmsbFUbnFgEBBXRodW5rGxVG4DoWAQEGbWV0aG9kAQtmMTExX3RvXzExMhUABuABAAAA4BUAB+AAAAAAThVG4DQQAQxMaW5lSW5mb05vZGUfTptEH04BBE1haW5EAQ90b3AtbGV2ZWwgc2NvcGUBBG5vbmW+vhUAAd8V305GTk4JAQAAAAAAAAAJ//////////9MTExMAwADAAUAAAX//xYBAQZtZXRob2QsBwAWAlYkH06bRAEGVHlwZW9mLAcAFgNWJB9Om0QBBHN2ZWMo4iQfTptETxYBViQfTptEAQRzdmVjFgRWJB9Om0QBBHN2ZWMo4yjkGhfgAQRub25lFgMBBm1ldGhvZCwHACjlGxVG5AEBXhYDViQfTptElyQfTp5EAQNWYWzhFgFWKOEWBFYkH06eRAELbGl0ZXJhbF9wb3co4CXhKOI6KOMVAAbkAQAAAAEAAAABAAAAAQAAAAAAAADkFQAH5AAAAAAAAAAAAAAAAAAAAAAAAAAAThVG4DQsCwAfTgEETWFpbkQBBG5vbmUBBG5vbmW/vhUAAeGifRXhAAhORk5OCQEAAAAAAAAACf//////////TExMTAMAAwAFAAAF//86ThUABucBAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAAAAAOcVAAfnAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABOFUbgNCwLAB9OAQRNYWluRCwNAAEEbm9uZb6+FQAB3xXfTkZOTgkBAAAAAAAAAAn//////////0xMTEwDAAMABQAABf//" + @eval Main function f111_to_112 end + Core.eval(Main, with(Serialization.current_module => Main) do + deserialize(IOBuffer(base64decode(f_data))) + end) + @test @invokelatest(Main.f111_to_112(16)) == 256 + end +end diff --git a/stdlib/Sockets/src/Sockets.jl b/stdlib/Sockets/src/Sockets.jl index f9e0f2f88dd78..58438b152d825 100644 --- a/stdlib/Sockets/src/Sockets.jl +++ b/stdlib/Sockets/src/Sockets.jl @@ -220,7 +220,7 @@ end # Disables dual stack mode. const UV_TCP_IPV6ONLY = 1 -# Disables dual stack mode. Only available when using ipv6 binf +# Disables dual stack mode. Only available when using ipv6 bind const UV_UDP_IPV6ONLY = 1 # Indicates message was truncated because read buffer was too small. The @@ -780,7 +780,7 @@ end """ leave_multicast_group(sock::UDPSocket, group_addr, interface_addr = nothing) -Remove a socket from a particular multicast group defined by `group_addr`. +Remove a socket from a particular multicast group defined by `group_addr`. If `interface_addr` is given, specifies a particular interface for multi-homed systems. Use `join_multicast_group()` to enable reception of a group. """ diff --git a/stdlib/Sockets/src/addrinfo.jl b/stdlib/Sockets/src/addrinfo.jl index db03cdb7c1dc8..8e12a41f08e3b 100644 --- a/stdlib/Sockets/src/addrinfo.jl +++ b/stdlib/Sockets/src/addrinfo.jl @@ -129,7 +129,7 @@ Uses the operating system's underlying getaddrinfo implementation, which may do a DNS lookup. # Examples -```jldoctest +```jldoctest; filter = r"(ip\\"::1\\"|ERROR: DNSError:.*|Stacktrace:(\\n \\[0-9]+\\].*)*)" julia> getaddrinfo("localhost", IPv6) ip"::1" diff --git a/stdlib/Sockets/test/runtests.jl b/stdlib/Sockets/test/runtests.jl index 26f95d4ce1819..5a822315ba2cd 100644 --- a/stdlib/Sockets/test/runtests.jl +++ b/stdlib/Sockets/test/runtests.jl @@ -547,14 +547,12 @@ end fetch(r) end - let addr = Sockets.InetAddr(ip"127.0.0.1", 4444) - srv = listen(addr) + let addr = Sockets.InetAddr(ip"192.0.2.5", 4444) s = Sockets.TCPSocket() Sockets.connect!(s, addr) r = @async close(s) @test_throws Base._UVError("connect", Base.UV_ECANCELED) Sockets.wait_connected(s) fetch(r) - close(srv) end end diff --git a/stdlib/SparseArrays.version b/stdlib/SparseArrays.version index 9498fbe4943f9..78026afda0489 100644 --- a/stdlib/SparseArrays.version +++ b/stdlib/SparseArrays.version @@ -1,4 +1,4 @@ SPARSEARRAYS_BRANCH = main -SPARSEARRAYS_SHA1 = f3610c07fe0403792743d9c9802a25642a5f2d18 +SPARSEARRAYS_SHA1 = 30201abcb41e558a4b5cc23b11dc5676c5655c0b SPARSEARRAYS_GIT_URL := https://github.com/JuliaSparse/SparseArrays.jl.git SPARSEARRAYS_TAR_URL = https://api.github.com/repos/JuliaSparse/SparseArrays.jl/tarball/$1 diff --git a/stdlib/Statistics.version b/stdlib/Statistics.version index e6f0a62b3cec5..e22fa135f74cd 100644 --- a/stdlib/Statistics.version +++ b/stdlib/Statistics.version @@ -1,4 +1,4 @@ STATISTICS_BRANCH = master -STATISTICS_SHA1 = 77bd5707f143eb624721a7df28ddef470e70ecef +STATISTICS_SHA1 = 22dee82f9824d6045e87aa4b97e1d64fe6f01d8d STATISTICS_GIT_URL := https://github.com/JuliaStats/Statistics.jl.git STATISTICS_TAR_URL = https://api.github.com/repos/JuliaStats/Statistics.jl/tarball/$1 diff --git a/stdlib/StyledStrings.version b/stdlib/StyledStrings.version index c72f7a8399725..8600b0875b045 100644 --- a/stdlib/StyledStrings.version +++ b/stdlib/StyledStrings.version @@ -1,4 +1,4 @@ STYLEDSTRINGS_BRANCH = main -STYLEDSTRINGS_SHA1 = 8985a37ac054c37d084a03ad2837208244824877 +STYLEDSTRINGS_SHA1 = 1aafc2f3abb6a977ee36a87206f9ce6446a8ae86 STYLEDSTRINGS_GIT_URL := https://github.com/JuliaLang/StyledStrings.jl.git STYLEDSTRINGS_TAR_URL = https://api.github.com/repos/JuliaLang/StyledStrings.jl/tarball/$1 diff --git a/stdlib/SuiteSparse_jll/Project.toml b/stdlib/SuiteSparse_jll/Project.toml index cfe46b3fd9569..ff454476db148 100644 --- a/stdlib/SuiteSparse_jll/Project.toml +++ b/stdlib/SuiteSparse_jll/Project.toml @@ -3,11 +3,13 @@ uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" version = "7.10.1+0" [deps] -libblastrampoline_jll = "8e850b90-86db-534c-a0d3-1478176c7d93" -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +libblastrampoline_jll = "8e850b90-86db-534c-a0d3-1478176c7d93" [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.13" [extras] diff --git a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl index b0699f132eb7f..1dcb2d24e4bc7 100644 --- a/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl +++ b/stdlib/SuiteSparse_jll/src/SuiteSparse_jll.jl @@ -2,139 +2,276 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/SuiteSparse_jll.jl baremodule SuiteSparse_jll -using Base, Libdl, libblastrampoline_jll - -const PATH_list = String[] -const LIBPATH_list = String[] +using Base, Libdl +using libblastrampoline_jll +if !(Sys.isfreebsd() || Sys.isapple()) + using CompilerSupportLibraries_jll +end export libamd, libbtf, libcamd, libccolamd, libcholmod, libcolamd, libklu, libldl, librbio, libspqr, libsuitesparseconfig, libumfpack # These get calculated in __init__() # Man I can't wait until these are automatically handled by an in-Base JLLWrappers clone. const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libamd_handle::Ptr{Cvoid} = C_NULL -libamd_path::String = "" -libbtf_handle::Ptr{Cvoid} = C_NULL + +libsuitesparseconfig_path::String = "" +const libsuitesparseconfig = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libsuitesparseconfig.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libsuitesparseconfig.7.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libsuitesparseconfig.so.7") + else + error("SuiteSparse_jll: Library 'libsuitesparseconfig' is not available for $(Sys.KERNEL)") + end +) + +libldl_path::String = "" +const libldl = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libldl.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libldl.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libldl.so.3") + else + error("SuiteSparse_jll: Library 'libldl' is not available for $(Sys.KERNEL)") + end +) + libbtf_path::String = "" -libcamd_handle::Ptr{Cvoid} = C_NULL +const libbtf = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libbtf.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libbtf.2.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libbtf.so.2") + else + error("SuiteSparse_jll: Library 'libbtf' is not available for $(Sys.KERNEL)") + end +) + +libcolamd_path::String = "" +const libcolamd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libcolamd.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libcolamd.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcolamd.so.3") + else + error("SuiteSparse_jll: Library 'libcolamd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) + +libamd_path::String = "" +const libamd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libamd.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libamd.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libamd.so.3") + else + error("SuiteSparse_jll: Library 'libamd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) + libcamd_path::String = "" -libccolamd_handle::Ptr{Cvoid} = C_NULL +const libcamd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libcamd.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libcamd.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcamd.so.3") + else + error("SuiteSparse_jll: Library 'libcamd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) + libccolamd_path::String = "" -libcholmod_handle::Ptr{Cvoid} = C_NULL +const libccolamd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libccolamd.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libccolamd.3.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libccolamd.so.3") + else + error("SuiteSparse_jll: Library 'libccolamd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) + +librbio_path::String = "" +const librbio = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("librbio.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("librbio.4.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("librbio.so.4") + else + error("SuiteSparse_jll: Library 'librbio' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libgcc_s] + else + LazyLibrary[libsuitesparseconfig] + end +) + libcholmod_path::String = "" -libcolamd_handle::Ptr{Cvoid} = C_NULL -libcolamd_path::String = "" -libklu_handle::Ptr{Cvoid} = C_NULL +const libcholmod = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libcholmod.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libcholmod.5.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libcholmod.so.5") + else + error("SuiteSparse_jll: Library 'libcholmod' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + LazyLibrary[ + libsuitesparseconfig, libamd, libcamd, libccolamd, libcolamd, libblastrampoline, libgcc_s + ] + else + LazyLibrary[ + libsuitesparseconfig, libamd, libcamd, libccolamd, libcolamd, libblastrampoline + ] + end +) + libklu_path::String = "" -libldl_handle::Ptr{Cvoid} = C_NULL -libldl_path::String = "" -librbio_handle::Ptr{Cvoid} = C_NULL -librbio_path::String = "" -libspqr_handle::Ptr{Cvoid} = C_NULL +const libklu = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libklu.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libklu.2.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libklu.so.2") + else + error("SuiteSparse_jll: Library 'libklu' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libamd, libcolamd, libbtf, libgcc_s] + else + LazyLibrary[libsuitesparseconfig, libamd, libcolamd, libbtf] + end +) + libspqr_path::String = "" -libsuitesparseconfig_handle::Ptr{Cvoid} = C_NULL -libsuitesparseconfig_path::String = "" -libumfpack_handle::Ptr{Cvoid} = C_NULL +const libspqr = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libspqr.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libspqr.4.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libspqr.so.4") + else + error("SuiteSparse_jll: Library 'libspqr' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() + LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline, libgcc_s] + elseif Sys.isfreebsd() || Sys.isapple() + LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline] + else + LazyLibrary[libsuitesparseconfig, libcholmod, libblastrampoline, libstdcxx, libgcc_s] + end +) + libumfpack_path::String = "" +const libumfpack = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libumfpack.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libumfpack.6.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libumfpack.so.6") + else + error("SuiteSparse_jll: Library 'libumfpack' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libsuitesparseconfig, libamd, libcholmod, libblastrampoline, libgcc_s] + else + LazyLibrary[libsuitesparseconfig, libamd, libcholmod, libblastrampoline] + end +) + +function eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + libblastrampoline_jll.eager_mode() -if Sys.iswindows() - const libamd = "libamd.dll" - const libbtf = "libbtf.dll" - const libcamd = "libcamd.dll" - const libccolamd = "libccolamd.dll" - const libcholmod = "libcholmod.dll" - const libcolamd = "libcolamd.dll" - const libklu = "libklu.dll" - const libldl = "libldl.dll" - const librbio = "librbio.dll" - const libspqr = "libspqr.dll" - const libsuitesparseconfig = "libsuitesparseconfig.dll" - const libumfpack = "libumfpack.dll" -elseif Sys.isapple() - const libamd = "@rpath/libamd.3.dylib" - const libbtf = "@rpath/libbtf.2.dylib" - const libcamd = "@rpath/libcamd.3.dylib" - const libccolamd = "@rpath/libccolamd.3.dylib" - const libcholmod = "@rpath/libcholmod.5.dylib" - const libcolamd = "@rpath/libcolamd.3.dylib" - const libklu = "@rpath/libklu.2.dylib" - const libldl = "@rpath/libldl.3.dylib" - const librbio = "@rpath/librbio.4.dylib" - const libspqr = "@rpath/libspqr.4.dylib" - const libsuitesparseconfig = "@rpath/libsuitesparseconfig.7.dylib" - const libumfpack = "@rpath/libumfpack.6.dylib" -else - const libamd = "libamd.so.3" - const libbtf = "libbtf.so.2" - const libcamd = "libcamd.so.3" - const libccolamd = "libccolamd.so.3" - const libcholmod = "libcholmod.so.5" - const libcolamd = "libcolamd.so.3" - const libklu = "libklu.so.2" - const libldl = "libldl.so.3" - const librbio = "librbio.so.4" - const libspqr = "libspqr.so.4" - const libsuitesparseconfig = "libsuitesparseconfig.so.7" - const libumfpack = "libumfpack.so.6" + dlopen(libamd) + dlopen(libbtf) + dlopen(libcamd) + dlopen(libccolamd) + dlopen(libcholmod) + dlopen(libcolamd) + dlopen(libklu) + dlopen(libldl) + dlopen(librbio) + dlopen(libspqr) + dlopen(libsuitesparseconfig) + dlopen(libumfpack) end +is_available() = true function __init__() - libblastrampoline_jll.eager_mode() - # BSD-3-Clause - global libamd_handle = dlopen(libamd) - global libamd_path = dlpath(libamd_handle) - global libcamd_handle = dlopen(libcamd) - global libcamd_path = dlpath(libcamd_handle) - global libccolamd_handle = dlopen(libccolamd) - global libccolamd_path = dlpath(libccolamd_handle) - global libcolamd_handle = dlopen(libcolamd) - global libcolamd_path = dlpath(libcolamd_handle) - global libsuitesparseconfig_handle = dlopen(libsuitesparseconfig) - global libsuitesparseconfig_path = dlpath(libsuitesparseconfig_handle) + global libamd_path = string(libamd.path) + global libcamd_path = string(libcamd.path) + global libccolamd_path = string(libccolamd.path) + global libcolamd_path = string(libcolamd.path) + global libsuitesparseconfig_path = string(libsuitesparseconfig.path) # LGPL-2.1+ - global libbtf_handle = dlopen(libbtf) - global libbtf_path = dlpath(libbtf_handle) - global libklu_handle = dlopen(libklu) - global libklu_path = dlpath(libklu_handle) - global libldl_handle = dlopen(libldl) - global libldl_path = dlpath(libldl_handle) + global libbtf_path = string(libbtf.path) + global libklu_path = string(libklu.path) + global libldl_path = string(libldl.path) # GPL-2.0+ if Base.USE_GPL_LIBS - global libcholmod_handle = dlopen(libcholmod) - global libcholmod_path = dlpath(libcholmod_handle) - global librbio_handle = dlopen(librbio) - global librbio_path = dlpath(librbio_handle) - global libspqr_handle = dlopen(libspqr) - global libspqr_path = dlpath(libspqr_handle) - global libumfpack_handle = dlopen(libumfpack) - global libumfpack_path = dlpath(libumfpack_handle) + global libcholmod_path = string(libcholmod.path) + global librbio_path = string(librbio.path) + global libspqr_path = string(libspqr.path) + global libumfpack_path = string(libumfpack.path) end global artifact_dir = dirname(Sys.BINDIR) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libamd_path() = libamd_path -get_libbtf_path() = libbtf_path -get_libcamd_path() = libcamd_path -get_libccolamd_path() = libccolamd_path -get_libcholmod_path() = libcholmod_path -get_libcolamd_path() = libcolamd_path -get_libklu_path() = libklu_path -get_libldl_path() = libldl_path -get_librbio_path() = librbio_path -get_libspqr_path() = libspqr_path -get_libsuitesparseconfig_path() = libsuitesparseconfig_path -get_libumfpack_path() = libumfpack_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module SuiteSparse_jll diff --git a/stdlib/TOML/src/TOML.jl b/stdlib/TOML/src/TOML.jl index b37a5ca83c251..d8cb0b063189a 100644 --- a/stdlib/TOML/src/TOML.jl +++ b/stdlib/TOML/src/TOML.jl @@ -31,7 +31,7 @@ _readstring(f::AbstractString) = isfile(f) ? read(f, String) : error(repr(f), ": Parser() Constructor for a TOML `Parser`. Note that in most cases one does not need to -explicitly create a `Parser` but instead one directly use use +explicitly create a `Parser` but instead one directly uses [`TOML.parsefile`](@ref) or [`TOML.parse`](@ref). Using an explicit parser will however reuse some internal data structures which can be beneficial for performance if a larger number of small files are parsed. @@ -148,4 +148,6 @@ Internals.reinit!(p::Parser, str::String; filepath::Union{Nothing, String}=nothi Internals.parse(p::Parser) = Internals.parse(p._p) Internals.tryparse(p::Parser) = Internals.tryparse(p._p) +include("precompile.jl") + end diff --git a/stdlib/TOML/src/precompile.jl b/stdlib/TOML/src/precompile.jl new file mode 100644 index 0000000000000..f3f45478ad28b --- /dev/null +++ b/stdlib/TOML/src/precompile.jl @@ -0,0 +1,38 @@ +if Base.generating_output() +let + # Test TOML content + test_toml = """ + title = "Example" + with_quotes = \"quoted\" + number = 42 + float = 3.14 + boolean = true + date = 2023-01-01 + datetime = 2023-01-01T12:00:00Z + time = 12:00:00 + array = [1, 2, 3] + inline = {a=1, b=1.0, c=[1,2,3], d="foo"} + + [nested] + key = "value" + """ + + test_dict = TOML.parse(test_toml) + + TOML.parse(test_toml) + + io_stream = IOBuffer(test_toml) + TOML.parse(io_stream) + + mktemp() do path, io + write(io, test_toml) + close(io) + TOML.parsefile(path) + end + + mktemp() do path, io + TOML.print(io, test_dict) + close(io) + end +end +end diff --git a/stdlib/TOML/src/print.jl b/stdlib/TOML/src/print.jl index c6c046b9b40c6..741fd96e548a8 100644 --- a/stdlib/TOML/src/print.jl +++ b/stdlib/TOML/src/print.jl @@ -33,7 +33,6 @@ function print_toml_escaped(io::IO, s::AbstractString) end end -const MbyFunc = Union{Function, Nothing} const TOMLValue = Union{AbstractVector, AbstractDict, Bool, Integer, AbstractFloat, AbstractString, Dates.DateTime, Dates.Time, Dates.Date, Base.TOML.DateTime, Base.TOML.Time, Base.TOML.Date} @@ -59,8 +58,8 @@ function printkey(io::IO, keys::Vector{String}) end end -function to_toml_value(f::MbyFunc, value) - if f === nothing +function to_toml_value(@nospecialize(f::Function), value) + if f === identity error("type `$(typeof(value))` is not a valid TOML type, pass a conversion function to `TOML.print`") end toml_value = f(value) @@ -75,12 +74,12 @@ end ########## # Fallback -function printvalue(f::MbyFunc, io::IO, value, sorted::Bool) +function printvalue(f::Function, io::IO, value, sorted::Bool) toml_value = to_toml_value(f, value) @invokelatest printvalue(f, io, toml_value, sorted) end -function printvalue(f::MbyFunc, io::IO, value::AbstractVector, sorted::Bool) +function printvalue(f::Function, io::IO, value::AbstractVector, sorted::Bool) Base.print(io, "[") for (i, x) in enumerate(value) i != 1 && Base.print(io, ", ") @@ -89,7 +88,7 @@ function printvalue(f::MbyFunc, io::IO, value::AbstractVector, sorted::Bool) Base.print(io, "]") end -function printvalue(f::MbyFunc, io::IO, value::TOMLValue, sorted::Bool) +function printvalue(f::Function, io::IO, value::TOMLValue, sorted::Bool) value isa Base.TOML.DateTime && (value = Dates.DateTime(value)) value isa Base.TOML.Time && (value = Dates.Time(value)) value isa Base.TOML.Date && (value = Dates.Date(value)) @@ -117,7 +116,7 @@ function print_integer(io::IO, value::Integer) return end -function print_inline_table(f::MbyFunc, io::IO, value::AbstractDict, sorted::Bool) +function print_inline_table(f::Function, io::IO, value::AbstractDict, sorted::Bool) vkeys = collect(keys(value)) if sorted sort!(vkeys) @@ -138,15 +137,14 @@ end # Tables # ########## -is_table(value) = isa(value, AbstractDict) -is_array_of_tables(value) = isa(value, AbstractArray) && - length(value) > 0 && ( - isa(value, AbstractArray{<:AbstractDict}) || - all(v -> isa(v, AbstractDict), value) - ) -is_tabular(value) = is_table(value) || @invokelatest(is_array_of_tables(value)) +is_table(@nospecialize(value)) = isa(value, AbstractDict) +is_array_of_tables(@nospecialize(value)) = + isa(value, AbstractArray) && + length(value) > 0 && (isa(value, AbstractArray{<:AbstractDict}) || + all(v -> isa(v, AbstractDict), value)) +is_tabular(@nospecialize(value)) = is_table(value) || @invokelatest(is_array_of_tables(value)) -function print_table(f::MbyFunc, io::IO, a::AbstractDict, +function print_table(f::Function, io::IO, a::AbstractDict, ks::Vector{String} = String[]; indent::Int = 0, first_block::Bool = true, @@ -228,7 +226,11 @@ end # API # ####### -print(f::MbyFunc, io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = print_table(f, io, a; sorted, by, inline_tables) -print(f::MbyFunc, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = print(f, stdout, a; sorted, by, inline_tables) -print(io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = print_table(nothing, io, a; sorted, by, inline_tables) -print( a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = print(nothing, stdout, a; sorted, by, inline_tables) +print(f::Function, io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = + print_table(f, io, a; sorted, by, inline_tables) +print(f::Function, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = + print(f, stdout, a; sorted, by, inline_tables) +print(io::IO, a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = + print_table(identity, io, a; sorted, by, inline_tables) +print(a::AbstractDict; sorted::Bool=false, by=identity, inline_tables::IdSet{<:AbstractDict}=IdSet{Dict{String}}()) = + print(identity, stdout, a; sorted, by, inline_tables) diff --git a/stdlib/TOML/test/print.jl b/stdlib/TOML/test/print.jl index e8a6431cb34a7..9734d96b3c8c1 100644 --- a/stdlib/TOML/test/print.jl +++ b/stdlib/TOML/test/print.jl @@ -58,7 +58,7 @@ end [option] """ d = TOML.parse(s) - @test toml_str(d) == "user = \"me\"\n\n[julia]\n\n[option]\n" + @test toml_str(d; sorted=true) == "user = \"me\"\n\n[julia]\n\n[option]\n" end @testset "special characters" begin @@ -83,17 +83,28 @@ loaders = ["gzip", { driver = "csv", args = {delim = "\t"}}] @testset "vec with dicts and non-dicts" begin # https://github.com/JuliaLang/julia/issues/45340 d = Dict("b" => Any[111, Dict("a" => 222, "d" => 333)]) - @test toml_str(d) == "b = [111, {a = 222, d = 333}]\n" + @test toml_str(d) == (sizeof(Int) == 8 ? + "b = [111, {a = 222, d = 333}]\n" : + "b = [111, {d = 333, a = 222}]\n") + d = Dict("b" => Any[Dict("a" => 222, "d" => 333), 111]) - @test toml_str(d) == "b = [{a = 222, d = 333}, 111]\n" + @test toml_str(d) == (sizeof(Int) == 8 ? + "b = [{a = 222, d = 333}, 111]\n" : + "b = [{d = 333, a = 222}, 111]\n") d = Dict("b" => Any[Dict("a" => 222, "d" => 333)]) - @test toml_str(d) == """ - [[b]] - a = 222 - d = 333 - """ + @test toml_str(d) == (sizeof(Int) == 8 ? + """ + [[b]] + a = 222 + d = 333 + """ : + """ + [[b]] + d = 333 + a = 222 + """) # https://github.com/JuliaLang/julia/pull/57584 d = Dict("b" => [MyStruct(1), MyStruct(2)]) diff --git a/stdlib/Test/docs/src/index.md b/stdlib/Test/docs/src/index.md index 7451f8434f4fa..ab06531152554 100644 --- a/stdlib/Test/docs/src/index.md +++ b/stdlib/Test/docs/src/index.md @@ -59,7 +59,6 @@ julia> @test foo("f") == 20 Test Failed at none:1 Expression: foo("f") == 20 Evaluated: 1 == 20 - ERROR: There was an error during testing ``` @@ -230,7 +229,6 @@ Test Passed julia> @test 1 ≈ 0.999999 Test Failed at none:1 Expression: 1 ≈ 0.999999 - ERROR: There was an error during testing ``` You can specify relative and absolute tolerances by setting the `rtol` and `atol` keyword arguments of `isapprox`, respectively, @@ -478,13 +476,13 @@ Using our knowledge of `Test.jl`, here are some example tests we could add to `m @testset "Testset 1" begin @test 2 == simple_add(1, 1) @test 3.5 == simple_add(1, 2.5) - @test_throws MethodError simple_add(1, "A") - @test_throws MethodError simple_add(1, 2, 3) + @test_throws MethodError simple_add(1, "A") + @test_throws MethodError simple_add(1, 2, 3) end @testset "Testset 2" begin @test 1.0 == type_multiply(1.0, 1.0) - @test isa(type_multiply(2.0, 2.0), Float64) + @test isa(type_multiply(2.0, 2.0), Float64) @test_throws MethodError type_multiply(1, 2.5) end ``` diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index e3a52baf4fafa..7f73662c87938 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -182,7 +182,7 @@ function Base.show(io::IO, t::Fail) print(io, "\n") if t.backtrace !== nothing # Capture error message and indent to match - join(io, (" " * line for line in split(t.backtrace, "\n")), "\n") + join(io, (" " * line for line in filter!(!isempty, split(t.backtrace, "\n"))), "\n") end end elseif t.test_type === :test_throws_nothing @@ -199,7 +199,6 @@ function Base.show(io::IO, t::Fail) print(io, "\n Context: ", t.context) end end - println(io) # add some visual space to separate sequential failures end """ @@ -215,15 +214,22 @@ struct Error <: Result orig_expr::String value::String backtrace::String + context::Union{Nothing, String} source::LineNumberNode - function Error(test_type::Symbol, orig_expr, value, bt, source::LineNumberNode) - if test_type === :test_error - bt = scrub_exc_stack(bt, nothing, extract_file(source)) - end - if test_type === :test_error || test_type === :nontest_error - bt_str = try # try the latest world for this, since we might have eval'd new code for show - Base.invokelatest(sprint, Base.show_exception_stack, bt; context=stdout) + function Error(test_type::Symbol, orig_expr, value, excs::Union{Base.ExceptionStack,Nothing}, + source::LineNumberNode, context::Union{Nothing, String}=nothing) + @nospecialize orig_expr value + bt_str = "" + if !isnothing(excs) + if test_type === :test_error + excs = scrub_exc_stack(excs, nothing, extract_file(source)) + end + if test_type === :test_error || test_type === :nontest_error + bt_str = try + # try the latest world for this, since we might have eval'd new code for show + # Apply REPL backtrace scrubbing to hide REPL internals, similar to how REPL.jl handles it + Base.invokelatest(sprint, Base.show_exception_stack, Base.scrub_repl_backtrace(excs); context=stdout) catch ex "#=ERROR showing exception stack=# " * try @@ -232,8 +238,7 @@ struct Error <: Result "of type " * string(typeof(ex)) end end - else - bt_str = "" + end end value = try # try the latest world for this, since we might have eval'd new code for show Base.invokelatest(sprint, show, value, context = :limit => true) @@ -249,8 +254,14 @@ struct Error <: Result string(orig_expr), value, bt_str, + context, source) end + + # Internal constructor for creating Error with pre-processed values (used by ContextTestSet) + function Error(test_type::Symbol, orig_expr::String, value::String, backtrace::String, context::Union{Nothing, String}, source::LineNumberNode) + return new(test_type, orig_expr, value, backtrace, context, source) + end end function Base.show(io::IO, t::Error) @@ -268,8 +279,11 @@ function Base.show(io::IO, t::Error) elseif t.test_type === :test_error println(io, " Test threw exception") println(io, " Expression: ", t.orig_expr) + if t.context !== nothing + println(io, " Context: ", t.context) + end # Capture error message and indent to match - join(io, (" " * line for line in split(t.backtrace, "\n")), "\n") + join(io, (" " * line for line in filter!(!isempty, split(t.backtrace, "\n"))), "\n") elseif t.test_type === :test_unbroken # A test that was expected to fail did not println(io, " Unexpected Pass") @@ -279,7 +293,7 @@ function Base.show(io::IO, t::Error) # we had an error outside of a @test println(io, " Got exception outside of a @test") # Capture error message and indent to match - join(io, (" " * line for line in split(t.backtrace, "\n")), "\n") + join(io, (" " * line for line in filter!(!isempty, split(t.backtrace, "\n"))), "\n") end end @@ -336,7 +350,7 @@ end struct Threw <: ExecutionResult exception - backtrace::Union{Nothing,Vector{Any}} + current_exceptions::Base.ExceptionStack source::LineNumberNode end @@ -736,7 +750,7 @@ end # An internal function, called by the code generated by the @test # macro to actually perform the evaluation and manage the result. -function do_test(result::ExecutionResult, orig_expr) +function do_test(result::ExecutionResult, @nospecialize orig_expr) # get_testset() returns the most recently added test set # We then call record() with this test set and the test result if isa(result, Returned) @@ -752,30 +766,30 @@ function do_test(result::ExecutionResult, orig_expr) Fail(:test, orig_expr, result.data, value, nothing, result.source, false) else # If the result is non-Boolean, this counts as an Error - Error(:test_nonbool, orig_expr, value, nothing, result.source) + Error(:test_nonbool, orig_expr, value, nothing, result.source, nothing) end else # The predicate couldn't be evaluated without throwing an # exception, so that is an Error and not a Fail @assert isa(result, Threw) - testres = Error(:test_error, orig_expr, result.exception, result.backtrace::Vector{Any}, result.source) + testres = Error(:test_error, orig_expr, result.exception, result.current_exceptions, result.source, nothing) end isa(testres, Pass) || trigger_test_failure_break(result) record(get_testset(), testres) end -function do_broken_test(result::ExecutionResult, orig_expr) +function do_broken_test(result::ExecutionResult, @nospecialize orig_expr) testres = Broken(:test, orig_expr) # Assume the test is broken and only change if the result is true if isa(result, Returned) value = result.value if isa(value, Bool) if value - testres = Error(:test_unbroken, orig_expr, value, nothing, result.source) + testres = Error(:test_unbroken, orig_expr, value, nothing, result.source, nothing) end else # If the result is non-Boolean, this counts as an Error - testres = Error(:test_nonbool, orig_expr, value, nothing, result.source) + testres = Error(:test_nonbool, orig_expr, value, nothing, result.source, nothing) end end record(get_testset(), testres) @@ -785,17 +799,29 @@ end """ @test_throws exception expr + @test_throws extype pattern expr Tests that the expression `expr` throws `exception`. The exception may specify either a type, a string, regular expression, or list of strings occurring in the displayed error message, a matching function, or a value (which will be tested for equality by comparing fields). + +In the two-argument form, `@test_throws exception expr`, the `exception` can be a type or a pattern. + +In the three-argument form, `@test_throws extype pattern expr`, both the exception type and +a message pattern are tested. The `extype` must be a type, and `pattern` may be +a string, regular expression, or list of strings occurring in the displayed error message, +a matching function, or a value. + Note that `@test_throws` does not support a trailing keyword form. !!! compat "Julia 1.8" The ability to specify anything other than a type or a value as `exception` requires Julia v1.8 or later. +!!! compat "Julia 1.13" + The three-argument form `@test_throws extype pattern expr` requires Julia v1.12 or later. + # Examples ```jldoctest julia> @test_throws BoundsError [1, 2, 3][4] @@ -809,13 +835,19 @@ Test Passed julia> @test_throws "Try sqrt(Complex" sqrt(-1) Test Passed Message: "DomainError with -1.0:\\nsqrt was called with a negative real argument but will only return a complex result if called with a complex argument. Try sqrt(Complex(x))." + +julia> @test_throws ErrorException "error foo" error("error foo 1") +Test Passed + Thrown: ErrorException ``` -In the final example, instead of matching a single string it could alternatively have been performed with: +In the third example, instead of matching a single string it could alternatively have been performed with: - `["Try", "Complex"]` (a list of strings) - `r"Try sqrt\\([Cc]omplex"` (a regular expression) - `str -> occursin("complex", str)` (a matching function) + +In the final example, both the exception type (`ErrorException`) and message pattern (`"error foo"`) are tested. """ macro test_throws(extype, ex) orig_ex = Expr(:inert, ex) @@ -833,6 +865,22 @@ macro test_throws(extype, ex) return :(do_test_throws($result, $orig_ex, $(esc(extype)))) end +macro test_throws(extype, pattern, ex) + orig_ex = Expr(:inert, ex) + ex = Expr(:block, __source__, esc(ex)) + result = quote + try + Returned($ex, nothing, $(QuoteNode(__source__))) + catch _e + if $(esc(extype)) != InterruptException && _e isa InterruptException + rethrow() + end + Threw(_e, Base.current_exceptions(), $(QuoteNode(__source__))) + end + end + return :(do_test_throws($result, $orig_ex, $(esc(extype)), $(esc(pattern)))) +end + const MACROEXPAND_LIKE = Symbol.(("@macroexpand", "@macroexpand1", "macroexpand")) function isequalexception(@nospecialize(a), @nospecialize(b)) @@ -850,72 +898,105 @@ end # An internal function, called by the code generated by @test_throws # to evaluate and catch the thrown exception - if it exists -function do_test_throws(result::ExecutionResult, orig_expr, extype) +function do_test_throws(result::ExecutionResult, @nospecialize(orig_expr), extype, pattern=nothing) if isa(result, Threw) # Check that the right type of exception was thrown success = false message_only = false exc = result.exception - # NB: Throwing LoadError from macroexpands is deprecated, but in order to limit - # the breakage in package tests we add extra logic here. - from_macroexpand = - orig_expr isa Expr && - orig_expr.head in (:call, :macrocall) && - orig_expr.args[1] in MACROEXPAND_LIKE - if isa(extype, Type) - success = - if from_macroexpand && extype == LoadError && exc isa Exception - Base.depwarn("macroexpand no longer throws a LoadError so `@test_throws LoadError ...` is deprecated and passed without checking the error type!", :do_test_throws) - true - elseif extype == ErrorException && isa(exc, FieldError) - Base.depwarn(lazy"Using ErrorException to test field access is deprecated; use FieldError instead.", :do_test_throws) - true - else - isa(exc, extype) - end - elseif isa(extype, Exception) || !isa(exc, Exception) - if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc) - extype = extype.error # deprecated + + # Handle three-argument form (type + pattern) + if pattern !== nothing + # In 3-arg form, first argument must be a type + if !isa(extype, Type) + testres = Fail(:test_throws_wrong, orig_expr, extype, exc, nothing, result.source, false, "First argument must be an exception type in three-argument form") + record(get_testset(), testres) + return end - # Support `UndefVarError(:x)` meaning `UndefVarError(:x, scope)` for any `scope`. - # Retains the behaviour from pre-v1.11 when `UndefVarError` didn't have `scope`. - if isa(extype, UndefVarError) && !isdefined(extype, :scope) - success = exc isa UndefVarError && exc.var == extype.var - else isa(exc, typeof(extype)) - success = isequalexception(exc, extype) + + # Format combined expected value for display + pattern_str = isa(pattern, AbstractString) ? repr(pattern) : + isa(pattern, Function) ? "< match function >" : + string(pattern) + combined_expected = string(extype) * " with pattern " * pattern_str + + # Check both type and pattern + type_success = isa(exc, extype) + if type_success + exc_msg = sprint(showerror, exc) + pattern_success = contains_warn(exc_msg, pattern) + success = pattern_success + else + success = false end + extype = combined_expected # Use combined format for all results else - message_only = true - exc = sprint(showerror, exc) - success = contains_warn(exc, extype) - exc = repr(exc) - if isa(extype, AbstractString) - extype = repr(extype) - elseif isa(extype, Function) - extype = "< match function >" + # Original two-argument form logic + # NB: Throwing LoadError from macroexpands is deprecated, but in order to limit + # the breakage in package tests we add extra logic here. + from_macroexpand = + orig_expr isa Expr && + orig_expr.head in (:call, :macrocall) && + orig_expr.args[1] in MACROEXPAND_LIKE + if isa(extype, Type) + success = + if from_macroexpand && extype == LoadError && exc isa Exception + Base.depwarn("macroexpand no longer throws a LoadError so `@test_throws LoadError ...` is deprecated and passed without checking the error type!", :do_test_throws) + true + elseif extype == ErrorException && isa(exc, FieldError) + Base.depwarn(lazy"Using ErrorException to test field access is deprecated; use FieldError instead.", :do_test_throws) + true + else + isa(exc, extype) + end + elseif isa(extype, Exception) || !isa(exc, Exception) + if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc) + extype = extype.error # deprecated + end + # Support `UndefVarError(:x)` meaning `UndefVarError(:x, scope)` for any `scope`. + # Retains the behaviour from pre-v1.11 when `UndefVarError` didn't have `scope`. + if isa(extype, UndefVarError) && !isdefined(extype, :scope) + success = exc isa UndefVarError && exc.var == extype.var + else isa(exc, typeof(extype)) + success = isequalexception(exc, extype) + end + else + message_only = true + exc = sprint(showerror, exc) + success = contains_warn(exc, extype) + exc = repr(exc) + if isa(extype, AbstractString) + extype = repr(extype) + elseif isa(extype, Function) + extype = "< match function >" + end end end if success testres = Pass(:test_throws, orig_expr, extype, exc, result.source, message_only) else - if result.backtrace !== nothing - bt = scrub_exc_stack(result.backtrace, nothing, extract_file(result.source)) - bt_str = try # try the latest world for this, since we might have eval'd new code for show - Base.invokelatest(sprint, Base.show_exception_stack, bt; context=stdout) - catch ex - "#=ERROR showing exception stack=# " * - try - sprint(Base.showerror, ex, catch_backtrace(); context=stdout) - catch - "of type " * string(typeof(ex)) - end - end - else - bt_str = nothing + excs = result.current_exceptions + bt = scrub_exc_stack(excs, nothing, extract_file(result.source)) + bt_str = try # try the latest world for this, since we might have eval'd new code for show + Base.invokelatest(sprint, Base.show_exception_stack, bt; context=stdout) + catch ex + "#=ERROR showing exception stack=# " * + try + sprint(Base.showerror, ex, catch_backtrace(); context=stdout) + catch + "of type " * string(typeof(ex)) + end end testres = Fail(:test_throws_wrong, orig_expr, extype, exc, nothing, result.source, message_only, bt_str) end else + # Handle no exception case - need to format extype properly for 3-arg form + if pattern !== nothing + pattern_str = isa(pattern, AbstractString) ? repr(pattern) : + isa(pattern, Function) ? "< match function >" : + string(pattern) + extype = string(extype) * " with pattern " * pattern_str + end testres = Fail(:test_throws_nothing, orig_expr, extype, nothing, nothing, result.source, false) end record(get_testset(), testres) @@ -1063,7 +1144,7 @@ end A simple fallback test set that throws immediately on a failure. """ struct FallbackTestSet <: AbstractTestSet end -fallback_testset = FallbackTestSet() +const fallback_testset = FallbackTestSet() struct FallbackTestSetException <: Exception msg::String @@ -1109,6 +1190,13 @@ function record(c::ContextTestSet, t::Fail) context = t.context === nothing ? context : string(t.context, "\n ", context) record(c.parent_ts, Fail(t.test_type, t.orig_expr, t.data, t.value, context, t.source, t.message_only)) end +function record(c::ContextTestSet, t::Error) + context = string(c.context_name, " = ", c.context) + context = t.context === nothing ? context : string(t.context, "\n ", context) + # Create a new Error with the same data but updated context using internal constructor + new_error = Error(t.test_type, t.orig_expr, t.value, t.backtrace, context, t.source) + record(c.parent_ts, new_error) +end #----------------------------------------------------------------------- @@ -1169,12 +1257,13 @@ end # but do not terminate. Print a backtrace. function record(ts::DefaultTestSet, t::Union{Fail, Error}; print_result::Bool=TESTSET_PRINT_ENABLE[]) if print_result + println() # add some visual space to separate sequential failures print(ts.description, ": ") # don't print for interrupted tests if !(t isa Error) || t.test_type !== :test_interrupted print(t) if !isa(t, Error) # if not gets printed in the show method - Base.show_backtrace(stdout, scrub_backtrace(backtrace(), ts.file, extract_file(t.source))) + Base.show_backtrace(stdout, scrub_backtrace(backtrace(), ts.file, extract_file(t.source)); prefix=" ") end println() end @@ -1332,7 +1421,7 @@ function finish(ts::DefaultTestSet; print_results::Bool=TESTSET_PRINT_ENABLE[]) end # return the testset so it is returned from the @testset macro - ts + return ts end # Recursive function that finds the column that the result counts @@ -1356,15 +1445,15 @@ get_alignment(ts, depth::Int) = 0 # Recursive function that fetches backtraces for any and all errors # or failures the testset and its children encountered function filter_errors(ts::DefaultTestSet) - efs = [] + efs = Any[] for t in ts.results if isa(t, DefaultTestSet) append!(efs, filter_errors(t)) elseif isa(t, Union{Fail, Error}) - append!(efs, [t]) + push!(efs, t) end end - efs + return efs end """ @@ -1650,14 +1739,14 @@ trigonometric identities | 4 4 0.2s # `@testset for` -When `@testset for` is used, the macro starts a new test for each iteration of +When `@testset for` is used, the macro starts a new test set for each iteration of the provided loop. The semantics of each test set are otherwise identical to that -of that `begin/end` case (as if used for each loop iteration). +of the `begin/end` case (as if used for each loop iteration). # `@testset let` When `@testset let` is used, the macro starts a *transparent* test set with -the given object added as a context object to any failing test contained +the given object added as a context object to any failing or erroring test contained therein. This is useful when performing a set of related tests on one larger object and it is desirable to print this larger object when any of the individual tests fail. Transparent test sets do not introduce additional levels @@ -1670,6 +1759,9 @@ parent test set (with the context object appended to any failing tests.) !!! compat "Julia 1.10" Multiple `let` assignments are supported since Julia 1.10. +!!! compat "Julia 1.13" + Context is shown when a test errors since Julia 1.13. + # Special implicit world age increment for `@testset begin` World age inside `@testset begin` increments implicitly after every statement. @@ -1687,7 +1779,6 @@ Test Failed at none:3 Expression: !(iszero(real(logi))) Evaluated: !(iszero(0.0)) Context: logi = 0.0 + 1.5707963267948966im - ERROR: There was an error during testing julia> @testset let logi = log(im), op = !iszero @@ -1699,7 +1790,6 @@ Test Failed at none:3 Evaluated: op(0.0) Context: logi = 0.0 + 1.5707963267948966im op = !iszero - ERROR: There was an error during testing ``` """ @@ -1728,6 +1818,10 @@ end trigger_test_failure_break(@nospecialize(err)) = ccall(:jl_test_failure_breakpoint, Cvoid, (Any,), err) +is_failfast_error(err::FailFastError) = true +is_failfast_error(err::LoadError) = is_failfast_error(err.error) # handle `include` barrier +is_failfast_error(err) = false + """ Generate the code for an `@testset` with a `let` argument. """ @@ -1839,10 +1933,10 @@ function testset_beginend_call(args, tests, source) # something in the test block threw an error. Count that as an # error in this test set trigger_test_failure_break(err) - if err isa FailFastError + if is_failfast_error(err) get_testset_depth() > 1 ? rethrow() : failfast_print() else - record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing)) end finally copy!(default_rng(), default_rng_orig) @@ -1927,8 +2021,10 @@ function testset_forloop(args, testloop, source) # Something in the test block threw an error. Count that as an # error in this test set trigger_test_failure_break(err) - if !isa(err, FailFastError) - record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)))) + if is_failfast_error(err) + get_testset_depth() > 1 ? rethrow() : failfast_print() + else + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing)) end end end @@ -1950,6 +2046,7 @@ function testset_forloop(args, testloop, source) # Handle `return` in test body if !first_iteration && !finish_errored pop_testset() + @assert @isdefined(ts) "Assertion to tell the compiler about the definedness of this variable" push!(arr, finish(ts)) end copy!(default_rng(), default_rng_orig) @@ -2136,7 +2233,7 @@ function _inferred(ex, mod, allow = :(Union{})) kwargs = gensym() quote $(esc(args)), $(esc(kwargs)), result = $(esc(Expr(:call, _args_and_call, ex.args[2:end]..., ex.args[1]))) - inftype = $(gen_call_with_extracted_types(mod, Base.infer_return_type, :($(ex.args[1])($(args)...; $(kwargs)...)))) + inftype = $(gen_call_with_extracted_types(mod, Base.infer_return_type, :($(ex.args[1])($(args)...; $(kwargs)...)); is_source_reflection = false)) end else # No keywords @@ -2200,7 +2297,6 @@ function detect_ambiguities(mods::Module...; end function examine(mt::Core.MethodTable) for m in Base.MethodList(mt) - m.sig == Tuple && continue # ignore Builtins is_in_mods(parentmodule(m), recursive, mods) || continue world = Base.get_world_counter() ambig = Ref{Int32}(0) @@ -2217,30 +2313,7 @@ function detect_ambiguities(mods::Module...; end end end - work = Base.loaded_modules_array() - filter!(mod -> mod === parentmodule(mod), work) # some items in loaded_modules_array are not top modules (really just Base) - while !isempty(work) - mod = pop!(work) - for n in names(mod, all = true) - Base.isdeprecated(mod, n) && continue - if !isdefined(mod, n) - if is_in_mods(mod, recursive, mods) - if allowed_undefineds === nothing || GlobalRef(mod, n) ∉ allowed_undefineds - println("Skipping ", mod, '.', n) # typically stale exports - end - end - continue - end - f = Base.unwrap_unionall(getfield(mod, n)) - if isa(f, Module) && f !== mod && parentmodule(f) === mod && nameof(f) === n - push!(work, f) - elseif isa(f, DataType) && isdefined(f.name, :mt) && parentmodule(f) === mod && nameof(f) === n && f.name.mt !== Symbol.name.mt && f.name.mt !== DataType.name.mt - examine(f.name.mt) - end - end - end - examine(Symbol.name.mt) - examine(DataType.name.mt) + examine(Core.methodtable) return collect(ambs) end @@ -2288,30 +2361,7 @@ function detect_unbound_args(mods...; push!(ambs, m) end end - work = Base.loaded_modules_array() - filter!(mod -> mod === parentmodule(mod), work) # some items in loaded_modules_array are not top modules (really just Base) - while !isempty(work) - mod = pop!(work) - for n in names(mod, all = true) - Base.isdeprecated(mod, n) && continue - if !isdefined(mod, n) - if is_in_mods(mod, recursive, mods) - if allowed_undefineds === nothing || GlobalRef(mod, n) ∉ allowed_undefineds - println("Skipping ", mod, '.', n) # typically stale exports - end - end - continue - end - f = Base.unwrap_unionall(getfield(mod, n)) - if isa(f, Module) && f !== mod && parentmodule(f) === mod && nameof(f) === n - push!(work, f) - elseif isa(f, DataType) && isdefined(f.name, :mt) && parentmodule(f) === mod && nameof(f) === n && f.name.mt !== Symbol.name.mt && f.name.mt !== DataType.name.mt - examine(f.name.mt) - end - end - end - examine(Symbol.name.mt) - examine(DataType.name.mt) + examine(Core.methodtable) return collect(ambs) end diff --git a/stdlib/Test/src/logging.jl b/stdlib/Test/src/logging.jl index b224d79e47cd9..a3a94a642f250 100644 --- a/stdlib/Test/src/logging.jl +++ b/stdlib/Test/src/logging.jl @@ -107,8 +107,8 @@ function Logging.handle_message(logger::TestLogger, level, msg, _module, if maxlog isa Core.BuiltinInts @lock logger.lock begin remaining = get!(logger.message_limits, id, Int(maxlog)::Int) + remaining == 0 && return logger.message_limits[id] = remaining - 1 - remaining > 0 || return end end end diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index e5b9bd4bdd089..1b571286abd8c 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -107,6 +107,25 @@ end @test_throws "\"" throw("\"") @test_throws Returns(false) throw(Returns(false)) end + +@testset "Pass - exception with pattern (3-arg form)" begin + # Test 3-argument form: @test_throws ExceptionType pattern expr + @test_throws ErrorException "error foo" error("error foo 1") + @test_throws DomainError r"sqrt.*negative" sqrt(-1) + @test_throws BoundsError "at index [2]" [1][2] + @test_throws ErrorException ["error", "foo"] error("error foo bar") + + # Test with function pattern + @test_throws ErrorException (s -> occursin("foo", s)) error("error foo bar") + + # Test output format + let result = @test_throws ErrorException "error foo" error("error foo 1") + output = sprint(show, result) + @test occursin("Test Passed", output) + @test occursin("Thrown: ErrorException", output) + end +end + # Test printing of Fail results include("nothrow_testset.jl") @@ -390,7 +409,7 @@ let retval_tests = @testset NoThrowTestSet begin ts = Test.DefaultTestSet("Mock for testing retval of record(::DefaultTestSet, ::T <: Result) methods") pass_mock = Test.Pass(:test, 1, 2, 3, LineNumberNode(0, "A Pass Mock")) @test Test.record(ts, pass_mock) isa Test.Pass - error_mock = Test.Error(:test, 1, 2, 3, LineNumberNode(0, "An Error Mock")) + error_mock = Test.Error(:test, 1, 2, nothing, LineNumberNode(0, "An Error Mock"), nothing) @test Test.record(ts, error_mock; print_result=false) isa Test.Error fail_mock = Test.Fail(:test, 1, 2, 3, nothing, LineNumberNode(0, "A Fail Mock"), false) @test Test.record(ts, fail_mock; print_result=false) isa Test.Fail @@ -402,6 +421,47 @@ let retval_tests = @testset NoThrowTestSet begin end end +@testset "Fail - exception with pattern (3-arg form)" begin + # Test type mismatch + let fails = @testset NoThrowTestSet begin + @test_throws ArgumentError "error foo" error("error foo 1") # Wrong type + end + @test length(fails) == 1 + @test fails[1] isa Test.Fail + @test fails[1].test_type === :test_throws_wrong + @test occursin("ArgumentError with pattern \"error foo\"", fails[1].data) + end + + # Test pattern mismatch + let fails = @testset NoThrowTestSet begin + @test_throws ErrorException "wrong pattern" error("error foo 1") # Wrong pattern + end + @test length(fails) == 1 + @test fails[1] isa Test.Fail + @test fails[1].test_type === :test_throws_wrong + @test occursin("ErrorException with pattern \"wrong pattern\"", fails[1].data) + end + + # Test no exception thrown + let fails = @testset NoThrowTestSet begin + @test_throws ErrorException "error foo" 1 + 1 # No exception + end + @test length(fails) == 1 + @test fails[1] isa Test.Fail + @test fails[1].test_type === :test_throws_nothing + @test occursin("ErrorException with pattern \"error foo\"", fails[1].data) + end + + # Test first argument must be a type + let fails = @testset NoThrowTestSet begin + @test_throws "not a type" "error foo" error("error foo 1") # First arg not a type + end + @test length(fails) == 1 + @test fails[1] isa Test.Fail + @test fails[1].test_type === :test_throws_wrong + end +end + @testset "printing of a TestSetException" begin tse_str = sprint(show, Test.TestSetException(1, 2, 3, 4, Vector{Union{Test.Error, Test.Fail}}())) @test occursin("1 passed", tse_str) @@ -816,9 +876,9 @@ end """) msg = read(pipeline(ignorestatus(`$(Base.julia_cmd()) --startup-file=no --color=no $runtests`), stderr=devnull), String) msg = win2unix(msg) - regex = r"((?:Tests|Other tests|Testset without source): Test Failed (?:.|\n)*?)\n\nStacktrace:(?:.|\n)*?(?=\n(?:Tests|Other tests))" + regex = r"((?:Tests|Other tests|Testset without source): Test Failed (?:.|\n)*?)\n Stacktrace:(?:.|\n)*?(?=\n(?:Tests|Other tests))" failures = map(eachmatch(regex, msg)) do m - m = match(r"(Tests|Other tests|Testset without source): .*? at (.*?)\n Expression: (.*)(?:.|\n)*\n+Stacktrace:\n((?:.|\n)*)", m.match) + m = match(r"(Tests|Other tests|Testset without source): .*? at (.*?)\n Expression: (.*)(?:.|\n)*\n Stacktrace:\n((?:.|\n)*)", m.match) (; testset = m[1], source = m[2], ex = m[3], stacktrace = m[4]) end @test length(failures) == 8 # 8 failed tests @@ -1394,7 +1454,7 @@ end @test occursin(expected, result) end end - @testset "failfast" begin + @testset "failfast begin-end" begin expected = r""" Test Summary: \| Fail Total +Time Foo \| 1 1 \s*\d*\.\ds @@ -1419,6 +1479,32 @@ end @test occursin(expected, result) end end + @testset "failfast for-loop" begin + expected = r""" + Test Summary: \| Fail Total +Time + Foo \| 1 1 \s*\d*\.\ds + 1 \| 1 1 \s*\d*\.\ds + """ + mktemp() do f, _ + write(f, + """ + using Test + + @testset "Foo" failfast=true begin + @testset "\$x" for x in 1:2 + @test false + end + @testset "Bar" begin + @test false + @test true + end + end + """) + cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f` + result = read(pipeline(ignorestatus(cmd), stderr=devnull), String) + @test occursin(expected, result) + end + end @testset "failfast passes to child testsets" begin expected = r""" Test Summary: \| Fail Total +Time @@ -1866,3 +1952,59 @@ end @test _escape_call(:((==).(x, y))) == (; func=Expr(:., esc(:(==))), args, kwargs, quoted_func=QuoteNode(Expr(:., :(==)))) end end + +@testset "Context display in @testset let blocks" begin + # Mock parent testset that just captures results + struct MockParentTestSet <: Test.AbstractTestSet + results::Vector{Any} + MockParentTestSet() = new([]) + end + Test.record(ts::MockParentTestSet, t) = (push!(ts.results, t); t) + Test.finish(ts::MockParentTestSet) = ts + + @testset "context shown when a context testset fails" begin + mock_parent1 = MockParentTestSet() + ctx_ts1 = Test.ContextTestSet(mock_parent1, :x, 42) + + fail_result = Test.Fail(:test, "x == 99", "42 == 99", "42", nothing, LineNumberNode(1, :test), false) + Test.record(ctx_ts1, fail_result) + + @test length(mock_parent1.results) == 1 + recorded_fail = mock_parent1.results[1] + @test recorded_fail isa Test.Fail + @test recorded_fail.context !== nothing + @test occursin("x = 42", recorded_fail.context) + end + + @testset "context shown when a context testset errors" begin + mock_parent2 = MockParentTestSet() + ctx_ts2 = Test.ContextTestSet(mock_parent2, :x, 42) + + # Use internal constructor to create Error with pre-processed values + error_result = Test.Error(:test_error, "error(\"test\")", "ErrorException(\"test\")", "test\nStacktrace:\n [1] error()", nothing, LineNumberNode(1, :test)) + Test.record(ctx_ts2, error_result) + + @test length(mock_parent2.results) == 1 + recorded_error = mock_parent2.results[1] + @test recorded_error isa Test.Error + @test recorded_error.context !== nothing + @test occursin("x = 42", recorded_error.context) + + # Context shows up in string representation + error_str = sprint(show, recorded_error) + @test occursin("Context:", error_str) + @test occursin("x = 42", error_str) + + # Multiple variables context + mock_parent3 = MockParentTestSet() + ctx_ts3 = Test.ContextTestSet(mock_parent3, :(x, y), (42, "hello")) + + error_result2 = Test.Error(:test_error, "error(\"test\")", "ErrorException(\"test\")", "test\nStacktrace:\n [1] error()", nothing, LineNumberNode(1, :test)) + Test.record(ctx_ts3, error_result2) + + recorded_error2 = mock_parent3.results[1] + @test recorded_error2 isa Test.Error + @test recorded_error2.context !== nothing + @test occursin("(x, y) = (42, \"hello\")", recorded_error2.context) + end +end diff --git a/stdlib/UUIDs/src/UUIDs.jl b/stdlib/UUIDs/src/UUIDs.jl index 989e0e16e2e58..881e65b218d87 100644 --- a/stdlib/UUIDs/src/UUIDs.jl +++ b/stdlib/UUIDs/src/UUIDs.jl @@ -2,7 +2,7 @@ """ This module provides universally unique identifiers (UUIDs), -along with functions creating the different variants. +along with functions for creating the different variants. """ module UUIDs diff --git a/stdlib/Unicode/test/runtests.jl b/stdlib/Unicode/test/runtests.jl index 7fa57508cffbf..2af7015afa249 100644 --- a/stdlib/Unicode/test/runtests.jl +++ b/stdlib/Unicode/test/runtests.jl @@ -284,6 +284,8 @@ end @test_throws BoundsError graphemes("äöüx", 2:5) @test_throws BoundsError graphemes("äöüx", 5:5) @test_throws ArgumentError graphemes("äöüx", 0:1) + + @test @allocated(length(graphemes("äöüx"))) == 0 end @testset "#3721, #6939 up-to-date character widths" begin diff --git a/stdlib/Zlib_jll/src/Zlib_jll.jl b/stdlib/Zlib_jll/src/Zlib_jll.jl index fb043c7143789..a52168bf244b0 100644 --- a/stdlib/Zlib_jll/src/Zlib_jll.jl +++ b/stdlib/Zlib_jll/src/Zlib_jll.jl @@ -4,41 +4,43 @@ baremodule Zlib_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export libz # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libz_handle::Ptr{Cvoid} = C_NULL -libz_path::String = "" -if Sys.iswindows() - const libz = "libz.dll" -elseif Sys.isapple() - const libz = "@rpath/libz.1.dylib" -else - const libz = "libz.so.1" +libz_path::String = "" +const libz = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libz.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libz.1.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libz.so.1") + else + error("Zlib_jll: Library 'libz' is not available for $(Sys.KERNEL)") + end +) + +function eager_mode() + dlopen(libz) end +is_available() = true function __init__() - global libz_handle = dlopen(libz) - global libz_path = dlpath(libz_handle) + global libz_path = string(libz.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libz_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libz_path() = libz_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module Zlib_jll diff --git a/stdlib/Zstd_jll/Project.toml b/stdlib/Zstd_jll/Project.toml new file mode 100644 index 0000000000000..1f8172cdc75bc --- /dev/null +++ b/stdlib/Zstd_jll/Project.toml @@ -0,0 +1,17 @@ +name = "Zstd_jll" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[deps] +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[compat] +CompilerSupportLibraries_jll = "1.3.0" +julia = "1.6" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/stdlib/Zstd_jll/src/Zstd_jll.jl b/stdlib/Zstd_jll/src/Zstd_jll.jl new file mode 100644 index 0000000000000..30481a12e0979 --- /dev/null +++ b/stdlib/Zstd_jll/src/Zstd_jll.jl @@ -0,0 +1,109 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## dummy stub for https://github.com/JuliaBinaryWrappers/Zstd_jll.j: +# +baremodule Zstd_jll +using Base, Libdl +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end + +export libzstd, zstd, zstdmt + +# These get calculated in __init__() +const PATH = Ref("") +const PATH_list = String[] +const LIBPATH = Ref("") +const LIBPATH_list = String[] +artifact_dir::String = "" + +libzstd_path::String = "" +zstd_path::String = "" +zstdmt_path::String = "" +const libzstd = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libzstd-1.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libzstd.1.dylib") + elseif Sys.islinux() || Sys.isfreebsd() + BundledLazyLibraryPath("libzstd.so.1") + else + error("Zstd_jll: Library 'libzstd' is not available for $(Sys.KERNEL)") + end; + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libgcc_s] + else + LazyLibrary[] + end +) + +if Sys.iswindows() + const zstd_exe = "zstd.exe" + const zstdmt_exe = "zstdmt.exe" +else + const zstd_exe = "zstd" + const zstdmt_exe = "zstdmt" +end + +if Sys.iswindows() + const pathsep = ';' +elseif Sys.isapple() + const pathsep = ':' +else + const pathsep = ':' +end + +if Sys.iswindows() +function adjust_ENV(cmd::Cmd) + dllPATH = Sys.BINDIR + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? dllPATH : "$dllPATH$pathsep$oldPATH" + return addenv(cmd, "PATH"=>newPATH) +end +else +adjust_ENV(cmd::Cmd) = cmd +end + +function adjust_ENV() + addPATH = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR) + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? addPATH : "$addPATH$pathsep$oldPATH" + return ("PATH"=>newPATH,) +end + +function zstd(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstd()) + end +end +function zstdmt(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstdmt()) + end +end +zstd() = adjust_ENV(`$zstd_path`) +zstdmt() = adjust_ENV(`$zstdmt_path`) + +# Function to eagerly dlopen our library and thus resolve all dependencies +function eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + dlopen(libzstd) +end + +is_available() = true + +function __init__() + global libzstd_path = string(libzstd.path) + global zstd_path = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstd_exe) + global zstdmt_path = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_exe) + global artifact_dir = dirname(Sys.BINDIR) +end + +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end + +end # module Zstd_jll diff --git a/stdlib/Zstd_jll/test/runtests.jl b/stdlib/Zstd_jll/test/runtests.jl new file mode 100644 index 0000000000000..5cfa2a1375c73 --- /dev/null +++ b/stdlib/Zstd_jll/test/runtests.jl @@ -0,0 +1,7 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test, Zstd_jll + +@testset "Zstd_jll" begin + @test ccall((:ZSTD_versionNumber, libzstd), Cuint, ()) == 1_05_07 +end diff --git a/stdlib/dSFMT_jll/src/dSFMT_jll.jl b/stdlib/dSFMT_jll/src/dSFMT_jll.jl index b84bf0d8204ae..93fc26dc87a53 100644 --- a/stdlib/dSFMT_jll/src/dSFMT_jll.jl +++ b/stdlib/dSFMT_jll/src/dSFMT_jll.jl @@ -5,41 +5,41 @@ baremodule dSFMT_jll using Base, Libdl -const PATH_list = String[] -const LIBPATH_list = String[] - export libdSFMT # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libdSFMT_handle::Ptr{Cvoid} = C_NULL -libdSFMT_path::String = "" -if Sys.iswindows() - const libdSFMT = "libdSFMT.dll" -elseif Sys.isapple() - const libdSFMT = "@rpath/libdSFMT.dylib" -else - const libdSFMT = "libdSFMT.so" +libdSFMT_path::String = "" +const libdSFMT = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libdSFMT.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libdSFMT.dylib") + else + BundledLazyLibraryPath("libdSFMT.so") + end +) + +function eager_mode() + dlopen(libdSFMT) end +is_available() = true function __init__() - global libdSFMT_handle = dlopen(libdSFMT) - global libdSFMT_path = dlpath(libdSFMT_handle) + global libdSFMT_path = string(libdSFMT.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libdSFMT_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libdSFMT_path() = libdSFMT_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module dSFMT_jll diff --git a/stdlib/libLLVM_jll/Project.toml b/stdlib/libLLVM_jll/Project.toml index ca342518b6ba1..364e4338840eb 100644 --- a/stdlib/libLLVM_jll/Project.toml +++ b/stdlib/libLLVM_jll/Project.toml @@ -1,12 +1,18 @@ name = "libLLVM_jll" uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a" -version = "19.1.7+1" +version = "20.1.2+1" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +Zlib_jll = "83775a58-1f1d-513f-b197-d71354ab007a" +Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" [compat] +CompilerSupportLibraries_jll = "1.3.0" +Zlib_jll = "1" +Zstd_jll = "1.5.7" julia = "1.13" [extras] diff --git a/stdlib/libLLVM_jll/src/libLLVM_jll.jl b/stdlib/libLLVM_jll/src/libLLVM_jll.jl index be2acb34faa65..2edff186b13a1 100644 --- a/stdlib/libLLVM_jll/src/libLLVM_jll.jl +++ b/stdlib/libLLVM_jll/src/libLLVM_jll.jl @@ -3,43 +3,59 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/libLLVM_jll.jl baremodule libLLVM_jll -using Base, Libdl +using Base, Libdl, Zlib_jll, Zstd_jll -const PATH_list = String[] -const LIBPATH_list = String[] +if !Sys.isapple() + using CompilerSupportLibraries_jll +end export libLLVM # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libLLVM_handle::Ptr{Cvoid} = C_NULL -libLLVM_path::String = "" -if Sys.iswindows() - const libLLVM = "$(Base.libllvm_name).dll" -elseif Sys.isapple() - const libLLVM = "@rpath/libLLVM.dylib" -else - const libLLVM = "$(Base.libllvm_name).so" +libLLVM_path::String = "" +const libLLVM = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("$(Base.libllvm_name).dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libLLVM.dylib") + else + BundledLazyLibraryPath("$(Base.libllvm_name).so") + end, + dependencies = if Sys.isapple() + LazyLibrary[libz, libzstd] + elseif Sys.isfreebsd() + LazyLibrary[libz, libzstd, libgcc_s] + else + LazyLibrary[libz, libzstd, libstdcxx, libgcc_s] + end +) + +function eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + Zlib_jll.eager_mode() + # Zstd_jll.eager_mode() # Not lazy yet + dlopen(libLLVM) end +is_available() = true function __init__() - global libLLVM_handle = dlopen(libLLVM) - global libLLVM_path = dlpath(libLLVM_handle) + global libLLVM_path = string(libLLVM.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libLLVM_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libLLVM_path() = libLLVM_path +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end end # module libLLVM_jll diff --git a/stdlib/libLLVM_jll/test/runtests.jl b/stdlib/libLLVM_jll/test/runtests.jl index e04076f4145a5..8025c2cb2e693 100644 --- a/stdlib/libLLVM_jll/test/runtests.jl +++ b/stdlib/libLLVM_jll/test/runtests.jl @@ -4,5 +4,5 @@ using Test, Libdl, libLLVM_jll @testset "libLLVM_jll" begin # Try to find a symbol from the C API of libLLVM as a simple sanity check. - @test dlsym(libLLVM_jll.libLLVM_handle, :LLVMContextCreate; throw_error=false) !== nothing + @test dlsym(libLLVM_jll.libLLVM, :LLVMContextCreate; throw_error=false) !== nothing end diff --git a/stdlib/libblastrampoline_jll/Project.toml b/stdlib/libblastrampoline_jll/Project.toml index d1dde4c6074a7..ea4f645dc153c 100644 --- a/stdlib/libblastrampoline_jll/Project.toml +++ b/stdlib/libblastrampoline_jll/Project.toml @@ -1,13 +1,13 @@ name = "libblastrampoline_jll" uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.12.0+0" +version = "5.13.1+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" [compat] -julia = "1.12" +julia = "1.13" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl b/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl index 6cdc1b4ac3ce6..a75e2e30db2fd 100644 --- a/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl +++ b/stdlib/libblastrampoline_jll/src/libblastrampoline_jll.jl @@ -13,8 +13,6 @@ const PATH_list = String[] const LIBPATH = Ref("") const LIBPATH_list = String[] artifact_dir::String = "" -libblastrampoline_path::String = "" - # Because LBT needs to have a weak-dependence on OpenBLAS (or any other BLAS) # we must manually construct a list of which modules and libraries we're going @@ -33,16 +31,19 @@ function add_dependency!(mod::Module, lib::LazyLibrary, on_load_callback::Functi push!(on_load_callbacks, on_load_callback) end -# NOTE: keep in sync with `Base.libblas_name` and `Base.liblapack_name`. -const _libblastrampoline_path = if Sys.iswindows() - BundledLazyLibraryPath("libblastrampoline-5.dll") -elseif Sys.isapple() - BundledLazyLibraryPath("libblastrampoline.5.dylib") -else - BundledLazyLibraryPath("libblastrampoline.so.5") -end -const libblastrampoline = LazyLibrary(_libblastrampoline_path, dependencies=[], - on_load_callback=libblastrampoline_on_load_callback) +libblastrampoline_path::String = "" +const libblastrampoline = LazyLibrary( + # NOTE: keep in sync with `Base.libblas_name` and `Base.liblapack_name`. + if Sys.iswindows() + BundledLazyLibraryPath("libblastrampoline-5.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libblastrampoline.5.dylib") + else + BundledLazyLibraryPath("libblastrampoline.so.5") + end, + dependencies = LazyLibrary[], + on_load_callback = libblastrampoline_on_load_callback +) function eager_mode() for mod in eager_mode_modules @@ -53,9 +54,15 @@ end is_available() = true function __init__() - global libblastrampoline_path = string(_libblastrampoline_path) + global libblastrampoline_path = string(libblastrampoline.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libblastrampoline_path) push!(LIBPATH_list, LIBPATH[]) end + +if Base.generating_output() + precompile(eager_mode, ()) + precompile(is_available, ()) +end + end # module libblastrampoline_jll diff --git a/stdlib/nghttp2_jll/Project.toml b/stdlib/nghttp2_jll/Project.toml index 4520427d7ca76..ea748ec6b6c81 100644 --- a/stdlib/nghttp2_jll/Project.toml +++ b/stdlib/nghttp2_jll/Project.toml @@ -3,10 +3,12 @@ uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" version = "1.65.0+0" [deps] -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +CompilerSupportLibraries_jll = "e66e0078-7015-5450-92f7-15fbd957f2ae" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [compat] +CompilerSupportLibraries_jll = "1.3.0" julia = "1.11" [extras] diff --git a/stdlib/nghttp2_jll/src/nghttp2_jll.jl b/stdlib/nghttp2_jll/src/nghttp2_jll.jl index 5057299614aa5..e0a1559a85628 100644 --- a/stdlib/nghttp2_jll/src/nghttp2_jll.jl +++ b/stdlib/nghttp2_jll/src/nghttp2_jll.jl @@ -3,42 +3,48 @@ ## dummy stub for https://github.com/JuliaBinaryWrappers/nghttp2_jll.jl baremodule nghttp2_jll using Base, Libdl - -const PATH_list = String[] -const LIBPATH_list = String[] +if Sys.iswindows() && Sys.WORD_SIZE == 32 + using CompilerSupportLibraries_jll +end export libnghttp2 # These get calculated in __init__() const PATH = Ref("") +const PATH_list = String[] const LIBPATH = Ref("") +const LIBPATH_list = String[] artifact_dir::String = "" -libnghttp2_handle::Ptr{Cvoid} = C_NULL -libnghttp2_path::String = "" -if Sys.iswindows() - const libnghttp2 = "libnghttp2-14.dll" -elseif Sys.isapple() - const libnghttp2 = "@rpath/libnghttp2.14.dylib" -else - const libnghttp2 = "libnghttp2.so.14" +libnghttp2_path::String = "" +const libnghttp2 = LazyLibrary( + if Sys.iswindows() + BundledLazyLibraryPath("libnghttp2-14.dll") + elseif Sys.isapple() + BundledLazyLibraryPath("libnghttp2.14.dylib") + else + BundledLazyLibraryPath("libnghttp2.so.14") + end, + dependencies = if Sys.iswindows() && Sys.WORD_SIZE == 32 + LazyLibrary[libgcc_s] + else + LazyLibrary[] + end +) + +function eager_mode() + @static if @isdefined CompilerSupportLibraries_jll + CompilerSupportLibraries_jll.eager_mode() + end + dlopen(libnghttp2) end +is_available() = true function __init__() - global libnghttp2_handle = dlopen(libnghttp2) - global libnghttp2_path = dlpath(libnghttp2_handle) + global libnghttp2_path = string(libnghttp2.path) global artifact_dir = dirname(Sys.BINDIR) LIBPATH[] = dirname(libnghttp2_path) push!(LIBPATH_list, LIBPATH[]) end -# JLLWrappers API compatibility shims. Note that not all of these will really make sense. -# For instance, `find_artifact_dir()` won't actually be the artifact directory, because -# there isn't one. It instead returns the overall Julia prefix. -is_available() = true -find_artifact_dir() = artifact_dir -dev_jll() = error("stdlib JLLs cannot be dev'ed") -best_wrapper = nothing -get_libnghttp2_path() = libnghttp2_path - end # module nghttp2_jll diff --git a/stdlib/p7zip_jll/src/p7zip_jll.jl b/stdlib/p7zip_jll/src/p7zip_jll.jl index a2a90a2450ea6..af461c6719632 100644 --- a/stdlib/p7zip_jll/src/p7zip_jll.jl +++ b/stdlib/p7zip_jll/src/p7zip_jll.jl @@ -4,14 +4,10 @@ baremodule p7zip_jll using Base -const PATH_list = String[] -const LIBPATH_list = String[] - export p7zip # These get calculated in __init__() const PATH = Ref("") -const LIBPATH = Ref("") artifact_dir::String = "" p7zip_path::String = "" if Sys.iswindows() @@ -21,71 +17,44 @@ else end if Sys.iswindows() - const LIBPATH_env = "PATH" - const LIBPATH_default = "" const pathsep = ';' elseif Sys.isapple() - const LIBPATH_env = "DYLD_FALLBACK_LIBRARY_PATH" - const LIBPATH_default = "~/lib:/usr/local/lib:/lib:/usr/lib" const pathsep = ':' else - const LIBPATH_env = "LD_LIBRARY_PATH" - const LIBPATH_default = "" const pathsep = ':' end -function adjust_ENV!(env::Dict{keytype(Base.EnvDict),valtype(Base.EnvDict)}, PATH::String, LIBPATH::String, adjust_PATH::Bool, adjust_LIBPATH::Bool) - if adjust_LIBPATH - LIBPATH_base = get(env, LIBPATH_env, expanduser(LIBPATH_default)) - if !isempty(LIBPATH_base) - env[LIBPATH_env] = string(LIBPATH, pathsep, LIBPATH_base) - else - env[LIBPATH_env] = LIBPATH - end - end - if adjust_PATH && (LIBPATH_env != "PATH" || !adjust_LIBPATH) - if adjust_PATH - if !isempty(get(env, "PATH", "")) - env["PATH"] = string(PATH, pathsep, env["PATH"]) - else - env["PATH"] = PATH - end - end - end - return env +function adjust_ENV() + addPATH = PATH[] + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? addPATH : "$addPATH$pathsep$oldPATH" + return ("PATH"=>newPATH,) end -function p7zip(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) - env = adjust_ENV!(copy(ENV), PATH[], LIBPATH[], adjust_PATH, adjust_LIBPATH) - withenv(env...) do - return f(p7zip_path) +function p7zip(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + return f(p7zip()) end end -function p7zip(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) - env = adjust_ENV!(copy(ENV), PATH[], LIBPATH[], adjust_PATH, adjust_LIBPATH) - return Cmd(Cmd([p7zip_path]); env) -end +# the 7z.exe we ship has no dependencies, so it needs no PATH adjustment +p7zip(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) = `$p7zip_path` function init_p7zip_path() # Prefer our own bundled p7zip, but if we don't have one, pick it up off of the PATH - # If this is an in-tree build, `7z` will live in `bindir`. Otherwise, it'll be in `private_libexecdir` - for bundled_p7zip_path in (joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, p7zip_exe), - joinpath(Sys.BINDIR, p7zip_exe)) - if isfile(bundled_p7zip_path) - global p7zip_path = abspath(bundled_p7zip_path) - return - end + # Our `7z` lives in `private_libexecdir` + bundled_p7zip_path = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, p7zip_exe) + if isfile(bundled_p7zip_path) + global p7zip_path = abspath(bundled_p7zip_path) + else + global p7zip_path = something(Sys.which(p7zip_exe), p7zip_exe) end - global p7zip_path = something(Sys.which(p7zip_exe), p7zip_exe) end function __init__() global artifact_dir = dirname(Sys.BINDIR) init_p7zip_path() - PATH[] = dirname(p7zip_path) - push!(PATH_list, PATH[]) - append!(LIBPATH_list, [joinpath(Sys.BINDIR, Base.LIBDIR, "julia"), joinpath(Sys.BINDIR, Base.LIBDIR)]) - LIBPATH[] = join(LIBPATH_list, pathsep) + PATH[] = path = dirname(p7zip_path) + nothing end # JLLWrappers API compatibility shims. Note that not all of these will really make sense. diff --git a/stdlib/stdlib.mk b/stdlib/stdlib.mk index 006b7a276a3b3..3184ac9c3305f 100644 --- a/stdlib/stdlib.mk +++ b/stdlib/stdlib.mk @@ -9,7 +9,7 @@ INDEPENDENT_STDLIBS := \ SparseArrays Statistics StyledStrings SuiteSparse_jll Tar Test TOML Unicode UUIDs \ dSFMT_jll GMP_jll libLLVM_jll LLD_jll LLVMLibUnwind_jll LibUnwind_jll LibUV_jll \ LibCURL_jll LibSSH2_jll LibGit2_jll nghttp2_jll MozillaCACerts_jll \ - MPFR_jll OpenLibm_jll OpenSSL_jll PCRE2_jll p7zip_jll Zlib_jll + MPFR_jll OpenLibm_jll OpenSSL_jll PCRE2_jll p7zip_jll Zlib_jll Zstd_jll STDLIBS := $(STDLIBS_WITHIN_SYSIMG) $(INDEPENDENT_STDLIBS) VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) diff --git a/sysimage.mk b/sysimage.mk index 550373bfba588..e98dceb6a0128 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -11,6 +11,8 @@ sysimg-ji: $(build_private_libdir)/sysbase.ji sysimg-bc: $(build_private_libdir)/sys-bc.a sysimg-release: $(build_private_libdir)/sys.$(SHLIB_EXT) sysimg-debug: $(build_private_libdir)/sys-debug.$(SHLIB_EXT) +sysbase-release: $(build_private_libdir)/sysbase.$(SHLIB_EXT) +sysbase-debug: $(build_private_libdir)/sysbase-debug.$(SHLIB_EXT) VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION) @@ -18,7 +20,7 @@ $(build_private_libdir)/%.$(SHLIB_EXT): $(build_private_libdir)/%-o.a @$(call PRINT_LINK, $(CXX) $(LDFLAGS) -shared $(fPIC) -L$(build_private_libdir) -L$(build_libdir) -L$(build_shlibdir) -o $@ \ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) \ $(if $(findstring -debug,$(notdir $@)),-ljulia-internal-debug -ljulia-debug,-ljulia-internal -ljulia) \ - $$([ $(OS) = WINNT ] && echo '' $(LIBM) -lssp --disable-auto-import --disable-runtime-pseudo-reloc)) + $$([ $(OS) = WINNT ] && echo '' $(LIBM) -lssp -Wl,--disable-auto-import -Wl,--disable-runtime-pseudo-reloc)) @$(INSTALL_NAME_CMD)$(notdir $@) $@ @$(DSYMUTIL) $@ @@ -31,31 +33,48 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \ base/abstractset.jl \ base/iddict.jl \ base/idset.jl \ + base/anyall.jl \ base/array.jl \ + base/baseext.jl \ base/bitarray.jl \ base/bitset.jl \ base/bool.jl \ + base/c.jl \ + base/checked.jl \ + base/cmem.jl \ + base/coreio.jl \ + base/coreir.jl \ base/ctypes.jl \ base/error.jl \ base/essentials.jl \ base/expr.jl \ base/exports.jl \ + base/flfrontend.jl \ + base/float.jl \ + base/gcutils.jl \ base/generator.jl \ + base/genericmemory.jl \ base/int.jl \ base/indices.jl \ base/iterators.jl \ base/invalidation.jl \ + base/module.jl \ base/namedtuple.jl \ + base/ntuple.jl \ base/number.jl \ base/operators.jl \ base/options.jl \ + base/ordering.jl \ base/pair.jl \ base/pointer.jl \ base/promotion.jl \ + base/public.jl \ base/range.jl \ + base/refvalue.jl \ + base/rounding.jl \ base/runtime_internals.jl \ + base/strings/lazy.jl \ base/traits.jl \ - base/refvalue.jl \ base/tuple.jl) COMPILER_SRCS += $(shell find $(JULIAHOME)/Compiler/src -name \*.jl -and -not -name verifytrim.jl -and -not -name show.jl) # sort these to remove duplicates @@ -70,7 +89,7 @@ RELDATADIR := $(call rel_path,$(JULIAHOME)/base,$(build_datarootdir))/ # <-- mak $(build_private_libdir)/basecompiler.ji: $(COMPILER_SRCS) @$(call PRINT_JULIA, cd $(JULIAHOME)/base && \ JULIA_NUM_THREADS=1 $(call spawn,$(JULIA_EXECUTABLE)) $(HEAPLIM) --output-ji $(call cygpath_w,$@).tmp \ - --startup-file=no --warn-overwrite=yes -g$(BOOTSTRAP_DEBUG_LEVEL) -O1 Base_compiler.jl --buildroot $(RELBUILDROOT) --dataroot $(RELDATADIR)) + --startup-file=no --warn-overwrite=yes --depwarn=error -g$(BOOTSTRAP_DEBUG_LEVEL) -O1 Base_compiler.jl --buildroot $(RELBUILDROOT) --dataroot $(RELDATADIR)) @mv $@.tmp $@ define base_builder @@ -79,7 +98,7 @@ $$(build_private_libdir)/basecompiler$1-o.a $$(build_private_libdir)/basecompile WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ JULIA_NUM_THREADS=1 \ $$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp \ - --startup-file=no --warn-overwrite=yes -g$$(BOOTSTRAP_DEBUG_LEVEL) Base_compiler.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR)) + --startup-file=no --warn-overwrite=yes --depwarn=error -g$$(BOOTSTRAP_DEBUG_LEVEL) Base_compiler.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR)) @mv $$@.tmp $$@ $$(build_private_libdir)/sysbase$1.ji: $$(build_private_libdir)/basecompiler$1.$$(SHLIB_EXT) $$(JULIAHOME)/VERSION $$(BASE_SRCS) $$(STDLIB_SRCS) @$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \ @@ -87,7 +106,7 @@ $$(build_private_libdir)/sysbase$1.ji: $$(build_private_libdir)/basecompiler$1.$ WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ JULIA_NUM_THREADS=1 \ $$(call spawn, $$(JULIA_EXECUTABLE)) -g1 $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-ji $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \ - --startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) sysimg.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR); then \ + --startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) sysimg.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR); then \ echo '*** This error might be fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \ false; \ fi ) @@ -102,12 +121,13 @@ $$(build_private_libdir)/sysbase$1-o.a $$(build_private_libdir)/sysbase$1-bc.a : WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ JULIA_NUM_THREADS=1 \ $$(call spawn, $$(JULIA_EXECUTABLE)) -g1 $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \ - --startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) sysimg.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR); then \ + --startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) sysimg.jl --buildroot $$(RELBUILDROOT) --dataroot $$(RELDATADIR); then \ echo '*** This error might be fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \ false; \ fi ) @mv $$@.tmp $$@ -$$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_private_libdir)/sysbase$1.$$(SHLIB_EXT) $$(JULIAHOME)/contrib/generate_precompile.jl +build_sysbase_$1 := $$(or $$(CROSS_BOOTSTRAP_SYSBASE),$$(build_private_libdir)/sysbase$1.$$(SHLIB_EXT)) +$$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_sysbase_$1) $$(JULIAHOME)/contrib/generate_precompile.jl @$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \ if ! JULIA_BINDIR=$$(call cygpath_w,$(build_bindir)) \ WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ @@ -116,7 +136,8 @@ $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(buil JULIA_DEPOT_PATH=':' \ JULIA_NUM_THREADS=1 \ $$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \ - --startup-file=no --warn-overwrite=yes --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $(JULIA_PRECOMPILE); then \ + $(bootstrap_julia_flags) \ + --startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $(JULIA_PRECOMPILE); then \ echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \ false; \ fi ) diff --git a/test/Makefile b/test/Makefile index 9b151cd213274..8b9fc139488f4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -24,22 +24,27 @@ EMBEDDING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/embedding" "CC=$(CC GCEXT_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/gcext" "CC=$(CC)" -TRIMMING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(JULIAHOME)/usr/bin" "CC=$(CC)" +TRIMMING_ARGS := "JULIA=$(JULIA_EXECUTABLE)" "BIN=$(SRCDIR)/trimming" "CC=$(CC)" +TEST_JULIA_OPTIONS := --check-bounds=yes --startup-file=no --depwarn=error +TEST_SCRIPT_OPTIONS := --buildroot=$(call cygpath_w,$(BUILDROOT)) default: $(TESTS): @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) + +install-revise-deps: + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise --help-list install_revise_deps) $(addprefix revise-, $(TESTS)): revise-% : @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise $*) relocatedepot: @rm -rf $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) @mkdir $(SRCDIR)/relocatedepot @cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @@ -47,12 +52,12 @@ relocatedepot: @cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg4 $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) + $(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) $@) revise-relocatedepot: revise-% : @rm -rf $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) + $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise $*) @mkdir $(SRCDIR)/relocatedepot @cp -R $(build_datarootdir)/julia $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @@ -60,7 +65,7 @@ revise-relocatedepot: revise-% : @cp -R $(SRCDIR)/RelocationTestPkg3 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg4 $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) + $(call PRINT_JULIA, $(call spawn,RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) $(TEST_JULIA_OPTIONS) ./runtests.jl $(TEST_SCRIPT_OPTIONS) --revise $*) embedding: @$(MAKE) -C $(SRCDIR)/$@ check $(EMBEDDING_ARGS) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index d1f30eacafacc..bff6e28f8b19b 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -831,6 +831,9 @@ function test_cat(::Type{TestAbstractArray}) r = rand(Float32, 56, 56, 64, 1); f(r) = cat(r, r, dims=(3,)) @inferred f(r); + + #58866 - ensure proper dimension calculation for 0-dimension elements + @test [zeros(1, 0) zeros(1,0); zeros(0,0) zeros(0, 0)] == Matrix{Float64}(undef, 1, 0) end function test_ind2sub(::Type{TestAbstractArray}) @@ -910,6 +913,26 @@ include("generic_map_tests.jl") generic_map_tests(map, map!) @test map!(-, [1]) == [-1] +@testset "#30624" begin + ### unstructured + @test map!(+, ones(3), ones(3), ones(3), [1]) == [3, 1, 1] + @test map!(+, ones(3), [1], ones(3), ones(3)) == [3, 1, 1] + @test map!(+, [1], [1], [], []) == [1] + @test map!(+, [[1]], [1], [], []) == [[1]] + + # TODO: decide if input axes & lengths should be validated + # @test_throws BoundsError map!(+, ones(1), ones(2)) + # @test_throws BoundsError map!(+, ones(1), ones(2, 2)) + + @test map!(+, ones(3), view(ones(2, 3), 1:2, 2:3), ones(3)) == [2, 2, 2] + @test map!(+, ones(3), ones(2, 2), ones(3)) == [2, 2, 2] + + ### structured (all mapped arguments are <:AbstractArray equal ndims > 1) + @test map!(+, ones(4), ones(2, 2), ones(2, 2)) == [2, 2, 2, 2] + @test map!(+, ones(4), ones(2, 2), ones(1, 2)) == [2, 2, 1, 1] + # @test_throws BoundsError map!(+, ones(3), ones(2, 2), ones(2, 2)) +end + test_UInt_indexing(TestAbstractArray) test_13315(TestAbstractArray) test_checksquare() @@ -1723,6 +1746,9 @@ using Base: typed_hvncat @test ["A";;"B";;"C";;"D"] == ["A" "B" "C" "D"] @test ["A";"B";;"C";"D"] == ["A" "C"; "B" "D"] @test [["A";"B"];;"C";"D"] == ["A" "C"; "B" "D"] + + #58866 - ensure proper dimension calculation for 0-dimension elements + @test [zeros(1, 0) zeros(1,0);;; zeros(0,0) zeros(0, 0)] == Array{Float64, 3}(undef, 1, 0, 0) end @testset "stack" begin @@ -2171,6 +2197,22 @@ end @test one(Mat([1 2; 3 4])) == Mat([1 0; 0 1]) @test one(Mat([1 2; 3 4])) isa Mat + + @testset "SizedArray" begin + S = [1 2; 3 4] + A = SizedArrays.SizedArray{(2,2)}(S) + @test one(A) == one(typeof(A)) + @test oneunit(A) == oneunit(typeof(A)) + M = fill(A, 2, 2) + O = one(M) + for I in CartesianIndices(M) + if I[1] == I[2] + @test O[I] == one(S) + else + @test O[I] == zero(S) + end + end + end end @testset "copyto! with non-AbstractArray src" begin @@ -2227,3 +2269,50 @@ end @test_throws MethodError isreal(G) end end + +@testset "similar/reshape for AbstractOneTo" begin + A = [1,2] + @testset "reshape" begin + @test reshape(A, 2, SizedArrays.SOneTo(1)) == reshape(A, 2, 1) + @test reshape(A, Base.OneTo(2), SizedArrays.SOneTo(1)) == reshape(A, 2, 1) + @test reshape(A, SizedArrays.SOneTo(1), 2) == reshape(A, 1, 2) + @test reshape(A, SizedArrays.SOneTo(1), Base.OneTo(2)) == reshape(A, 1, 2) + end + @testset "similar" begin + b = similar(A, SizedArrays.SOneTo(1), big(2)) + @test b isa Array{Int, 2} + @test size(b) == (1, 2) + b = similar(A, SizedArrays.SOneTo(1), Base.OneTo(2)) + @test b isa Array{Int, 2} + @test size(b) == (1, 2) + b = similar(A, SizedArrays.SOneTo(1), 2, Base.OneTo(2)) + @test b isa Array{Int, 3} + @test size(b) == (1, 2, 2) + + @test_throws "no method matching $Int(::$Infinity)" similar(ones(2), OneToInf()) + end +end + +@testset "effect inference for `iterate` for `Array` and for `Memory`" begin + for El ∈ (Float32, Real, Any) + for Arr ∈ (Memory{El}, Array{El, 0}, Vector{El}, Matrix{El}, Array{El, 3}) + effects = Base.infer_effects(iterate, Tuple{Arr, Int}) + @test Base.Compiler.is_effect_free(effects) + @test Base.Compiler.is_terminates(effects) + @test Base.Compiler.is_notaskstate(effects) + @test Base.Compiler.is_noub(effects) + @test Base.Compiler.is_nonoverlayed(effects) + @test Base.Compiler.is_nortcall(effects) + end + end +end + +@testset "iterate for linear indexing" begin + A = [1 2; 3 4] + v = view(A, :) + @test sum(x for x in v) == sum(A) + v = view(A, 1:2:lastindex(A)) + @test sum(x for x in v) == sum(A[1:2:end]) + v2 = view(A, Base.IdentityUnitRange(1:length(A))) + @test sum(x for x in v2) == sum(A) +end diff --git a/test/ambiguous.jl b/test/ambiguous.jl index 0f29817e74dd5..df3ba89e3eb2a 100644 --- a/test/ambiguous.jl +++ b/test/ambiguous.jl @@ -19,7 +19,7 @@ include("testenv.jl") @test length(methods(ambig, (Int, Int))) == 1 @test length(methods(ambig, (UInt8, Int))) == 0 -@test length(Base.methods_including_ambiguous(ambig, (UInt8, Int))) == 3 +@test length(Base.methods_including_ambiguous(ambig, (UInt8, Int))) == 2 @test ambig("hi", "there") == 1 @test ambig(3.1, 3.2) == 5 @@ -42,7 +42,6 @@ let err = try errstr = String(take!(io)) @test occursin(" ambig(x, y::Integer)\n @ $curmod_str", errstr) @test occursin(" ambig(x::Integer, y)\n @ $curmod_str", errstr) - @test occursin(" ambig(x::Number, y)\n @ $curmod_str", errstr) @test occursin("Possible fix, define\n ambig(::Integer, ::Integer)", errstr) end @@ -160,7 +159,7 @@ ambig(::Signed, ::Int) = 3 ambig(::Int, ::Signed) = 4 end ambs = detect_ambiguities(Ambig48312) -@test length(ambs) == 4 +@test length(ambs) == 1 # only ambiguous over (Int, Int), which is 3 or 4 module UnboundAmbig55868 module B @@ -287,7 +286,7 @@ end @test isempty(methods(Ambig8.f, (Int,))) @test isempty(methods(Ambig8.g, (Int,))) for f in (Ambig8.f, Ambig8.g) - @test length(methods(f, (Integer,))) == 2 # 1 is also acceptable + @test length(methods(f, (Integer,))) == 2 # 3 is also acceptable @test length(methods(f, (Signed,))) == 1 # 2 is also acceptable @test length(Base.methods_including_ambiguous(f, (Signed,))) == 2 @test f(0x00) == 1 @@ -413,7 +412,7 @@ let has_ambig = Ref(Int32(0)) ms = Base._methods_by_ftype(Tuple{typeof(fnoambig), Any, Any}, nothing, 4, Base.get_world_counter(), false, Ref(typemin(UInt)), Ref(typemax(UInt)), has_ambig) @test ms isa Vector @test length(ms) == 4 - @test has_ambig[] == 0 + @test has_ambig[] == 1 # 0 is better, but expensive and probably unnecessary to compute end # issue #11407 @@ -459,15 +458,38 @@ struct U55231{P} end struct V55231{P} end U55231(::V55231) = nothing (::Type{T})(::V55231) where {T<:U55231} = nothing -@test length(methods(U55231)) == 2 +@test length(methods(U55231)) == 1 U55231(a, b) = nothing -@test length(methods(U55231)) == 3 +@test length(methods(U55231)) == 2 struct S55231{P} end struct T55231{P} end (::Type{T})(::T55231) where {T<:S55231} = nothing S55231(::T55231) = nothing -@test length(methods(S55231)) == 2 +@test length(methods(S55231)) == 1 S55231(a, b) = nothing -@test length(methods(S55231)) == 3 +@test length(methods(S55231)) == 2 + +ambig10() = 1 +ambig10(a::Vararg{Any}) = 2 +ambig10(a::Vararg{Union{Int32,Int64}}) = 6 +ambig10(a::Vararg{Matrix}) = 4 +ambig10(a::Vararg{Number}) = 7 +ambig10(a::Vararg{N}) where {N<:Number} = 5 +let ambig = Ref{Int32}(0) + ms = Base._methods_by_ftype(Tuple{typeof(ambig10), Vararg}, nothing, -1, Base.get_world_counter(), false, Ref{UInt}(typemin(UInt)), Ref{UInt}(typemax(UInt)), ambig) + @test ms isa Vector + @test length(ms) == 6 + @test_broken ambig[] == 0 +end +let ambig = Ref{Int32}(0) + ms = Base._methods_by_ftype(Tuple{typeof(ambig10), Vararg{Number}}, nothing, -1, Base.get_world_counter(), false, Ref{UInt}(typemin(UInt)), Ref{UInt}(typemax(UInt)), ambig) + @test ms isa Vector + @test length(ms) == 4 + @test_broken ambig[] == 0 + @test ms[1].method === which(ambig10, ()) + @test ms[2].method === which(ambig10, (Vararg{Union{Int32, Int64}},)) + @test ms[3].method === which(ambig10, Tuple{Vararg{N}} where N<:Number,) + @test ms[4].method === which(ambig10, (Vararg{Number},)) +end nothing diff --git a/test/arrayops.jl b/test/arrayops.jl index b2da3eac6386b..84b9c8e7b2f6a 100644 --- a/test/arrayops.jl +++ b/test/arrayops.jl @@ -100,6 +100,18 @@ using Dates @test Array{eltype(a)}(a) !== a @test Vector(a) !== a end +@testset "effect inference for `reshape` for `Array`" begin + for Arr ∈ (Array{<:Any, 0}, Vector, Matrix, Array{<:Any, 3}) + for Shape ∈ (Tuple{Int}, Tuple{Int, Int}) + effects = Base.infer_effects(reshape, Tuple{Arr{Float32}, Shape}) + @test Base.Compiler.is_effect_free(effects) + @test Base.Compiler.is_terminates(effects) + @test Base.Compiler.is_notaskstate(effects) + @test Base.Compiler.is_noub(effects) + @test Base.Compiler.is_nortcall(effects) + end + end +end @testset "reshaping SubArrays" begin a = Array(reshape(1:5, 1, 5)) @testset "linearfast" begin @@ -1761,6 +1773,12 @@ end end end +@testset "reverse zero dims" begin + a = fill(3) + @test a == reverse(a) + @test a === reverse!(a) +end + @testset "isdiag, istril, istriu" begin # Scalar @test isdiag(3) @@ -2197,7 +2215,7 @@ end # All we really care about is that we have an optimized # implementation, but the seed is a useful way to check that. -@test hash(CartesianIndex()) == Base.IteratorsMD.cartindexhash_seed +@test hash(CartesianIndex()) == Base.IteratorsMD.cartindexhash_seed ⊻ Base.HASH_SEED @test hash(CartesianIndex(1, 2)) != hash((1, 2)) @testset "itr, iterate" begin @@ -2384,7 +2402,7 @@ end M = [1 2 3; 4 5 6; 7 8 9] @test eachrow(M) == eachslice(M, dims = 1) == [[1, 2, 3], [4, 5, 6], [7, 8, 9]] @test eachcol(M) == eachslice(M, dims = 2) == [[1, 4, 7], [2, 5, 8], [3, 6, 9]] - @test_throws DimensionMismatch eachslice(M, dims = 4) + @test eachslice(M, dims = 4) == [[1 2 3; 4 5 6; 7 8 9;;;]] SR = @inferred eachrow(M) @test SR[2] isa eltype(SR) @@ -2449,6 +2467,32 @@ end @test_throws BoundsError A[2,3] = [4,5] @test_throws BoundsError A[2,3] .= [4,5] end + + @testset "trailing dimensions" begin + v = collect(1:3) + + S2 = eachslice(v; dims = 2, drop=true) + @test S2 isa AbstractSlices{<:AbstractVector, 1} + @test size(S2) == (1,) + @test S2[1] == v + + S2K = eachslice(v; dims = 2, drop=false) + @test S2K isa AbstractSlices{<:AbstractVector, 2} + @test size(S2K) == (1,1) + @test S2K[1,1] == v + + M = reshape(1:6, 2, 3) + + S13 = eachslice(M; dims = (1,3)) + @test size(S13) == (2,1) + @test S13[2,1] == M[2,:,1] + + S13K = eachslice(M; dims = (1,3), drop=false) + @test size(S13K) == (2,1,1) + @test S13K[1,1,1] == M[1,:] + @test S13K[2,1,1] == M[2,:] + end + end ### diff --git a/test/atomics.jl b/test/atomics.jl index 7e9f29c23ca10..3572824741459 100644 --- a/test/atomics.jl +++ b/test/atomics.jl @@ -111,6 +111,7 @@ Base.show(io::IO, x::Int24) = print(io, "Int24(", Core.Intrinsics.zext_int(Int, ## Fields @noinline function _test_field_operators(r) + GC.gc(false) r = r[] TT = fieldtype(typeof(r), :x) T = typeof(getfield(r, :x)) @@ -147,6 +148,7 @@ test_field_operators(ARefxy{Float64}(123_10, 123_20)) @noinline function _test_field_orderings(r, x, y) @nospecialize x y + GC.gc(false) r = r[] TT = fieldtype(typeof(r), :x) @@ -328,8 +330,9 @@ test_field_orderings(ARefxy{Any}(true, false), true, false) test_field_orderings(ARefxy{Union{Nothing,Missing}}(nothing, missing), nothing, missing) test_field_orderings(ARefxy{Union{Nothing,Int}}(nothing, 123_1), nothing, 123_1) test_field_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) -test_field_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) +test_field_orderings(Complex{Real}(10.5, 30.5), Complex{Real}(20.5, 40.5)) test_field_orderings(Complex{Rational{Integer}}(10, 30), Complex{Rational{Integer}}(20, 40)) +test_field_orderings(Pair{NTuple{3,Float64},NTuple{3,Real}}((10.5,11.5,12.5), (30.5,40.5,50.5)), Pair{NTuple{3,Float64},NTuple{3,Real}}((110.5,111.5,112.5), (130.5,140.5,150.5))) test_field_orderings(10.0, 20.0) test_field_orderings(NaN, Inf) @@ -705,7 +708,7 @@ test_global_orderings(Any, true, false) test_global_orderings(Union{Nothing,Missing}, nothing, missing) test_global_orderings(Union{Nothing,Int}, nothing, 123_1) test_global_orderings(Complex{Int128}, Complex{Int128}(10, 30), Complex{Int128}(20, 40)) -test_global_orderings(Complex{Real}, Complex{Real}(10, 30), Complex{Real}(20, 40)) +test_global_orderings(Complex{Real}, Complex{Real}(10.5, 30.5), Complex{Real}(20.5, 40.5)) test_global_orderings(Float64, 10.0, 20.0) test_global_orderings(Float64, NaN, Inf) @@ -1024,15 +1027,17 @@ test_memory_operators(Float64) end @noinline function test_memory_orderings(T::Type, x, y) @nospecialize - xr = GenericMemoryRef(AtomicMemory{T}(undef, 1)) - memoryrefset!(xr, x, :unordered, true) # @atomic xr[] = x - yr = GenericMemoryRef(Memory{T}(undef, 1)) + xr = GenericMemoryRef(AtomicMemory{T}(undef, 2), 2) + memoryrefset!(xr, x, :unordered, true) # @atomic xr[2] = x + yr = GenericMemoryRef(Memory{T}(undef, 2), 2) yr[] = y + GC.gc(false) _test_memory_orderings(Ref(xr), Ref(yr), x, y) - xr = GenericMemoryRef(AtomicMemory{T}(undef, 1)) - memoryrefset!(xr, x, :unordered, true) # @atomic xr[] = x - yr = GenericMemoryRef(Memory{T}(undef, 1)) + xr = GenericMemoryRef(AtomicMemory{T}(undef, 2), 2) + memoryrefset!(xr, x, :unordered, true) # @atomic xr[2] = x + yr = GenericMemoryRef(Memory{T}(undef, 2), 2) yr[] = y + GC.gc(false) _test_memory_orderings(Ref{Any}(xr), Ref{Any}(yr), x, y) nothing end @@ -1047,7 +1052,8 @@ test_memory_orderings(Any, true, false) test_memory_orderings(Union{Nothing,Missing}, nothing, missing) test_memory_orderings(Union{Nothing,Int}, nothing, 123_1) test_memory_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) -test_memory_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) +test_memory_orderings(Complex{Real}(10.5, 30.5), Complex{Real}(20.5, 40.5)) +test_memory_orderings(Pair{NTuple{3,Float64},NTuple{3,Real}}((10.5,11.5,12.5), (30.5,40.5,50.5)), Pair{NTuple{3,Float64},NTuple{3,Real}}((110.5,111.5,112.5), (130.5,140.5,150.5))) test_memory_orderings(10.0, 20.0) test_memory_orderings(NaN, Inf) @@ -1099,3 +1105,14 @@ test_once_undef(Any) test_once_undef(Union{Nothing,Integer}) test_once_undef(UndefComplex{Any}) test_once_undef(UndefComplex{UndefComplex{Any}}) + +mutable struct Atomic57190 + @atomic x::Int +end + + +function add_one57190!() + @atomic (Atomic57190(0).x) += 1 +end + +@test add_one57190!() == 1 diff --git a/test/boundscheck_exec.jl b/test/boundscheck_exec.jl index af57c8ee6c156..2822176929f4a 100644 --- a/test/boundscheck_exec.jl +++ b/test/boundscheck_exec.jl @@ -239,6 +239,13 @@ if bc_opt != bc_off @test_throws BoundsError BadVector20469([1,2,3])[:] end +# Accumulate: do not set inbounds context for user-supplied functions +if bc_opt != bc_off + Base.@propagate_inbounds op58200(a, b) = (1, 2)[a] + (1, 2)[b] + @test_throws BoundsError accumulate(op58200, 1:10) + @test_throws BoundsError Base.accumulate_pairwise(op58200, 1:10) +end + # Ensure iteration over arrays is vectorizable function g27079(X) r = 0 @@ -343,7 +350,23 @@ if bc_opt == bc_default m1 === m2 end no_alias_prove(1) - @test_broken (@allocated no_alias_prove(5)) == 0 + @test (@allocated no_alias_prove(5)) == 0 +end + +@testset "automatic boundscheck elision for iteration on some important types" begin + if bc_opt != bc_on + @test !contains(sprint(code_llvm, iterate, (Memory{UInt8}, Int)), "unreachable") + + @test !contains(sprint(code_llvm, iterate, (Vector{UInt8}, Int)), "unreachable") + @test !contains(sprint(code_llvm, iterate, (Matrix{UInt8}, Int)), "unreachable") + @test !contains(sprint(code_llvm, iterate, (Array{UInt8,3}, Int)), "unreachable") + + @test !contains(sprint(code_llvm, iterate, (SubArray{Float64, 1, Vector{Float64}, Tuple{Base.Slice{Base.OneTo{Int64}}}, true}, Int)), "unreachable") + @test !contains(sprint(code_llvm, iterate, (SubArray{Float64, 2, Matrix{Float64}, Tuple{Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}}}, true}, Int)), "unreachable") + @test !contains(sprint(code_llvm, iterate, (SubArray{Float64, 2, Matrix{Float64}, Tuple{Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64}}, true}, Int)), "unreachable") + + @test !contains(sprint(code_llvm, iterate, (Base.CodeUnits{UInt8,String}, Int)), "unreachable") + end end end diff --git a/test/broadcast.jl b/test/broadcast.jl index 0f5bdf7a40bb1..a751d9d381ce6 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -53,6 +53,7 @@ ci(x) = CartesianIndex(x) @test @inferred(newindex(ci((2,2)), (true,), (-1,))) == 2 @test @inferred(newindex(ci((2,2)), (false,), (-1,))) == -1 @test @inferred(newindex(ci((2,2)), (), ())) == ci(()) +@test @inferred(newindex(ci((2,)), (true, false, false), (-1, -1, -1))) == ci((2, -1)) end @@ -853,29 +854,34 @@ let @test Dict(c .=> d) == Dict("foo" => 1, "bar" => 2) end -# Broadcasted iterable/indexable APIs -let - bc = Broadcast.instantiate(Broadcast.broadcasted(+, zeros(5), 5)) - @test IndexStyle(bc) == IndexLinear() - @test eachindex(bc) === Base.OneTo(5) - @test length(bc) === 5 - @test ndims(bc) === 1 - @test ndims(typeof(bc)) === 1 - @test bc[1] === bc[CartesianIndex((1,))] === 5.0 - @test copy(bc) == [v for v in bc] == collect(bc) - @test eltype(copy(bc)) == eltype([v for v in bc]) == eltype(collect(bc)) - @test ndims(copy(bc)) == ndims([v for v in bc]) == ndims(collect(bc)) == ndims(bc) - - bc = Broadcast.instantiate(Broadcast.broadcasted(+, zeros(5), 5*ones(1, 4))) - @test IndexStyle(bc) == IndexCartesian() - @test eachindex(bc) === CartesianIndices((Base.OneTo(5), Base.OneTo(4))) - @test length(bc) === 20 - @test ndims(bc) === 2 - @test ndims(typeof(bc)) === 2 - @test bc[1,1] == bc[CartesianIndex((1,1))] === 5.0 - @test copy(bc) == [v for v in bc] == collect(bc) - @test eltype(copy(bc)) == eltype([v for v in bc]) == eltype(collect(bc)) - @test ndims(copy(bc)) == ndims([v for v in bc]) == ndims(collect(bc)) == ndims(bc) +isdefined(Main, :OffsetArrays) || @eval Main include("testhelpers/OffsetArrays.jl") +using .Main.OffsetArrays +@testset "Broadcasted iterable/indexable APIs" begin + for f in (identity, x -> OffsetArray(x, ntuple(Returns(-1), ndims(x)))) + a = f(zeros(5)) + bc = Broadcast.instantiate(Broadcast.broadcasted(+, a, 5)) + @test IndexStyle(bc) == IndexLinear() + @test eachindex(bc) === eachindex(a) + @test length(bc) === 5 + @test ndims(bc) === 1 + @test ndims(typeof(bc)) === 1 + @test bc[1] === bc[CartesianIndex((1,))] === 5.0 + @test copy(bc) == [v for v in bc] == collect(bc) + @test eltype(copy(bc)) == eltype([v for v in bc]) == eltype(collect(bc)) + @test ndims(copy(bc)) == ndims([v for v in bc]) == ndims(collect(bc)) == ndims(bc) + + b = f(5*ones(1, 4)) + bc = Broadcast.instantiate(Broadcast.broadcasted(+, a, b)) + @test IndexStyle(bc) == IndexCartesian() + @test eachindex(bc) === CartesianIndices((axes(a, 1), axes(b, 2))) + @test length(bc) === 20 + @test ndims(bc) === 2 + @test ndims(typeof(bc)) === 2 + @test bc[1,1] == bc[CartesianIndex((1,1))] === 5.0 + @test copy(bc) == [v for v in bc] == collect(bc) + @test eltype(copy(bc)) == eltype([v for v in bc]) == eltype(collect(bc)) + @test ndims(copy(bc)) == ndims([v for v in bc]) == ndims(collect(bc)) == ndims(bc) + end struct MyFill{T,N} <: AbstractArray{T,N} val :: T @@ -1118,24 +1124,14 @@ end end @testset "inplace broadcast with trailing singleton dims" begin - for (a, b, c) in (([1, 2], reshape([3 4], :, 1), reshape([5, 6], :, 1, 1)), + for (a_, b_, c_) in (([1, 2], reshape([3 4], :, 1), reshape([5, 6], :, 1, 1)), ([1 2; 3 4], reshape([5 6; 7 8], 2, 2, 1), reshape([9 10; 11 12], 2, 2, 1, 1))) - - a_ = copy(a) - a_ .= b - @test a_ == dropdims(b, dims=(findall(==(1), size(b))...,)) - - a_ = copy(a) - a_ .= b - @test a_ == dropdims(b, dims=(findall(==(1), size(b))...,)) - - a_ = copy(a) - a_ .= b .+ c - @test a_ == dropdims(b .+ c, dims=(findall(==(1), size(c))...,)) - - a_ = copy(a) - a_ .*= c - @test a_ == dropdims(a .* c, dims=(findall(==(1), size(c))...,)) + for fun in (x -> OffsetArray(x, ntuple(Returns(1), ndims(x))), identity) + a, b, c = fun(a_), fun(b_), fun(c_) + @test (deepcopy(a) .= b) == dropdims(b, dims=(findall(==(1), size(b))...,)) + @test (deepcopy(a) .= b .+ c) == dropdims(b .+ c, dims=(findall(==(1), size(c))...,)) + @test (deepcopy(a) .*= c) == dropdims(a .* c, dims=(findall(==(1), size(c))...,)) + end end end diff --git a/test/buildkitetestjson.jl b/test/buildkitetestjson.jl index e9f969483c2aa..f0771e9c005d0 100644 --- a/test/buildkitetestjson.jl +++ b/test/buildkitetestjson.jl @@ -69,6 +69,8 @@ function result_dict(testset::Test.DefaultTestSet, prefix::String="") "arch" => string(Sys.ARCH), "julia_version" => string(VERSION), "testset" => testset.description, + "job_group" => get(ENV, "BUILDKITE_GROUP_LABEL", "unknown"), + "job_label" => get(ENV, "BUILDKITE_LABEL", "unknown"), ), # note we drop some of this from common_data before merging into individual results "history" => if !isnothing(testset.time_end) @@ -97,10 +99,9 @@ function generalize_file_paths(path::AbstractString) string(bindir_dir, pathsep) => "" ) @static if Sys.iswindows() - return replace(path, "\\" => "/") - else - return path + path = replace(path, "\\" => "/") end + return replace(path, "share/julia/" => "") end end @@ -143,6 +144,9 @@ const TEST_TYPE_MAP = Dict( :test_throws_nothing => "@test_throws" ) function get_test_call_str(result) + if result.test_type === :nontest_error + return "Got exception outside of a @test" + end prefix = get(TEST_TYPE_MAP, result.test_type, nothing) prefix === nothing && return error("Unknown test type $(repr(result.test_type))") return prefix == "@test_throws" ? "@test_throws $(result.data) $(result.orig_expr)" : "$prefix $(result.orig_expr)" @@ -258,9 +262,16 @@ function serialize_testset_result_file(dir::String, testset::Test.DefaultTestSet end # deserilalizes the results files and writes them to collated JSON files of 5000 max results -function write_testset_json_files(dir::String) +function write_testset_json_files(dir::String, testset::Test.DefaultTestSet) data = Dict{String,Any}[] read_files = String[] + # Set one result to represent the overall duration, given results have no duration + overall_ts = result_dict(testset) + # don't set location or file name for this result. They aren't required by BK + overall_ts["result"] = "unknown" + overall_ts["name"] = replace(get(ENV, "BUILDKITE_LABEL", "job label not found"), r":\w+:\s*" => "") + push!(data, overall_ts) + # Load all the serialized results files for res_dat in filter!(x -> occursin(r"^results.*\.dat$", x), readdir(dir)) res_file = joinpath(dir, res_dat) append!(data, Serialization.deserialize(res_file)) diff --git a/test/cartesian.jl b/test/cartesian.jl index 5db07b5316407..6097d4ca3770a 100644 --- a/test/cartesian.jl +++ b/test/cartesian.jl @@ -582,3 +582,48 @@ end c = CartesianIndex(3, 3) @test sprint(show, c) == "CartesianIndex(3, 3)" end + +@testset "CartesianIndex indexing with begin/end" begin + I = CartesianIndex(3,4) + @test I[begin] == I[1] + @test I[end] == I[2] +end + +@testset "in for a CartesianIndex StepRangeLen" begin + @testset for l in [0, 1, 4], r in Any[ + StepRangeLen(CartesianIndex(), CartesianIndex(), l), + StepRangeLen(CartesianIndex(1), CartesianIndex(0), l), + StepRangeLen(CartesianIndex(1), CartesianIndex(1), l), + StepRangeLen(CartesianIndex(1), CartesianIndex(4), l), + StepRangeLen(CartesianIndex(1), CartesianIndex(-4), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(0, 0), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(0, 4), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(0, -4), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(4, 0), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(-4, 0), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(4, 2), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(-4, 2), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(4, -2), l), + StepRangeLen(CartesianIndex(-1, 2), CartesianIndex(-4, -2), l), + StepRangeLen(CartesianIndex(-1, 2, 0), CartesianIndex(0, 0, 0), l), + StepRangeLen(CartesianIndex(-1, 2, 0), CartesianIndex(0, 0, -2), l), + ] + + if length(r) == 0 + @test !(first(r) in r) + @test !(last(r) in r) + end + for x in r + @test x in r + if step(r) != oneunit(x) + @test !((x + oneunit(x)) in r) + end + end + @test !(CartesianIndex(ntuple(x->0, ndims(r))) in r) + @test !(CartesianIndex(ntuple(x->typemax(Int), ndims(r))) in r) + @test !(CartesianIndex(ntuple(x->typemin(Int), ndims(r))) in r) + if ndims(r) > 1 + @test !(CartesianIndex(ntuple(x->0, ndims(r)-1)...) in r) + end + end +end diff --git a/test/ccall.jl b/test/ccall.jl index f193d16fc09e2..048d74af7c298 100644 --- a/test/ccall.jl +++ b/test/ccall.jl @@ -1979,3 +1979,11 @@ let llvm = sprint(code_llvm, gc_safe_ccall, ()) # check for the gc_safe store @test occursin("store atomic i8 2", llvm) end + +module Test57749 +using Test, Zstd_jll +const prefix = "Zstd version: " +const sym = :ZSTD_versionString +get_zstd_version() = prefix * unsafe_string(ccall((sym, libzstd), Cstring, ())) +@test startswith(get_zstd_version(), "Zstd") +end diff --git a/test/channels.jl b/test/channels.jl index f646b41cfa1a0..721eb478bd13a 100644 --- a/test/channels.jl +++ b/test/channels.jl @@ -463,15 +463,14 @@ end cb = first(async.cond.waitq) @test isopen(async) ccall(:uv_async_send, Cvoid, (Ptr{Cvoid},), async) - ccall(:uv_async_send, Cvoid, (Ptr{Cvoid},), async) - @test isempty(Base.Workqueue) Base.process_events() # schedule event Sys.iswindows() && Base.process_events() # schedule event (windows?) - @test length(Base.Workqueue) == 1 ccall(:uv_async_send, Cvoid, (Ptr{Cvoid},), async) @test tc[] == 0 yield() # consume event @test tc[] == 1 + ccall(:uv_async_send, Cvoid, (Ptr{Cvoid},), async) + Base.process_events() Sys.iswindows() && Base.process_events() # schedule event (windows?) yield() # consume event @test tc[] == 2 diff --git a/test/char.jl b/test/char.jl index 5523125529031..cd7997d4a37f5 100644 --- a/test/char.jl +++ b/test/char.jl @@ -290,6 +290,7 @@ Base.codepoint(c::ASCIIChar) = reinterpret(UInt8, c) @test !isempty(ASCIIChar('x')) @test ndims(ASCIIChar('x')) == 0 @test ndims(ASCIIChar) == 0 + @test Base.IteratorSize(ASCIIChar) === Base.HasShape{0}() @test firstindex(ASCIIChar('x')) == 1 @test lastindex(ASCIIChar('x')) == 1 @test eltype(ASCIIChar) == ASCIIChar diff --git a/test/choosetests.jl b/test/choosetests.jl index b07f5e8310ee6..17970313da01f 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -22,7 +22,7 @@ const TESTNAMES = [ "euler", "show", "client", "terminfo", "errorshow", "sets", "goto", "llvmcall", "llvmcall2", "ryu", "some", "meta", "stacktraces", "docs", "gc", - "misc", "threads", "stress", "binaryplatforms", "atexit", + "misc", "threads", "stress", "binaryplatforms","stdlib_dependencies", "atexit", "enums", "cmdlineargs", "int", "interpreter", "checked", "bitset", "floatfuncs", "precompile", "relocatedepot", "boundscheck", "error", "ambiguous", "cartesian", "osutils", @@ -100,6 +100,7 @@ function choosetests(choices = []) seed = rand(RandomDevice(), UInt128) ci_option_passed = false dryrun = false + buildroot = joinpath(@__DIR__, "..") for (i, t) in enumerate(choices) if t == "--skip" @@ -109,6 +110,8 @@ function choosetests(choices = []) exit_on_error = true elseif t == "--revise" use_revise = true + elseif startswith(t, "--buildroot=") + buildroot = t[(length("--buildroot=") + 1):end] elseif startswith(t, "--seed=") seed = parse(UInt128, t[(length("--seed=") + 1):end]) elseif t == "--ci" @@ -124,6 +127,7 @@ function choosetests(choices = []) --help-list : prints the options computed without running them --revise : load Revise --seed= : set the initial seed for all testgroups (parsed as a UInt128) + --buildroot= : set the build root directory (default: in-tree) --skip ... : skip test or collection tagged with TESTS: Can be special tokens, such as "all", "unicode", "stdlib", the names of stdlib \ @@ -261,5 +265,5 @@ function choosetests(choices = []) empty!(tests) end - return (; tests, net_on, exit_on_error, use_revise, seed) + return (; tests, net_on, exit_on_error, use_revise, buildroot, seed) end diff --git a/test/clangsa/GCPushPop.cpp b/test/clangsa/GCPushPop.cpp index 6736d3e181118..79cad28f4b9a5 100644 --- a/test/clangsa/GCPushPop.cpp +++ b/test/clangsa/GCPushPop.cpp @@ -8,15 +8,15 @@ void missingPop() { jl_value_t *x = NULL; JL_GC_PUSH1(&x); // expected-note{{GC frame changed here}} -} // expected-warning{{Non-popped GC frame present at end of function}} - // expected-note@-1{{Non-popped GC frame present at end of function}} +} // expected-warning@-1{{Non-popped GC frame present at end of function}} + // expected-note@-2{{Non-popped GC frame present at end of function}} void missingPop2() { jl_value_t **x; JL_GC_PUSHARGS(x, 2); // expected-note{{GC frame changed here}} -} // expected-warning{{Non-popped GC frame present at end of function}} - // expected-note@-1{{Non-popped GC frame present at end of function}} +} // expected-warning@-1{{Non-popped GC frame present at end of function}} + // expected-note@-2{{Non-popped GC frame present at end of function}} void superfluousPop() { JL_GC_POP(); // expected-warning{{JL_GC_POP without corresponding push}} diff --git a/test/clangsa/MissingRoots.c b/test/clangsa/MissingRoots.c index 0a0d5369eba44..84341f9410e1e 100644 --- a/test/clangsa/MissingRoots.c +++ b/test/clangsa/MissingRoots.c @@ -277,20 +277,6 @@ void nonconst_loads2() static inline void look_at_value2(jl_value_t *v) { look_at_value(v); } -void mtable(jl_value_t *f) { - look_at_value2((jl_value_t*)jl_gf_mtable(f)); - jl_value_t *val = NULL; - JL_GC_PUSH1(&val); - val = (jl_value_t*)jl_gf_mtable(f); - JL_GC_POP(); -} - -void mtable2(jl_value_t **v) { - jl_value_t *val = NULL; - JL_GC_PUSH1(&val); - val = (jl_value_t*)jl_gf_mtable(v[2]); - JL_GC_POP(); -} void tparam0(jl_value_t *atype) { look_at_value(jl_tparam0(atype)); diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 245b14e93769a..4bf8f62243838 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -130,9 +130,6 @@ end ("--startup-file=no", false), ("--startup-file=yes", true), - # ("--sysimage-native-code=no", false), # takes a lot longer (30s) - ("--sysimage-native-code=yes", true), - ("--pkgimages=yes", true), ("--pkgimages=no", false), ) @@ -243,6 +240,10 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` @test startswith(read(`$exename --help`, String), header) end + # Test to make sure that command line --help and --help-hidden do not return a description which is more than 100 characters wide + @test isempty(filter(x->length(x) > 100, readlines(`$exename -h`))) + @test isempty(filter(x->length(x) > 100, readlines(`$exename --help-hidden`))) + # ~ expansion in --project and JULIA_PROJECT if !Sys.iswindows() let expanded = abspath(expanduser("~/foo/Project.toml")) @@ -257,6 +258,16 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` @test expanded == readchomp(addenv(`$exename -e 'println(Base.active_project())'`, "JULIA_PROJECT" => "@foo", "HOME" => homedir())) end + # --project=@script handling + let expanded = abspath(joinpath(@__DIR__, "project", "ScriptProject")) + script = joinpath(expanded, "bin", "script.jl") + # Check running julia with --project=@script both within and outside the script directory + @testset "--@script from $name" for (name, dir) in [("project", expanded), ("outside", pwd())] + @test joinpath(expanded, "Project.toml") == readchomp(Cmd(`$exename --project=@script $script`; dir)) + @test joinpath(expanded, "SubProject", "Project.toml") == readchomp(Cmd(`$exename --project=@script/../SubProject $script`; dir)) + end + end + # handling of `@temp` in --project and JULIA_PROJECT @test tempdir() == readchomp(`$exename --project=@temp -e 'println(Base.active_project())'`)[1:lastindex(tempdir())] @test tempdir() == readchomp(addenv(`$exename -e 'println(Base.active_project())'`, "JULIA_PROJECT" => "@temp", "HOME" => homedir()))[1:lastindex(tempdir())] @@ -341,6 +352,7 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` # -t, --threads code = "print(Threads.threadpoolsize())" + code2 = "print(Threads.maxthreadid())" cpu_threads = ccall(:jl_effective_threads, Int32, ()) @test string(cpu_threads) == read(`$exename --threads auto -e $code`, String) == @@ -351,6 +363,11 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` withenv("JULIA_NUM_THREADS" => nt) do @test read(`$exename --threads=2 -e $code`, String) == read(`$exename -t 2 -e $code`, String) == "2" + if nt === nothing + @test read(`$exename -e $code2`, String) == "2" #default + interactive + elseif nt == "1" + @test read(`$exename -e $code2`, String) == "1" #if user asks for 1 give 1 + end end end # We want to test oversubscription, but on manycore machines, this can @@ -1011,6 +1028,14 @@ end @test v[2] == "" @test contains(v[3], "More than one command line CPU targets specified") end + + # Testing this more precisely would be very platform and build system dependent and brittle. + withenv("JULIA_CPU_TARGET" => "sysimage") do + v = readchomp(`$julia_path -E "Sys.sysimage_target()"`) + # Local builds will likely be "native" but CI shouldn't be. + invalid_results = Base.get_bool_env("CI", false) ? ("", "native", "sysimage") : ("", "sysimage",) + @test !in(v, invalid_results) + end end # Find the path of libjulia (or libjulia-debug, as the case may be) @@ -1076,22 +1101,12 @@ run(pipeline(devnull, `$(joinpath(Sys.BINDIR, Base.julia_exename())) --lisp`, de @test readchomperrors(`$(joinpath(Sys.BINDIR, Base.julia_exename())) -Cnative --lisp`) == (false, "", "ERROR: --lisp must be specified as the first argument") -# --sysimage-native-code={yes|no} -let exename = `$(Base.julia_cmd()) --startup-file=no` - @test readchomp(`$exename --sysimage-native-code=yes -E - "Bool(Base.JLOptions().use_sysimage_native_code)"`) == "true" - # TODO: Make this safe in the presence of two single-thread threadpools - # see https://github.com/JuliaLang/julia/issues/57198 - @test readchomp(`$exename --sysimage-native-code=no -t1,0 -E - "Bool(Base.JLOptions().use_sysimage_native_code)"`) == "false" -end - # backtrace contains line number info (esp. on windows #17179) -for precomp in ("yes", "no") - # TODO: Make this safe in the presence of two single-thread threadpools +let + # TODO: Make this safe in the presence of two single-thread threadpools with + # --sysimage-native-code=no, though that option is deprecated. # see https://github.com/JuliaLang/julia/issues/57198 - threads = precomp == "no" ? `-t1,0` : `` - succ, out, bt = readchomperrors(`$(Base.julia_cmd()) $threads --startup-file=no --sysimage-native-code=$precomp -E 'sqrt(-2)'`) + succ, out, bt = readchomperrors(`$(Base.julia_cmd()) --startup-file=no -E 'sqrt(-2)'`) @test !succ @test out == "" @test occursin(r"\.jl:(\d+)", bt) @@ -1327,3 +1342,16 @@ end timeout = 120 @test parse(Int,read(`$exename --timeout-for-safepoint-straggler=$timeout -E "Base.JLOptions().timeout_for_safepoint_straggler_s"`, String)) == timeout end + +@testset "--strip-metadata" begin + mktempdir() do dir + @test success(pipeline(`$(Base.julia_cmd()) --strip-metadata -t1,0 --output-o $(dir)/sys.o.a -e 0`, stderr=stderr, stdout=stdout)) + if isfile(joinpath(dir, "sys.o.a")) + Base.Linking.link_image(joinpath(dir, "sys.o.a"), joinpath(dir, "sys.so")) + @test readchomp(`$(Base.julia_cmd()) -t1,0 -J $(dir)/sys.so -E 'hasmethod(sort, (Vector{Int},), (:dims,))'`) == "true" + end + end +end + +# https://github.com/JuliaLang/julia/issues/58229 Recursion in jitlinking with inline=no +@test success(`$(Base.julia_cmd()) --inline=no -e 'Base.compilecache(Base.identify_package("Pkg"))'`) diff --git a/test/complex.jl b/test/complex.jl index e24e54febc815..ac867c6f0e5fb 100644 --- a/test/complex.jl +++ b/test/complex.jl @@ -932,6 +932,13 @@ end end end +@testset "eps" begin + @test eps(1.0+1.0im) === 3.1401849173675503e-16 + @test eps(Complex{Float64}) === eps(1.0+1.0im) + @test eps(Complex{Float32}) === 1.6858739f-7 + @test eps(Float32(1.0)+Float32(1.0)im) === eps(Complex{Float32}) +end + @testset "cis" begin @test cis(0.0+1.0im) ≈ 0.367879441171442321595523770161460867445811131031767834507836+0.0im @test cis(1.0+0.0im) ≈ 0.54030230586813971740093660744297660373231042061+0.84147098480789650665250232163029899962256306079im diff --git a/test/core.jl b/test/core.jl index 4f27c4896da9d..78d777c6ed84c 100644 --- a/test/core.jl +++ b/test/core.jl @@ -17,15 +17,17 @@ for (T, c) in ( (Core.CodeInstance, [:def, :owner, :rettype, :exctype, :rettype_const, :analysis_results, :time_infer_total, :time_infer_cache_saved, :time_infer_self]), (Core.Method, [#=:name, :module, :file, :line, :primary_world, :sig, :slot_syms, :external_mt, :nargs, :called, :nospecialize, :nkw, :isva, :is_for_opaque_closure, :constprop=#]), (Core.MethodInstance, [#=:def, :specTypes, :sparam_vals=#]), - (Core.MethodTable, [:module]), + (Core.MethodTable, [:cache, :module, :name]), + (Core.MethodCache, []), (Core.TypeMapEntry, [:sig, :simplesig, :guardsigs, :func, :isleafsig, :issimplesig, :va]), (Core.TypeMapLevel, []), - (Core.TypeName, [:name, :module, :names, :wrapper, :mt, :hash, :n_uninitialized, :flags]), + (Core.TypeName, [:name, :module, :names, :wrapper, :hash, :n_uninitialized, :flags]), (DataType, [:name, :super, :parameters, :instance, :hash]), (TypeVar, [:name, :ub, :lb]), (Core.Memory, [:length, :ptr]), (Core.GenericMemoryRef, [:mem, :ptr_or_offset]), (Task, [:metrics_enabled]), + (Core.BindingPartition, [:restriction, :kind]), ) @test Set((fieldname(T, i) for i in 1:fieldcount(T) if isconst(T, i))) == Set(c) end @@ -34,16 +36,18 @@ end for (T, c) in ( (Core.CodeInfo, []), (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :edges, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile, :time_compile]), - (Core.Method, [:primary_world, :deleted_world]), - (Core.MethodInstance, [:cache, :flags]), - (Core.MethodTable, [:defs, :leafcache, :cache, :max_args]), + (Core.Method, [:primary_world, :did_scan_source, :dispatch_status, :interferences]), + (Core.MethodInstance, [:cache, :flags, :dispatch_status]), + (Core.MethodTable, [:defs]), + (Core.MethodCache, [:leafcache, :cache, :var""]), (Core.TypeMapEntry, [:next, :min_world, :max_world]), (Core.TypeMapLevel, [:arg1, :targ, :name1, :tname, :list, :any]), - (Core.TypeName, [:cache, :linearcache]), + (Core.TypeName, [:cache, :linearcache, :Typeofwrapper, :max_args, :cache_entry_count]), (DataType, [:types, :layout]), (Core.Memory, []), (Core.GenericMemoryRef, []), (Task, [:_state, :running_time_ns, :finished_at, :first_enqueued_at, :last_started_running_at]), + (Core.BindingPartition, [:min_world, :max_world, :next]), ) @test Set((fieldname(T, i) for i in 1:fieldcount(T) if Base.isfieldatomic(T, i))) == Set(c) end @@ -305,22 +309,9 @@ end |> only == Type{typejoin(Int, UInt)} typejoin(Int, UInt, Float64) end |> only == Type{typejoin(Int, UInt, Float64)} -let res = @test_throws TypeError let - Base.Experimental.@force_compile - typejoin(1, 2) - nothing - end - err = res.value - @test err.func === :<: -end -let res = @test_throws TypeError let - Base.Experimental.@force_compile - typejoin(1, 2, 3) - nothing - end - err = res.value - @test err.func === :<: -end +@test typejoin(1, 2) === Any +@test typejoin(1, 2, 3) === Any +@test typejoin(Int, Int, 3) === Any # promote_typejoin returns a Union only with Nothing/Missing combined with concrete types for T in (Nothing, Missing) @@ -1215,8 +1206,8 @@ end # Module() constructor @test names(Module(:anonymous), all = true, imported = true) == [:anonymous] @test names(Module(:anonymous, false), all = true, imported = true) == [:anonymous] -@test invokelatest(getfield, Module(:anonymous, false, true), :Core) == Core -@test_throws UndefVarError invokelatest(getfield, Module(:anonymous, false, false), :Core) +@test invokelatest(getglobal, Module(:anonymous, false, true), :Core) == Core +@test_throws UndefVarError invokelatest(getglobal, Module(:anonymous, false, false), :Core) # exception from __init__() let didthrow = @@ -2289,6 +2280,31 @@ end x6074 = 6074 @test @X6074() == 6074 +# issues #48910, 54417 +macro X43151_nested() + quote my_value = "from_nested_macro" end +end +macro X43151_parent() + quote + my_value = "from_parent_macro" + @X43151_nested() + my_value + end +end +@test @X43151_parent() == "from_parent_macro" + +macro X43151_nested_escaping() + quote $(esc(:my_value)) = "from_nested_macro" end +end +macro X43151_parent_escaping() + quote + my_value = "from_parent_macro" + @X43151_nested_escaping() + my_value + end +end +@test @X43151_parent_escaping() == "from_nested_macro" + # issue #5536 test5536(a::Union{Real, AbstractArray}...) = "Splatting" test5536(a::Union{Real, AbstractArray}) = "Non-splatting" @@ -2645,11 +2661,17 @@ struct D14919 <: Function; end @test B14919()() == "It's a brand new world" @test C14919()() == D14919()() == "Boo." -for f in (:Any, :Function, :(Core.Builtin), :(Union{Nothing, Type}), :(Union{typeof(+), Type}), :(Union{typeof(+), typeof(-)}), :(Base.Callable)) - @test_throws ErrorException("Method dispatch is unimplemented currently for this method signature") @eval (::$f)() = 1 -end -for f in (:(Core.getfield), :((::typeof(Core.getfield))), :((::Core.IntrinsicFunction))) - @test_throws ErrorException("cannot add methods to a builtin function") @eval $f() = 1 +let ex_t = ErrorException, ex_r = r"cannot add methods to builtin function" + for f in (:(Core.Any), :(Core.Function), :(Core.Builtin), :(Base.Callable), :(Union{Nothing,F} where F), :(typeof(Core.getfield)), :(Core.IntrinsicFunction)) + @test_throws ex_t @eval (::$f)() = 1 + @test_throws ex_r @eval (::$f)() = 1 + end + @test_throws ex_t @eval (::Union{Nothing,F})() where {F<:Function} = 1 + @test_throws ex_r @eval (::Union{Nothing,F})() where {F<:Function} = 1 + for f in (:(Core.getfield),) + @test_throws ex_t @eval $f() = 1 + @test_throws ex_r @eval $f() = 1 + end end # issue #33370 @@ -4685,8 +4707,28 @@ end @test Macro_Yielding_Global_Assignment.x == 2 # issue #15718 -@test :(f($NaN)) == :(f($NaN)) -@test isequal(:(f($NaN)), :(f($NaN))) +function compare_test(x, y) + lx = Meta.lower(@__MODULE__, x) + ly = Meta.lower(@__MODULE__, y) + if isequal(x, y) + @test x == y + @test hash(x) == hash(y) + @test isequal(lx, ly) + @test lx == ly + @test hash(lx) == hash(ly) + true + else + @test x != y + @test !isequal(lx, ly) + @test lx != ly + false + end +end +@test compare_test(:(f($NaN)), :(f($NaN))) +@test !compare_test(:(1 + (1 * 1)), :(1 + (1 * 1.0))) +@test compare_test(:(1 + (1 * $NaN)), :(1 + (1 * $NaN))) +@test compare_test(QuoteNode(NaN), QuoteNode(NaN)) +@test !compare_test(QuoteNode(1), QuoteNode(1.0)) # PR #16011 Make sure dead code elimination doesn't delete push and pop # of metadata @@ -4951,6 +4993,9 @@ let ft = Base.datatype_fieldtypes @test !isdefined(ft(B12238.body.body)[1], :instance) # has free type vars end +# issue #54969 +@test !isdefined(Memory.body, :instance) + # `where` syntax in constructor definitions (A12238{T} where T<:Real)(x) = 0 @test A12238{<:Real}(0) == 0 @@ -5087,7 +5132,7 @@ function f16340(x::T) where T return g end let g = f16340(1) - @test isa(typeof(g).name.mt.defs.sig, UnionAll) + @test isa(only(methods(g)).sig, UnionAll) end # issue #16793 @@ -7340,7 +7385,7 @@ Array{Int}(undef, bignum, bignum, 0, bignum, bignum) # but also test that it does throw if the axes multiply to a multiple of typemax(UInt) @test_throws ArgumentError Array{Int}(undef, bignum, bignum) @test_throws ArgumentError Array{Int}(undef, 1, bignum, bignum) -# also test that we always throw erros for negative dims even if other dims are 0 or the product is positive +# also test that we always throw errors for negative dims even if other dims are 0 or the product is positive @test_throws ArgumentError Array{Int}(undef, 0, -4, -4) @test_throws ArgumentError Array{Int}(undef, -4, 1, 0) @test_throws ArgumentError Array{Int}(undef, -4, -4, 1) @@ -8308,11 +8353,12 @@ end module OverlayModule using Base.Experimental: @MethodTable, @overlay +using Test @MethodTable mt # long function def -@overlay mt function sin(x::Float64) - 1 +let m = @overlay mt function sin(x::Float64); 1; end + @test isa(m, Method) end # short function def @overlay mt cos(x::Float64) = 2 @@ -8528,3 +8574,9 @@ module GlobalBindingMulti using .M.C end @test GlobalBindingMulti.S === GlobalBindingMulti.M.C.S + +#58434 bitsegal comparison of oddly sized fields +primitive type ByteString58434 (18 * 8) end + +@test Base.datatype_isbitsegal(Tuple{ByteString58434}) == false +@test Base.datatype_haspadding(Tuple{ByteString58434}) == (length(Base.padding(Tuple{ByteString58434})) > 0) diff --git a/test/dict.jl b/test/dict.jl index 83d35ae18bb85..b3d69a76420ae 100644 --- a/test/dict.jl +++ b/test/dict.jl @@ -294,6 +294,14 @@ end @test eq(Dict{Int,Int}(), Dict{AbstractString,AbstractString}()) end +@testset "sizehint!" begin + d = Dict() + sizehint!(d, UInt(3)) + @test d == Dict() + sizehint!(d, 5) + @test isempty(d) +end + @testset "equality special cases" begin @test Dict(1=>0.0) == Dict(1=>-0.0) @test !isequal(Dict(1=>0.0), Dict(1=>-0.0)) diff --git a/test/docs.jl b/test/docs.jl index 568cab48bd1e5..6c625735ecc7e 100644 --- a/test/docs.jl +++ b/test/docs.jl @@ -119,7 +119,7 @@ end # issue #38819 module NoDocStrings end -@test meta(NoDocStrings) === invokelatest(getfield, NoDocStrings, Base.Docs.META) +@test meta(NoDocStrings) === invokelatest(getglobal, NoDocStrings, Base.Docs.META) # General tests for docstrings. diff --git a/test/embedding/embedding-test.jl b/test/embedding/embedding-test.jl index 34ef9a796ba56..0744cac679698 100644 --- a/test/embedding/embedding-test.jl +++ b/test/embedding/embedding-test.jl @@ -32,5 +32,5 @@ end @test lines[9] == "called bar" @test lines[10] == "calling new bar" @test lines[11] == " From worker 2:\tTaking over the world..." - @test readline(err) == "exception caught from C" + @test "exception caught from C" in readlines(err) end diff --git a/test/error.jl b/test/error.jl index f76a7809b08a9..0d8047aa92a44 100644 --- a/test/error.jl +++ b/test/error.jl @@ -109,8 +109,8 @@ end if mod ∉ visited push!(visited, mod) for name in names(mod, all=true) - isdefined(mod, name) || continue - value = getfield(mod, name) + isdefinedglobal(mod, name) || continue + value = getglobal(mod, name) if value isa Module value === Main && continue test_exceptions(value, visited) diff --git a/test/errorshow.jl b/test/errorshow.jl index 8d582ba8e538c..04ccc52a7df44 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -406,6 +406,8 @@ let err_str, @test occursin("MethodError: no method matching Bool()", err_str) err_str = @except_str :a() MethodError @test occursin("MethodError: objects of type Symbol are not callable", err_str) + err_str = @except_str missing(1) MethodError + @test occursin("MethodError: objects of type Missing are not callable", err_str) err_str = @except_str EightBitType() MethodError @test occursin("MethodError: no method matching $(curmod_prefix)EightBitType()", err_str) err_str = @except_str i() MethodError @@ -792,9 +794,11 @@ backtrace() Base.show_backtrace(io, bt) output = split(String(take!(io)), '\n') length(output) >= 8 || println(output) # for better errors when this fails - @test lstrip(output[3])[1:3] == "[1]" + @test lstrip(output[3])[1] == '┌' + @test lstrip(lstrip(output[3])[4:end])[1:3] == "[1]" @test occursin("g28442", output[3]) - @test lstrip(output[5])[1:3] == "[2]" + @test lstrip(output[5])[1] == '├' + @test lstrip(lstrip(output[5])[4:end])[1:3] == "[2]" @test occursin("f28442", output[5]) is_windows_32_bit = Sys.iswindows() && (Sys.WORD_SIZE == 32) if is_windows_32_bit @@ -802,18 +806,107 @@ backtrace() # https://github.com/JuliaLang/julia/issues/55900 # Instead of skipping them entirely, we skip one, and we loosen the other. - # Broken test: @test occursin("the above 2 lines are repeated 5000 more times", output[7]) - @test occursin("the above 2 lines are repeated ", output[7]) - @test occursin(" more times", output[7]) + # Broken test: @test occursin("repeated 5001 times", output[7]) + @test occursin("repeated ", output[7]) + @test occursin(" times", output[7]) # Broken test: @test lstrip(output[8])[1:7] == "[10003]" @test_broken false else - @test occursin("the above 2 lines are repeated 5000 more times", output[7]) + @test occursin("repeated 5001 times", output[7]) @test lstrip(output[8])[1:7] == "[10003]" end end +@testset "Long stacktrace printing - nested repeated single frame" begin + f28442a(n) = n ≤ 0 ? (return backtrace()) : g28442a(n - 1) + g28442a(n) = 80 > n > 20 ? h28442a(n - 1) : f28442a(n - 1) + h28442a(n) = n % 10 == 0 ? g28442a(n - 1) : h28442a(n - 1) + bt = f28442a(100) + io = IOBuffer() + Base.show_backtrace(io, bt) + output = split(String(take!(io)), '\n') + length(output) >= 21 || println(output) # for better errors when this fails + @test startswith(lstrip(output[3]), "┌ ") + @test lstrip(lstrip(output[3])[4:end])[1:3] == "[1]" + @test occursin("f28442a", output[3]) + @test startswith(lstrip(output[5]), "├ ") + @test lstrip(lstrip(output[5])[4:end])[1:3] == "[2]" + @test occursin("g28442a", output[5]) + + @test startswith(lstrip(output[8]), "┌┌ ") + @test occursin("h28442a", output[8]) + @test startswith(lstrip(output[11]), "├ ") + @test occursin("g28442a", output[11]) + + @test startswith(lstrip(output[14]), "┌ ") + @test occursin("f28442a", output[14]) + @test startswith(lstrip(output[16]), "├ ") + @test occursin("g28442a", output[16]) + + @test occursin("f28442a", output[19]) + + is_windows_32_bit = Sys.iswindows() && (Sys.WORD_SIZE == 32) + if is_windows_32_bit + # Assuming tests are broken on 32-bit Windows as above, no need to repeat loose tests here. + else + @test occursin("repeated 10 times", output[7]) + @test lstrip(lstrip(output[8])[7:end])[1:4] == "[21]" + @test occursin("repeated 9 times", output[10]) + @test lstrip(lstrip(output[11])[4:end])[1:4] == "[30]" + @test occursin("repeated 6 times", output[13]) + @test lstrip(lstrip(output[14])[4:end])[1:4] == "[81]" + @test lstrip(lstrip(output[16])[4:end])[1:4] == "[82]" + @test lstrip(output[19])[1:5] == "[101]" + @test lstrip(output[21])[1:5] == "[102]" + end +end + +@testset "Long stacktrace printing - nested cycles" begin + f28442b(n) = n ≤ 0 ? (return backtrace()) : g28442b(n - 1) + g28442b(n) = 80 > n > 60 || 40 > n > 20 ? h28442b(n - 1) : f28442b(n - 1) + h28442b(n) = g28442b(n - 1) + bt = f28442b(100) + io = IOBuffer() + Base.show_backtrace(io, bt) + output = split(String(take!(io)), '\n') + length(output) >= 21 || println(output) # for better errors when this fails + @test startswith(lstrip(output[3]), "┌ ") + @test lstrip(lstrip(output[3])[4:end])[1:3] == "[1]" + @test occursin("f28442b", output[3]) + @test startswith(lstrip(output[5]), "├ ") + @test lstrip(lstrip(output[5])[4:end])[1:3] == "[2]" + @test occursin("g28442b", output[5]) + + @test startswith(lstrip(output[8]), "┌┌ ") + @test occursin("h28442b", output[8]) + @test startswith(lstrip(output[10]), "├├ ") + @test occursin("g28442b", output[10]) + + @test startswith(lstrip(output[13]), "├┌ ") + @test occursin("f28442b", output[13]) + @test startswith(lstrip(output[15]), "├├ ") + @test occursin("g28442b", output[15]) + + @test occursin("f28442b", output[19]) + + is_windows_32_bit = Sys.iswindows() && (Sys.WORD_SIZE == 32) + if is_windows_32_bit + # Assuming tests are broken on 32-bit Windows as above, no need to repeat loose tests here. + else + @test occursin("repeated 10 times", output[7]) + @test lstrip(lstrip(output[8])[7:end])[1:4] == "[21]" + @test lstrip(lstrip(output[10])[7:end])[1:4] == "[22]" + @test occursin("repeated 10 times", output[12]) + @test lstrip(lstrip(output[13])[7:end])[1:4] == "[41]" + @test lstrip(lstrip(output[15])[7:end])[1:4] == "[42]" + @test occursin("repeated 10 times", output[17]) + @test occursin("repeated 2 times", output[18]) + @test lstrip(output[19])[1:5] == "[101]" + @test lstrip(output[21])[1:5] == "[102]" + end +end + @testset "Line number correction" begin getbt() = backtrace() bt = getbt() @@ -855,7 +948,8 @@ end # Check error message first errorMsg = sprint(Base.showerror, ex) - @test occursin("FieldError: type FieldFoo has no field `c`", errorMsg) + @test occursin("FieldError: type", errorMsg) + @test occursin("FieldFoo has no field `c`", errorMsg) @test occursin("available fields: `a`, `b`", errorMsg) @test occursin("Available properties: `x`, `y`", errorMsg) @@ -880,6 +974,24 @@ end @test occursin(hintExpected, errorMsg) end +module FieldErrorTest +struct Point end +p = Point() +end + +@testset "FieldError with changing fields" begin + # https://discourse.julialang.org/t/better-error-message-for-modified-structs-in-julia-1-12/129265 + err_str1 = @except_str FieldErrorTest.p.x FieldError + @test occursin("FieldErrorTest.Point", err_str1) + @eval FieldErrorTest struct Point{T} + x::T + y::T + end + err_str2 = @except_str FieldErrorTest.p.x FieldError + @test occursin("@world", err_str2) + @test occursin("FieldErrorTest.Point", err_str2) +end + # UndefVar error hints module A53000 export f @@ -941,6 +1053,24 @@ end @test_throws expected_message X.x end +# Module for UndefVarError world age testing +module TestWorldAgeUndef end + +@testset "UndefVarError world age hint" begin + ex = try + TestWorldAgeUndef.newvar + catch e + e + end + @test ex isa UndefVarError + + Core.eval(TestWorldAgeUndef, :(newvar = 42)) + + err_str = sprint(Base.showerror, ex) + @test occursin("The binding may be too new: running in world age", err_str) + @test occursin("while current world is", err_str) +end + # test showing MethodError with type argument struct NoMethodsDefinedHere; end let buf = IOBuffer() @@ -980,6 +1110,20 @@ for (func,str) in ((TestMethodShadow.:+,":+"), (TestMethodShadow.:(==),":(==)"), @test occursin("You may have intended to import Base.$str", sprint(Base.showerror, ex)) end +# Test hint for functions in modules of argument types (issue #58682) +module TestModuleHint + struct Bar end + length(x::Bar) = 42 +end +let ex = try + # Call Base.length on TestModuleHint.Bar - should suggest importing TestModuleHint.length + length(TestModuleHint.Bar()) + catch e + e + end::MethodError + @test occursin("may have intended to extend", sprint(Base.showerror, ex)) +end + # Test that implementation detail of include() is hidden from the user by default let bt = try @noinline include("testhelpers/include_error.jl") @@ -1038,7 +1182,7 @@ if (Sys.isapple() || Sys.islinux()) && Sys.ARCH === :x86_64 catch_backtrace() end bt_str = sprint(Base.show_backtrace, bt) - @test occursin(r"repeats \d+ times", bt_str) + @test occursin(r"repeated \d+ times", bt_str) end let bt = try @@ -1047,7 +1191,7 @@ if (Sys.isapple() || Sys.islinux()) && Sys.ARCH === :x86_64 catch_backtrace() end bt_str = sprint(Base.show_backtrace, bt) - @test occursin(r"the above 2 lines are repeated \d+ more times", bt_str) + @test occursin(r"repeated \d+ times", bt_str) end end end diff --git a/test/file.jl b/test/file.jl index 6425155c82965..a163bc07034ab 100644 --- a/test/file.jl +++ b/test/file.jl @@ -471,7 +471,7 @@ if Sys.iswindows() else @test filesize(dir) > 0 end -# We need both: one to check passed time, one to comapare file's mtime() +# We need both: one to check passed time, one to compare file's mtime() nowtime = time_ns() / 1e9 nowwall = time() # Allow 10s skew in addition to the time it took us to actually execute this code @@ -1383,7 +1383,7 @@ if !Sys.iswindows() stat_d_mv = stat(d_mv) # make sure d does not exist anymore @test !ispath(d) - # comare s, with d_mv + # compare s, with d_mv @test isfile(s) == isfile(d_mv) @test islink(s) == islink(d_mv) islink(s) && @test readlink(s) == readlink(d_mv) @@ -2165,3 +2165,5 @@ end @test dstat.total < 32PB @test dstat.used + dstat.available == dstat.total end + +@test Base.infer_return_type(stat, (String,)) == Base.Filesystem.StatStruct diff --git a/test/gc.jl b/test/gc.jl index 3e9f03ef40d92..7d4a3655a2438 100644 --- a/test/gc.jl +++ b/test/gc.jl @@ -82,6 +82,15 @@ end @testset "Full GC reasons" begin full_sweep_reasons_test() end + +@testset "GC Always Full" begin + prog = "using Test;\n + for _ in 1:10; GC.gc(); end;\n + reasons = Base.full_sweep_reasons();\n + @test reasons[:FULL_SWEEP_REASON_SWEEP_ALWAYS_FULL] >= 10;" + cmd = `$(Base.julia_cmd()) --depwarn=error --startup-file=no --gc-sweep-always-full -e $prog` + @test success(cmd) +end end @testset "Base.GC docstrings" begin diff --git a/test/gmp.jl b/test/gmp.jl index 0812775672969..c0edb462298ea 100644 --- a/test/gmp.jl +++ b/test/gmp.jl @@ -481,6 +481,18 @@ end bytes_to_export_to = Vector{UInt8}(undef, 2) Base.GMP.MPZ.export!(bytes_to_export_to, int_to_export_from, order=0) @test all(bytes_to_export_to .== bytes_to_import_from) + + # test export of 0 is T[0] + zero_to_export = BigInt(0) + bytes_to_export_to = Vector{UInt8}(undef, 0) + Base.GMP.MPZ.export!(bytes_to_export_to, zero_to_export, order=0) + @test bytes_to_export_to == UInt8[0] + + # test export on nonzero vector + x_to_export = BigInt(6) + bytes_to_export_to = UInt8[1, 2, 3, 4, 5] + Base.GMP.MPZ.export!(bytes_to_export_to, x_to_export, order=0) + @test bytes_to_export_to == UInt8[6, 0, 0, 0, 0] end @test isqrt(big(4)) == 2 @@ -796,3 +808,27 @@ t = Rational{BigInt}(0, 1) @test Base.GMP.MPQ.div!(-oo, zo) == -oz end end + +@testset "hashing" begin + for i in 1:10:100 + for shift in vcat(0:8, 9:8:81) + for sgn in (1, -1) + bint = sgn * (big(11)^i << shift) + bfloat = float(bint) + @test (hash(bint) == hash(bfloat)) == (bint == bfloat) + @test hash(bint, Base.HASH_SEED) == + @invoke(hash(bint::Real, Base.HASH_SEED)) + @test Base.hash_integer(bint, Base.HASH_SEED) == + @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED)) + end + end + end + + bint = big(0) + bfloat = float(bint) + @test (hash(bint) == hash(bfloat)) == (bint == bfloat) + @test hash(bint, Base.HASH_SEED) == + @invoke(hash(bint::Real, Base.HASH_SEED)) + @test Base.hash_integer(bint, Base.HASH_SEED) == + @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED)) +end diff --git a/test/hashing.jl b/test/hashing.jl index 173a31d10a6a9..1af73e45fc541 100644 --- a/test/hashing.jl +++ b/test/hashing.jl @@ -88,6 +88,7 @@ vals = Any[ Dict(42 => 101, 77 => 93), Dict{Any,Any}(42 => 101, 77 => 93), (1,2,3,4), (1.0,2.0,3.0,4.0), (1,3,2,4), ("a","b"), (SubString("a",1,1), SubString("b",1,1)), + join('c':'s'), SubString(join('a':'z'), 3, 19), # issue #6900 Dict(x => x for x in 1:10), Dict(7=>7,9=>9,4=>4,10=>10,2=>2,3=>3,8=>8,5=>5,6=>6,1=>1), @@ -108,7 +109,7 @@ vals = Any[ ["a", "b", 1, 2], ["a", 1, 2], ["a", "b", 2, 2], ["a", "a", 1, 2], ["a", "b", 2, 3] ] -for a in vals, b in vals +for (i, a) in enumerate(vals), b in vals[i:end] @test isequal(a,b) == (hash(a)==hash(b)) end @@ -178,8 +179,14 @@ end @test hash([1,2]) == hash(view([1,2,3,4],1:2)) let a = QuoteNode(1), b = QuoteNode(1.0) - @test (hash(a)==hash(b)) == (a==b) + @test hash(a) == hash(b) + @test a != b end +let a = QuoteNode(:(1 + 2)), b = QuoteNode(:(1 + 2)) + @test hash(a) == hash(b) + @test a == b +end + let a = Expr(:block, Core.SlotNumber(1)), b = Expr(:block, Core.SlotNumber(1)), @@ -249,7 +256,9 @@ end ) for a in vals, b in vals - @test isequal(a, b) == (Base.hash_64_32(a) == Base.hash_64_32(b)) + ha = Base.hash_64_32(a) + hb = Base.hash_64_32(b) + @test isequal(a, b) == (ha == hb) end end @@ -260,7 +269,9 @@ end ) for a in vals, b in vals - @test isequal(a, b) == (Base.hash_32_32(a) == Base.hash_32_32(b)) + ha = Base.hash_32_32(a) + hb = Base.hash_32_32(b) + @test isequal(a, b) == (ha == hb) end end end @@ -303,4 +314,10 @@ struct AUnionParam{T<:Union{Nothing,Float32,Float64}} end @test hash(5//3) == hash(big(5)//3) end -@test Core.Compiler.is_foldable_nothrow(Base.infer_effects(hash, Tuple{Type{Int}, UInt})) +@testset "concrete eval type hash" begin + @test Core.Compiler.is_foldable_nothrow(Base.infer_effects(hash, Tuple{Type{Int}, UInt})) + + f(h...) = hash(Char, h...); + src = only(code_typed(f, Tuple{UInt}))[1] + @test count(stmt -> Meta.isexpr(stmt, :foreigncall), src.code) == 0 +end diff --git a/test/intfuncs.jl b/test/intfuncs.jl index 38f29344d2f30..405d6f4644bb0 100644 --- a/test/intfuncs.jl +++ b/test/intfuncs.jl @@ -258,6 +258,13 @@ end @test invmod(T(3), T(124))::T == 83 end + for T in (Int8, Int16, Int32, Int64, Int128) + @test invmod(T(3), unsigned(T)(124)) == 83 + end + + # Verify issue described in PR 58010 is fixed + @test invmod(UInt8(3), UInt16(50000)) === 0x411b + for T in (Int8, UInt8) for x in typemin(T):typemax(T) for m in typemin(T):typemax(T) @@ -326,6 +333,14 @@ end end @testset "nextpow/prevpow" begin + fs = (prevpow, nextpow) + types = (Int8, BigInt, BigFloat) + for f ∈ fs, P ∈ types, R ∈ types, p ∈ 1:20, r ∈ 2:5 + q = P(p) + n = R(r) + @test f(r, p) == f(n, q) + end + @test nextpow(2, 3) == 4 @test nextpow(2, 4) == 4 @test nextpow(2, 7) == 8 @@ -339,7 +354,14 @@ end @test prevpow(10, 101.0) === 100 @test prevpow(10.0, 101) === 100.0 @test_throws DomainError prevpow(0, 3) - @test_throws DomainError prevpow(0, 3) + @test_throws DomainError prevpow(3, 0) + + # "argument is beyond the range of type of the base" + @test_throws DomainError prevpow(Int8(3), 243) + @test_throws DomainError nextpow(Int8(3), 243) + + # "result is beyond the range of type of the base" + @test_throws OverflowError nextpow(Int8(3), 82) end @testset "ndigits/ndigits0z" begin @@ -623,6 +645,19 @@ end @test Base.infer_effects(gcdx, (Int,Int)) |> Core.Compiler.is_foldable @test Base.infer_effects(invmod, (Int,Int)) |> Core.Compiler.is_foldable @test Base.infer_effects(binomial, (Int,Int)) |> Core.Compiler.is_foldable +@testset "concrete-foldability: `hastypemax`" begin + @test Base.infer_effects(Base.hastypemax, (Type,)) |> Core.Compiler.is_foldable + @test Base.infer_effects(Base.hastypemax, (DataType,)) |> Core.Compiler.is_foldable + for t in (Bool, Int, BigInt) + @test Base.infer_effects(Base.hastypemax, (Type{t},)) |> Core.Compiler.is_foldable + end +end + +@testset "`hastypemax`" begin + @test Base.hastypemax(Bool) + @test Base.hastypemax(Int) + @test !Base.hastypemax(BigInt) +end @testset "literal power" begin @testset for T in Base.uniontypes(Base.HWReal) diff --git a/test/iobuffer.jl b/test/iobuffer.jl index a9d58f4b7871e..6163e59beb567 100644 --- a/test/iobuffer.jl +++ b/test/iobuffer.jl @@ -6,6 +6,273 @@ ioslength(io::IOBuffer) = (io.seekable ? io.size : bytesavailable(io)) bufcontents(io::Base.GenericIOBuffer) = unsafe_string(pointer(io.data), io.size) +# Julia Base's internals uses the PipeBuffer, which is an unseekable IOBuffer. +# There are no public constructors to build such a buffer, but we need to test +# it anyway. +# I make a new method here such that if the implementation of Base.PipeBuffer +# changes, these tests will still work. +new_unseekable_buffer() = Base.GenericIOBuffer(Memory{UInt8}(), true, true, false, true, typemax(Int), false) + +@testset "Basic tests" begin + @test_throws ArgumentError IOBuffer(;maxsize=-1) + @test_throws ArgumentError IOBuffer([0x01]; maxsize=-1) + + # Test that sizehint actually will sizehint the vector, + v = UInt8[] + buf = IOBuffer(v; sizehint=64, write=true) + @test length(v.ref.mem) >= 64 + + # Test that you can't make an IOBuffer with a maxsize + # smaller than the size you actually give it + @test_throws ArgumentError IOBuffer([0x01, 0x02]; maxsize=1) + @test_throws ArgumentError IOBuffer(b"abcdefghij"; maxsize=8) +end + +@testset "Basic reading" begin + # Readavailable is equal to read + buf = IOBuffer("abcdef") + @test read(buf, UInt8) == UInt8('a') + @test bytesavailable(buf) == 5 + @test readavailable(buf) == b"bcdef" + + # Reading less than all the bytes + buf = IOBuffer(b"ABCDEFGHIJ") + @test read(buf, 1) == b"A" + @test read(buf, 3) == b"BCD" + + # Reading more bytes than available will not error + @test read(buf, 100) == b"EFGHIJ" + + # Passing truncate=false will still truncate an IOBuffer with no + # initialized data + @test isempty(read(IOBuffer(;sizehint=34, truncate=false))) +end + +@testset "Byte occursin GenericIOBuffer" begin + buf = IOBuffer(@view(collect(0x1f:0x3d)[1:end])) + @test occursin(0x1f, buf) + @test occursin(0x3d, buf) + @test occursin(0x2a, buf) + + @test !occursin(0xff, buf) + @test !occursin(0x00, buf) + + v = Vector{UInt8}("bcdefg") + pushfirst!(v, UInt8('a')) + buf = IOBuffer(v) + @test occursin(UInt8('a'), buf) + read(buf, UInt8) + @test !occursin(UInt8('a'), buf) + @test !occursin(0x00, buf) + + buf = IOBuffer("abcdefg") + @test occursin(UInt8('a'), buf) +end + +@testset "Non-Memory backed IOBuffer" begin + buf = IOBuffer(Test.GenericArray(collect(0x02:0x0d)), read=true) + @test read(buf) == 0x02:0x0d + + buf = IOBuffer(Test.GenericArray(collect(0x02:0x0d)), read=true) + @test read(buf, UInt8) == 0x02 + @test read(buf) == 0x03:0x0d + + v = view(collect(UInt8('a'):UInt8('z')), 4:10) + buf = IOBuffer(v, read=true, write=true) + @test read(buf, UInt8) == UInt8('d') + @test read(buf) == UInt8('e'):UInt8('j') + seekstart(buf) + @test read(buf, UInt8) == UInt8('d') + write(buf, UInt8('x')) + write(buf, "ABC") + seekstart(buf) + @test read(buf) == b"dxABCij" +end + +@testset "Copying" begin + # Test offset is preserved when copying + v = UInt8[] + pushfirst!(v, UInt8('a'), UInt8('b'), UInt8('c')) + buf = IOBuffer(v; write=true, read=true, append=true) + write(buf, "def") + read(buf, UInt16) + buf2 = copy(buf) + @test String(read(buf)) == "cdef" + @test String(read(buf2)) == "cdef" + + # Test copying with non-Memory backed GenericIOBuffer + buf = IOBuffer(Test.GenericArray(collect(0x02:0x0d)), read=true) + @test read(buf, UInt16) == 0x0302 + buf2 = copy(buf) + @test isreadable(buf2) + @test !iswritable(buf2) + @test read(buf2) == 0x04:0x0d + + # Test copying a non-seekable stream + buf = new_unseekable_buffer() + write(buf, "abcdef") + read(buf, UInt16) + mark(buf) + read(buf, UInt16) + buf2 = copy(buf) + @test read(buf2) == b"ef" + reset(buf2) + @test read(buf2) == b"cdef" + + # Test copying seekable stream + buf = IOBuffer() + write(buf, "abcdef") + seekstart(buf) + read(buf) + mark(buf) + buf2 = copy(buf) + @test reset(buf2) == 6 + seekstart(buf2) + @test read(buf2) == b"abcdef" + + # Test copying a taken buffer + buf = IOBuffer() + write(buf, "abcdef") + take!(buf) + buf2 = copy(buf) + @test eof(buf2) + seekstart(buf2) + @test eof(buf2) +end + +@testset "copyuntil" begin + a = IOBuffer(b"abcdeajdgabdfg") + b = IOBuffer(collect(b"xx"); write=true, read=true, append=true) + copyuntil(b, a, UInt8('a')) + @test read(b) == b"xx" + seekstart(b) + copyuntil(b, a, UInt8('a'); keep=true) + @test read(b) == b"xxbcdea" + seekstart(b) + copyuntil(b, a, UInt('w')) + @test read(b) == b"xxbcdeajdgabdfg" +end + +@testset "copyline" begin + a = IOBuffer(b"abcde\nabc\r\nabc\n\r\nac") + b = IOBuffer() + copyline(b, a) + @test take!(copy(b)) == b"abcde" + copyline(b, a) + @test take!(copy(b)) == b"abcdeabc" + copyline(b, a; keep=true) + @test take!(copy(b)) == b"abcdeabcabc\n" + copyline(b, a; keep=false) + @test take!(copy(b)) == b"abcdeabcabc\n" + copyline(b, a; keep=false) + @test take!(copy(b)) == b"abcdeabcabc\nac" + + # Test a current bug in copyline + a = Base.SecretBuffer("abcde\r\n") + b = IOBuffer() + write(b, "xxxxxxxxxx") + seek(b, 2) + copyline(b, a; keep=false) + Base.shred!(a) + @test take!(b) == b"xxabcdexxx" +end + +@testset "take!" begin + a = IOBuffer("abc") + @test take!(a) == b"abc" + + v = UInt8[] + pushfirst!(v, 0x0a) + buf = IOBuffer(v; write=true, append=true) + write(buf, "def") + @test take!(buf) == b"\ndef" + + v = view(collect(b"abcdefghij"), 3:9) + buf = IOBuffer(v; write=true, read=true) + read(buf, UInt8) + write(buf, "xxy") + @test take!(buf) == b"cxxyghi" + + v = view(collect(b"abcdefghij"), 3:9) + buf = IOBuffer(v; write=true, read=true) + + # Take on unseekable buffer does not return used bytes. + buf = new_unseekable_buffer() + write(buf, 0x61) + write(buf, "bcd") + @test read(buf, UInt8) == 0x61 + @test take!(buf) == b"bcd" + + # Compaction is reset after take! + buf = Base.GenericIOBuffer(Memory{UInt8}(), true, true, false, true, 100, false) + write(buf, rand(UInt8, 50)) + read(buf, 40) + write(buf, rand(UInt8, 100)) + mark(buf) + read(buf, 70) + @test position(buf) == 110 + @test length(buf.data) <= 100 + v = take!(buf) + write(buf, 0xf1) + @test position(buf) == 0 + @test !ismarked(buf) +end + +@testset "maxsize is preserved" begin + # After take! + buf = IOBuffer(; maxsize=3) + print(buf, "abcdef") + @test take!(buf) == b"abc" + print(buf, "abcdef") + @test take!(buf) == b"abc" + + # After resizing + buf = IOBuffer(;maxsize=128) + write(buf, collect(0x00:0x10)) + write(buf, collect(0x11:0x30)) + write(buf, collect(0x31:0x98)) + write(buf, collect(0x99:0xff)) + seekstart(buf) + @test read(buf) == 0x00:UInt8(127) + + # Edge case: When passing a Vector, does not error if the + # underlying mem is larger than maxsize + v = pushfirst!([0x01], 0x02) + io = IOBuffer(v; maxsize=2) + @test read(io) == b"\x02\x01" + + # Buffer will not write past maxsize, even if given a larger buffer + # And also even if the data is taken and replaced + v = sizehint!(UInt8[], 128) + io = IOBuffer(v; write=true, read=true, maxsize=12) + write(io, 0x01:0x0f) + seekstart(io) + @test read(io) == 0x01:0x0c + @test write(io, 0x01) == 0 + @test write(io, "abc") == 0 + @test take!(io).ref.mem === v.ref.mem + write(io, 0x01:0x0f) + @test take!(io) == 0x01:0x0c +end + +@testset "Write to self" begin + buffer = IOBuffer() + @test_throws ArgumentError write(buffer, buffer) + + # Write to another IOBuffer with limited size + to = IOBuffer(;maxsize=4) + from = IOBuffer(collect(b"abcdefghi")) + write(to, from) + @test String(take!(to)) == "abcd" + @test eof(from) + + # Write to another IOBuffer when closed + to = IOBuffer() + from = IOBuffer(collect(b"abcdefghi")) + close(from) + @test_throws ArgumentError write(to, from) +end + @testset "Read/write empty IOBuffer" begin io = IOBuffer() @test eof(io) @@ -33,7 +300,7 @@ bufcontents(io::Base.GenericIOBuffer) = unsafe_string(pointer(io.data), io.size) @test position(io) == 0 truncate(io, 10) @test position(io) == 0 - @test all(io.data .== 0) + @test all(view(io.data, 1:10) .== 0) @test write(io, Int16[1, 2, 3, 4, 5, 6]) === 12 seek(io, 2) truncate(io, 10) @@ -54,6 +321,36 @@ bufcontents(io::Base.GenericIOBuffer) = unsafe_string(pointer(io.data), io.size) @test_throws ArgumentError seek(io, 0) end +@testset "takestring!" begin + buf = IOBuffer() + write(buf, "abcø") + s = takestring!(buf) + @test isempty(takestring!(buf)) + @test s == "abcø" + write(buf, "xyz") + @test takestring!(buf) == "xyz" + buf = IOBuffer() + + # Test with a nonzero offset in the buffer + v = rand(UInt8, 8) + for i in 1:8 + pushfirst!(v, rand(UInt8)) + end + buf = IOBuffer(v) + s = String(copy(v)) + @test takestring!(buf) == s + + # Test with a non-writable IOBuffer + buf = IOBuffer(b"abcdef") + read(buf, UInt8) + @test takestring!(buf) == "abcdef" + + buf = new_unseekable_buffer() + write(buf, "abcde") + read(buf, UInt16) + @test takestring!(buf) == "cde" +end + @testset "Read/write readonly IOBuffer" begin io = IOBuffer("hamster\nguinea pig\nturtle") @test position(io) == 0 @@ -67,22 +364,89 @@ end @test_throws ArgumentError write(io,UInt8[0]) @test String(take!(io)) == "hamster\nguinea pig\nturtle" @test String(take!(io)) == "hamster\nguinea pig\nturtle" #should be unchanged - @test_throws ArgumentError Base.compact(io) # not writeable close(io) end +@testset "Truncate" begin + # Fails for non-writable and non-seekable + @test_throws ArgumentError truncate(PipeBuffer(), 0) + @test_throws ArgumentError truncate(IOBuffer(b"abcde"), 3) + + # Standard use + buf = IOBuffer(collect(b"abcdef"); write=true, read=true) + truncate(buf, 4) + @test read(buf) == b"abcd" + @test take!(buf) == b"abcd" + + # Mark is removed if beyond the size + buf = IOBuffer() + write(buf, "abcde") + seek(buf, 4) + mark(buf) + truncate(buf, 4) + @test !ismarked(buf) + + # Making it larger + buf = IOBuffer(collect(b"abcdef"); write=true, read=true) + seek(buf, 3) + truncate(buf, 3) + write(buf, 'X') + mark(buf) + truncate(buf, 5) + @test ismarked(buf) + @test reset(buf) == 4 + @test take!(buf) == b"abcX\0" + + # With offset + v = pushfirst!(UInt8[0x62, 0x63, 0x64], 0x61) + buf = IOBuffer(v; write=true, read=true) + seekstart(buf) + read(buf, UInt8) + mark(buf) + truncate(buf, 7) + @test reset(buf) == 1 + @test take!(buf) == b"abcd\0\0\0" +end + +@testset "Position of compactable buffer" begin + # Set maxsize, because otherwise compaction it too hard to reason about, + # and this test will be brittle + io = Base.GenericIOBuffer(Memory{UInt8}(), true, true, false, true, 100, false) + write(io, "abcd") + read(io, UInt16) + @test position(io) == 2 + write(io, "abcde"^80) + @test position(io) == 2 + read(io, 60) + @test position(io) == 62 + mark(io) + # Trigger compaction + write(io, rand(UInt8, 50)) + @test position(io) == 62 + v1 = read(io, 20) + @test position(io) == 82 + @test reset(io) == 62 + @test position(io) == 62 + v2 = read(io, 20) + @test v1 == v2 +end + @testset "PipeBuffer" begin - io = PipeBuffer() + io = new_unseekable_buffer() @test_throws EOFError read(io,UInt8) @test write(io,"pancakes\nwaffles\nblueberries\n") > 0 + + # PipeBuffer is append, so writing to it does not advance the position @test position(io) == 0 @test readline(io) == "pancakes" - Base.compact(io) @test readline(io) == "waffles" @test write(io,"whipped cream\n") > 0 @test readline(io) == "blueberries" + + # Pipebuffers do not support seeking, and therefore do not support truncation. @test_throws ArgumentError seek(io,0) @test_throws ArgumentError truncate(io,0) + @test readline(io) == "whipped cream" @test write(io,"pancakes\nwaffles\nblueberries\n") > 0 @test readlines(io) == String["pancakes", "waffles", "blueberries"] @@ -116,58 +480,6 @@ end end rm(fname) end - - Base.compact(io) - @test position(io) == 0 - @test ioslength(io) == 0 - Base._resize!(io,0) - Base.ensureroom(io,50) - @test position(io) == 0 - @test ioslength(io) == 0 - @test length(io.data) == 50 - Base.ensureroom(io,10) - @test ioslength(io) == 0 - @test length(io.data) == 50 - io.maxsize = 75 - Base.ensureroom(io,100) - @test ioslength(io) == 0 - @test length(io.data) == 75 - seekend(io) - @test ioslength(io) == 0 - @test position(io) == 0 - write(io,zeros(UInt8,200)) - @test ioslength(io) == 75 - @test length(io.data) == 75 - write(io,1) - @test ioslength(io) == 75 - @test length(io.data) == 75 - write(io,[1,2,3]) - @test ioslength(io) == 75 - @test length(io.data) == 75 - skip(io,1) - @test write(io,UInt8(104)) === 1 - skip(io,3) - @test write(io,b"apples") === 3 - skip(io,71) - @test write(io,'y') === 1 - @test read(io, String) == "happy" - @test eof(io) - write(io,zeros(UInt8,73)) - write(io,'a') - write(io,'b') - write(io,'c') - write(io,'d') - write(io,'e') - @test ioslength(io) == 75 - @test length(io.data) == 75 - @test position(io) == 0 - skip(io,72) - @test String(take!(io)) == "\0ab" - @test String(take!(io)) == "" - - # issues 4021 - print(io, true) - close(io) end @testset "issue 5453" begin @@ -221,6 +533,13 @@ end end end +@testset "issue #57962" begin + io = IOBuffer(repeat("x", 400)) + skip(io, 10) + skip(io, 400) + @test isempty(read(io)) +end + @testset "pr #11554" begin io = IOBuffer(SubString("***αhelloworldω***", 4, 16)) io2 = IOBuffer(Vector{UInt8}(b"goodnightmoon"), read=true, write=true) @@ -248,9 +567,6 @@ end truncate(io2, io2.size - 2) @test read(io2, String) == "goodnightmoonhelloworld" seek(io2, 0) - write(io2, io2) - @test read(io2, String) == "" - @test bufcontents(io2) == "goodnightmoonhelloworld" end # issue #11917 @@ -347,24 +663,42 @@ end @test n == 5 end -@testset "Base.compact" begin - a = Base.GenericIOBuffer(UInt8[], true, true, false, true, typemax(Int)) - mark(a) # mark at position 0 - write(a, "Hello!") - @test Base.compact(a) === nothing # because pointer > mark - close(a) - b = Base.GenericIOBuffer(UInt8[], true, true, false, true, typemax(Int)) - write(b, "Hello!") - read(b) - mark(b) # mark at position 6 - write(b, "Goodbye!") # now pointer is > mark but mark is > 0 - Base.compact(b) - @test readline(b) == "Goodbye!" - close(b) +@testset "Compacting" begin + # Compacting works + buf = Base.GenericIOBuffer(UInt8[], true, true, false, true, 20, false) + mark(buf) + write(buf, "Hello"^5) + reset(buf) + unmark(buf) + read(buf, UInt8) + read(buf, UInt8) + write(buf, "a!") + @test length(buf.data) == 20 + @test String(take!(buf)) == "llo" * "Hello"^3 * "a!" + + # Compacting does not do anything when mark == 0 + buf = Base.GenericIOBuffer(UInt8[], true, true, false, true, 5, false) + mark(buf) + write(buf, "Hello") + reset(buf) + mark(buf) + read(buf, UInt8) + read(buf, UInt8) + @test write(buf, "a!") == 0 + @test take!(buf) == b"llo" + + # Compacting without maxsize still works + buf = new_unseekable_buffer() + data = repeat(b"abcdefg", 100) + write(buf, data) + read(buf, 600) + data_len = length(buf.data) + write(buf, view(data, 1:500)) + @test length(buf.data) == data_len end @testset "peek(::GenericIOBuffer)" begin - io = Base.GenericIOBuffer(UInt8[], true, true, false, true, typemax(Int)) + io = Base.GenericIOBuffer(UInt8[], true, true, false, true, typemax(Int), false) write(io, "こんにちは") @test peek(io) == 0xe3 @test peek(io, Char) == 'こ' @@ -381,19 +715,33 @@ end v = @view a[1:2] io = IOBuffer() write(io,1) + write(io,0) seek(io,0) - @test Base.read_sub(io,v,1,1) == [1,0] + @test read!(io, v) == [1, 0] end @testset "with offset" begin b = pushfirst!([0x02], 0x01) @test take!(IOBuffer(b)) == [0x01, 0x02] + + # Read-only buffer does not take control of underlying buffer + v = pushfirst!([0x62, 0x63], 0x61) + buf = IOBuffer(v; write=false) + @test read(buf) == b"abc" + @test v == b"abc" # v is unchanged + + # Truncate end @testset "#54636 reading from non-dense vectors" begin data = 0x00:0xFF io = IOBuffer(data) @test read(io) == data + seekstart(io) + @test read(io, UInt16) === ltoh(0x0100) + out = IOBuffer() + write(out, io) + @test take!(out) == data[3:end] data = @view(collect(0x00:0x0f)[begin:2:end]) io = IOBuffer(data) diff --git a/test/iterators.jl b/test/iterators.jl index df4fa63b433b8..11e7b3756fb3e 100644 --- a/test/iterators.jl +++ b/test/iterators.jl @@ -988,6 +988,12 @@ end @test accumulate(+, (x^2 for x in 1:3); init=100) == [101, 105, 114] end +@testset "issue #58109" begin + i = Iterators.map(identity, 3) + j = Iterators.map(sqrt, 7) + @test (@inferred Base.IteratorSize(i)) === @inferred Base.IteratorSize(eltype([i, j])) +end + @testset "IteratorSize trait for zip" begin @test (@inferred Base.IteratorSize(zip())) == Base.IsInfinite() # for zip of empty tuple @test (@inferred Base.IteratorSize(zip((1,2,3), repeated(0)))) == Base.HasLength() # for zip of ::HasLength and ::IsInfinite @@ -1133,6 +1139,70 @@ end end end +@testset "nth" begin + + Z = Array{Int,0}(undef) + Z[] = 17 + it_result_pairs = Dict( + (Z, 1) => 17, + (collect(1:100), 23) => 23, + (10:6:1000, 123) => 10 + 6 * 122, + ("∀ϵ>0", 3) => '>', + ((1, 3, 5, 10, 78), 2) => 3, + (reshape(1:30, (5, 6)), 21) => 21, + (3, 1) => 3, + (true, 1) => true, + ('x', 1) => 'x', + (4 => 5, 2) => 5, + (view(Z), 1) => 17, + (view(reshape(1:30, (5, 6)), 2:4, 2:6), 10) => 22, + ((x^2 for x in 1:10), 9) => 81, + (Iterators.Filter(isodd, 1:10), 3) => 5, + (Iterators.flatten((1:10, 50:60)), 15) => 54, + (pairs(50:60), 7) => 7 => 56, + (zip(1:10, 21:30, 51:60), 6) => (6, 26, 56), + (Iterators.product(1:3, 10:12), 3) => (3, 10), + (Iterators.repeated(3.14159, 5), 4) => 3.14159, + ((a=2, b=3, c=5, d=7, e=11), 4) => 7, + (Iterators.cycle(collect(1:100)), 9999) => 99, + (Iterators.cycle([1, 2, 3, 4, 5], 5), 25) => 5, + (Iterators.cycle("String", 10), 16) => 'i', + (Iterators.cycle(((),)), 1000) => () + ) + + + @testset "iter: $IT" for (IT, n) in keys(it_result_pairs) + @test it_result_pairs[(IT, n)] == nth(IT, n) + @test_throws BoundsError nth(IT, -42) + + IT isa Iterators.Cycle && continue # cycles are infinite so never OOB + @test_throws BoundsError nth(IT, 999999999) + end + + empty_cycle = Iterators.cycle([]) + @test_throws BoundsError nth(empty_cycle, 42) + + # test the size unknown branch for cycles + # only generate odd numbers so we know the actual length + # but the iterator is still SizeUnknown() + it_size_unknown = Iterators.filter(isodd, 1:2:10) + @test Base.IteratorSize(it_size_unknown) isa Base.SizeUnknown + @test length(collect(it_size_unknown)) == 5 + + cycle_size_unknown = Iterators.cycle(it_size_unknown) + finite_cycle_size_unknown = Iterators.cycle(it_size_unknown, 5) + @test nth(cycle_size_unknown, 2) == 3 + @test nth(cycle_size_unknown, 20) == 9 # mod1(20, 5) = 5, wraps 4 times + @test nth(finite_cycle_size_unknown, 2) == 3 + @test nth(finite_cycle_size_unknown, 20) == 9 + @test_throws BoundsError nth(finite_cycle_size_unknown, 30) # only wraps 5 times, max n is 5 * 5 = 25 +end + @testset "Iterators docstrings" begin @test isempty(Docs.undocumented_names(Iterators)) end + +# Filtered list comprehension (`Filter` construct) type inference +@test Base.infer_return_type((Vector{Any},)) do xs + [x for x in xs if x isa Int] +end == Vector{Int} diff --git a/test/llvmpasses/alloc-opt-pass.ll b/test/llvmpasses/alloc-opt-pass.ll index 665687e86835d..c6c279ae36fc6 100644 --- a/test/llvmpasses/alloc-opt-pass.ll +++ b/test/llvmpasses/alloc-opt-pass.ll @@ -73,6 +73,11 @@ L3: ; preds = %L2, %L1, %0 } ; CHECK-LABEL: }{{$}} +declare void @external_function() + +declare ptr addrspace(10) @external_function2() + + ; CHECK-LABEL: @legal_int_types ; CHECK: alloca [12 x i8] ; CHECK-NOT: alloca i96 @@ -89,21 +94,6 @@ define void @legal_int_types() { } ; CHECK-LABEL: }{{$}} -declare void @external_function() - -declare ptr addrspace(10) @external_function2() - -declare ptr @julia.ptls_states() - -declare ptr @julia.get_pgcstack() - -declare noalias ptr addrspace(10) @julia.gc_alloc_obj(ptr, i64, ptr addrspace(10)) - -declare ptr @julia.pointer_from_objref(ptr addrspace(11)) - -declare token @llvm.julia.gc_preserve_begin(...) - -declare void @llvm.julia.gc_preserve_end(token) ; CHECK-LABEL: @memref_collision ; OPAQUE: call ptr @julia.ptls_states() @@ -171,13 +161,13 @@ define void @initializers() { %pgcstack = call ptr @julia.get_pgcstack() %ptls = call ptr @julia.ptls_states() %ptls_i8 = bitcast ptr %ptls to ptr - %var1 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 1, ptr addrspace(10) @tag) #1 + %var1 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 1, ptr addrspace(10) @tag) #4 %var2 = addrspacecast ptr addrspace(10) %var1 to ptr addrspace(11) %var3 = call ptr @julia.pointer_from_objref(ptr addrspace(11) %var2) - %var4 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 2, ptr addrspace(10) @tag) #2 + %var4 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 2, ptr addrspace(10) @tag) #7 %var5 = addrspacecast ptr addrspace(10) %var4 to ptr addrspace(11) %var6 = call ptr @julia.pointer_from_objref(ptr addrspace(11) %var5) - %var7 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 3, ptr addrspace(10) @tag) #3 + %var7 = call ptr addrspace(10) @julia.gc_alloc_obj(ptr %ptls_i8, i64 3, ptr addrspace(10) @tag) #1 %var8 = addrspacecast ptr addrspace(10) %var7 to ptr addrspace(11) %var9 = call ptr @julia.pointer_from_objref(ptr addrspace(11) %var8) ret void @@ -203,14 +193,154 @@ union_move9: ; No predecessors! } ; CHECK-LABEL: }{{$}} +@0 = private unnamed_addr constant ptr inttoptr (i64 4373799056 to ptr), !julia.constgv !0 +@1 = private unnamed_addr constant i64 0, align 8 + +; CHECK-LABEL: @cmpxchg +; CHECK: alloca +; CHECK: alloca +; CHECK: %20 = cmpxchg ptr %2, +define swiftcc i64 @"cmpxchg"(ptr nonnull swiftself "gcstack" %0) #0 { + %2 = alloca i64, align 16 + %3 = call ptr @julia.get_pgcstack() + %4 = getelementptr inbounds i8, ptr %3, i32 -152 + %5 = getelementptr inbounds i8, ptr %4, i32 168 + %6 = load ptr, ptr %5, align 8, !tbaa !4 + %7 = getelementptr inbounds i8, ptr %6, i32 16 + %8 = load ptr, ptr %7, align 8, !tbaa !8, !invariant.load !0 + fence syncscope("singlethread") seq_cst + call void @julia.safepoint(ptr %8) + fence syncscope("singlethread") seq_cst + %9 = load ptr, ptr @0, align 8, !tbaa !8, !invariant.load !0, !alias.scope !10, !noalias !13, !nonnull !0, !dereferenceable !18, !align !19 + %10 = ptrtoint ptr %9 to i64 + %11 = inttoptr i64 %10 to ptr + %12 = getelementptr inbounds i8, ptr %3, i32 -152 + %13 = addrspacecast ptr %11 to ptr addrspace(10) + call void @llvm.lifetime.start.p0(i64 8, ptr %2) + %14 = call noalias nonnull align 8 dereferenceable(8) ptr addrspace(10) @julia.gc_alloc_obj(ptr %12, i64 8, ptr addrspace(10) %13) #7 + %15 = addrspacecast ptr addrspace(10) %14 to ptr addrspace(11) + call void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) align 8 %15, ptr align 8 @1, i64 8, i1 false), !tbaa !20, !alias.scope !23, !noalias !24 + %16 = addrspacecast ptr addrspace(10) %14 to ptr addrspace(11) + %17 = load atomic i64, ptr addrspace(11) %16 monotonic, align 8, !tbaa !25, !alias.scope !23, !noalias !24 + br label %19 + +18: ; preds = %19 + ret i64 %21 + +19: ; preds = %19, %1 + %20 = phi i64 [ %17, %1 ], [ %23, %19 ] + %21 = call swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself "gcstack" %3, i64 signext %20, i64 signext 1) + %22 = cmpxchg ptr addrspace(11) %16, i64 %20, i64 %21 seq_cst monotonic, align 8, !tbaa !25, !alias.scope !23, !noalias !24 + %23 = extractvalue { i64, i1 } %22, 0 + %24 = extractvalue { i64, i1 } %22, 1 + br i1 %24, label %18, label %19 +} + +; CHECK-LABEL: }{{$}} +; CHECK-LABEL: @atomicrmw +; CHECK: alloca +; CHECK: alloca +; CHECK: atomicrmw xchg ptr %2, +define swiftcc i64 @"atomicrmw"(ptr nonnull swiftself "gcstack" %0) #0 { + %2 = alloca i64, align 16 + %3 = call ptr @julia.get_pgcstack() + %4 = getelementptr inbounds i8, ptr %3, i32 -152 + %5 = getelementptr inbounds i8, ptr %4, i32 168 + %6 = load ptr, ptr %5, align 8, !tbaa !4 + %7 = getelementptr inbounds i8, ptr %6, i32 16 + %8 = load ptr, ptr %7, align 8, !tbaa !8, !invariant.load !0 + fence syncscope("singlethread") seq_cst + call void @julia.safepoint(ptr %8) + fence syncscope("singlethread") seq_cst + %9 = load ptr, ptr @0, align 8, !tbaa !8, !invariant.load !0, !alias.scope !10, !noalias !13, !nonnull !0, !dereferenceable !18, !align !19 + %10 = ptrtoint ptr %9 to i64 + %11 = inttoptr i64 %10 to ptr + %12 = getelementptr inbounds i8, ptr %3, i32 -152 + %13 = addrspacecast ptr %11 to ptr addrspace(10) + call void @llvm.lifetime.start.p0(i64 8, ptr %2) + %14 = call noalias nonnull align 8 dereferenceable(8) ptr addrspace(10) @julia.gc_alloc_obj(ptr %12, i64 8, ptr addrspace(10) %13) #7 + %15 = addrspacecast ptr addrspace(10) %14 to ptr addrspace(11) + call void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) align 8 %15, ptr align 8 @1, i64 8, i1 false), !tbaa !20, !alias.scope !23, !noalias !24 + %16 = addrspacecast ptr addrspace(10) %14 to ptr addrspace(11) + %17 = load atomic i64, ptr addrspace(11) %16 monotonic, align 8, !tbaa !25, !alias.scope !23, !noalias !24 + %18 = call swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself "gcstack" %3, i64 signext %17, i64 signext 1) + %19 = atomicrmw xchg ptr addrspace(11) %16, i64 %18 seq_cst, align 8, !tbaa !25, !alias.scope !23, !noalias !24 ; preds = %19 + ret i64 %19 +} + +declare ptr @julia.ptls_states() + +declare ptr @julia.pointer_from_objref(ptr addrspace(11)) + +declare token @llvm.julia.gc_preserve_begin(...) + +declare void @llvm.julia.gc_preserve_end(token) + +declare ptr @julia.get_pgcstack() + +; Function Attrs: mustprogress nounwind willreturn memory(inaccessiblemem: readwrite) +declare nonnull align 8 dereferenceable(8) ptr addrspace(10) @ijl_box_int64(i64 signext) #2 + +; Function Attrs: memory(argmem: readwrite, inaccessiblemem: readwrite) +declare void @julia.safepoint(ptr) #3 + +; Function Attrs: mustprogress nounwind willreturn allockind("alloc") allocsize(1) memory(argmem: read, inaccessiblemem: readwrite) +declare noalias nonnull ptr addrspace(10) @julia.gc_alloc_obj(ptr, i64, ptr addrspace(10)) #4 + ; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite) -declare void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #0 +declare void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #5 + ; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite) -declare void @llvm.memcpy.p0.p11.i64(ptr noalias nocapture writeonly, ptr addrspace(11) noalias nocapture readonly, i64, i1 immarg) #0 +declare void @llvm.memcpy.p0.p11.i64(ptr noalias nocapture writeonly, ptr addrspace(11) noalias nocapture readonly, i64, i1 immarg) #5 + ; Function Attrs: nocallback nofree nounwind willreturn memory(argmem: readwrite) -declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #0 +declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #5 + +declare swiftcc i64 @"jlsys_+_47"(ptr nonnull swiftself, i64 signext, i64 signext) #0 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #6 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #6 + +attributes #0 = { "probe-stack"="inline-asm" } +attributes #1 = { nounwind willreturn allockind("alloc,zeroed") allocsize(1) memory(argmem: read, inaccessiblemem: readwrite) } +attributes #2 = { mustprogress nounwind willreturn memory(inaccessiblemem: readwrite) } +attributes #3 = { memory(argmem: readwrite, inaccessiblemem: readwrite) } +attributes #4 = { mustprogress nounwind willreturn allockind("alloc") allocsize(1) memory(argmem: read, inaccessiblemem: readwrite) } +attributes #5 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) } +attributes #6 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #7 = { nounwind willreturn allockind("alloc,uninitialized") allocsize(1) memory(argmem: read, inaccessiblemem: readwrite) } +attributes #8 = { nounwind willreturn memory(inaccessiblemem: readwrite) } + +!llvm.module.flags = !{!1, !2, !3} + +!0 = !{} +!1 = !{i32 2, !"Dwarf Version", i32 4} +!2 = !{i32 2, !"Debug Info Version", i32 3} +!3 = !{i32 2, !"julia.optlevel", i32 2} +!4 = !{!5, !5, i64 0} +!5 = !{!"jtbaa_gcframe", !6, i64 0} +!6 = !{!"jtbaa", !7, i64 0} +!7 = !{!"jtbaa"} +!8 = !{!9, !9, i64 0, i64 1} +!9 = !{!"jtbaa_const", !6, i64 0} +!10 = !{!11} +!11 = !{!"jnoalias_const", !12} +!12 = !{!"jnoalias"} +!13 = !{!14, !15, !16, !17} +!14 = !{!"jnoalias_gcframe", !12} +!15 = !{!"jnoalias_stack", !12} +!16 = !{!"jnoalias_data", !12} +!17 = !{!"jnoalias_typemd", !12} +!18 = !{i64 56} +!19 = !{i64 16} +!20 = !{!21, !21, i64 0} +!21 = !{!"jtbaa_value", !22, i64 0} +!22 = !{!"jtbaa_data", !6, i64 0} +!23 = !{!16} +!24 = !{!14, !15, !17, !11} +!25 = !{!26, !26, i64 0} +!26 = !{!"jtbaa_mutab", !21, i64 0} -attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) } -attributes #1 = { allockind("alloc") } -attributes #2 = { allockind("alloc,uninitialized") } -attributes #3 = { allockind("alloc,zeroed") } diff --git a/test/llvmpasses/atomic-modify.ll b/test/llvmpasses/atomic-modify.ll new file mode 100644 index 0000000000000..23e1949f3ad0a --- /dev/null +++ b/test/llvmpasses/atomic-modify.ll @@ -0,0 +1,288 @@ +; This file is a part of Julia. License is MIT: https://julialang.org/license + +; RUN: opt --load-pass-plugin=libjulia-codegen%shlibext -passes='ExpandAtomicModify' -S %s | FileCheck %s + +declare {i8, i8} @julia.atomicmodify.i8(ptr, ptr, i8, i8, ...) +declare {double, double} @julia.atomicmodify.f64(ptr, ptr, i8, i8, ...) +declare double @llvm.maxnum.f64(double %Val0, double %Val1) + +define i8 @add.i8(i8 %x, i8 %y) { + %z = add i8 %x, %y + ret i8 %z +} + +define i8 @sub.i8(i8 %x, i8 %y) { + %z = sub i8 %x, %y + ret i8 %z +} + +define i8 @subx.i8(i8 %x, i8 %y) { + %z = sub i8 %y, %x + ret i8 %z +} + +define i8 @add.i8.zext(i8 %x, i1 %y) { + %y8 = zext i1 %y to i8 + %z = add i8 %x, %y8 + ret i8 %z +} + +define i8 @and.i8(i8 %x, i8 %y) { + %z = and i8 %x, %y + ret i8 %z +} + +define i8 @nand.i8(i8 %x, i8 %y) { + %z = and i8 %x, %y + %w = xor i8 %z, -1 + ret i8 %w +} + +define i8 @nand.i8.zext(i8 %x, i1 %y) { + %y8 = zext i1 %y to i8 + %z = and i8 %y8, %x + %w = xor i8 %z, -1 + ret i8 %w +} + +define i8 @xchg.i8(i8 %x, i8 %y) { + ret i8 %y +} + +define double @fadd.f64(double %x, double %y) { + %z = fadd double %y, %x + ret double %z +} + +define double @fmax.f64(double %x, double %y) { + %z = call double @llvm.maxnum.f64(double %y, double %x) + ret double %z +} + +define internal i8 @0(i8 %x, i8 %y) unnamed_addr { + %z = call i8 @add.i8(i8 %x, i8 %y) + ret i8 %z +} + +define internal i8 @1(i8 %x, i8 %y) unnamed_addr { + %z = call i8 @0(i8 %x, i8 %y) + ret i8 %z +} + +define internal i8 @2(i8 %x, i8 %y, ptr %f) unnamed_addr { + %z = call i8 %f(i8 %x, i8 %y) + ret i8 %z +} + +define i8 @mod_i8_add(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_add +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_add_new(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_add +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK-NEXT: [[newval:%.*]] = add i8 %0, %b +; CHECK-NEXT: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8, i8 5, i8 1, i8 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_addfence(ptr %a) { +; CHECK-LABEL: @mod_i8_addfence +; CHECK: %0 = atomicrmw or ptr %a, i8 0 release, align 1 +; CHECK-NEXT: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8, i8 5, i8 1, i8 0) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_add_zext(ptr %a, i1 %b) { +; CHECK-LABEL: @mod_i8_add_zext +; CHECK: [[b8:%.*]] = zext i1 %b to i8 +; CHECK: %0 = atomicrmw add ptr %a, i8 [[b8]] release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8.zext, i8 5, i8 1, i1 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_add_zext_new(ptr %a, i1 %b) { +; CHECK-LABEL: @mod_i8_add_zext +; CHECK: [[b8:%.*]] = zext i1 %b to i8 +; CHECK-NEXT: %0 = atomicrmw add ptr %a, i8 [[b8]] release, align 1 +; CHECK-NEXT: [[newval:%.*]] = add i8 %0, [[b8]] +; CHECK-NEXT: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @add.i8.zext, i8 5, i8 1, i1 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_sub(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_sub +; CHECK: %0 = atomicrmw sub ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @sub.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_subx(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_subx +; CHECK: [[newval:%.*]] = call i8 @subx.i8(i8 %loaded, i8 %b) +; CHECK: [[success:%.*]] = cmpxchg ptr %a, i8 %loaded, i8 [[newval]] +; CHECK: [[oldval:%.*]] = extractvalue { i8, i1 } [[success:%.*]], 0 +; CHECK: ret i8 [[oldval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @subx.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_subx_new(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_subx_new +; CHECK: [[newval:%.*]] = call i8 @subx.i8(i8 %loaded, i8 %b) +; CHECK: [[oldval:%.*]] = cmpxchg ptr %a, i8 %loaded, i8 [[newval]] +; CHECK: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @subx.i8, i8 5, i8 1, i8 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_nand(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_nand +; CHECK: %0 = atomicrmw nand ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @nand.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_nand_new(ptr %a, i1 %b) { +; CHECK-LABEL: @mod_i8_nand_new +; CHECK: [[b8:%.*]] = zext i1 %b to i8 +; CHECK: %0 = atomicrmw nand ptr %a, i8 [[b8]] release, align 1 +; CHECK: [[newand:%.*]] = and i8 [[b8]], %0 +; CHECK: [[newval:%.*]] = xor i8 [[newand:%.*]], -1 +; CHECK: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @nand.i8.zext, i8 5, i8 1, i1 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_andxchg(ptr %a) { +; CHECK-LABEL: @mod_i8_andxchg +; CHECK: %0 = atomicrmw xchg ptr %a, i8 0 release, align 1 +; CHECK-NEXT: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @and.i8, i8 5, i8 1, i8 0) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_xchg(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_xchg +; CHECK: %0 = atomicrmw xchg ptr %a, i8 %b release, align 1 +; CHECK-NEXT: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @xchg.i8, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_xchg_new(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_xchg_new +; CHECK: %0 = atomicrmw xchg ptr %a, i8 %b release, align 1 +; CHECK-NEXT: ret i8 %b +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @xchg.i8, i8 5, i8 1, i8 %b) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define double @mod_i8_fadd(ptr %a, double %b) { +; CHECK-LABEL: @mod_i8_fadd +; CHECK: %0 = atomicrmw fadd ptr %a, double %b release, align 8 +; CHECK: ret double %0 +top: + %oldnew = call {double, double} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.f64(ptr align(8) %a, ptr @fadd.f64, i8 5, i8 1, double %b) + %oldval = extractvalue {double, double} %oldnew, 0 + ret double %oldval +} + +define double @mod_i8_fmax(ptr %a, double %b) { +; CHECK-LABEL: @mod_i8_fmax +; CHECK: %0 = atomicrmw fmax ptr %a, double %b release, align 8 +; CHECK: ret double %0 +top: + %oldnew = call {double, double} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.f64(ptr align(8) %a, ptr @fmax.f64, i8 5, i8 1, double %b) + %oldval = extractvalue {double, double} %oldnew, 0 + ret double %oldval +} + +define i8 @mod_i8_indirect0(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_indirect0 +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @0, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_indirect1(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_indirect1 +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @1, i8 5, i8 1, i8 %b) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_indirect2(ptr %a, i8 %b, ptr %f) { +; CHECK-LABEL: @mod_i8_indirect2 +; CHECK: [[newval:%.*]] = call i8 %f(i8 %loaded, i8 %b) +; CHECK: [[success:%.*]] = cmpxchg ptr %a, i8 %loaded, i8 [[newval]] +; CHECK: [[oldval:%.*]] = extractvalue { i8, i1 } [[success:%.*]], 0 +; CHECK: ret i8 [[oldval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @2, i8 5, i8 1, i8 %b, ptr %f) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} + +define i8 @mod_i8_indirect2_new(ptr %a, i8 %b, ptr %f) { +; CHECK-LABEL: @mod_i8_indirect2_new +; CHECK: [[newval:%.*]] = call i8 %f(i8 %loaded, i8 %b) +; CHECK: [[oldval:%.*]] = cmpxchg ptr %a, i8 %loaded, i8 [[newval]] +; CHECK: ret i8 [[newval]] +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @2, i8 5, i8 1, i8 %b, ptr %f) + %newval = extractvalue {i8, i8} %oldnew, 1 + ret i8 %newval +} + +define i8 @mod_i8_indirect3(ptr %a, i8 %b) { +; CHECK-LABEL: @mod_i8_indirect3 +; CHECK: %0 = atomicrmw add ptr %a, i8 %b release, align 1 +; CHECK: ret i8 %0 +top: + %oldnew = call {i8, i8} (ptr, ptr, i8, i8, ...) @julia.atomicmodify.i8(ptr align(1) %a, ptr @2, i8 5, i8 1, i8 %b, ptr @0) + %oldval = extractvalue {i8, i8} %oldnew, 0 + ret i8 %oldval +} diff --git a/test/llvmpasses/gcroots.ll b/test/llvmpasses/gcroots.ll index 9f9282cd3c870..d8c1438e4ff63 100644 --- a/test/llvmpasses/gcroots.ll +++ b/test/llvmpasses/gcroots.ll @@ -764,7 +764,7 @@ define i8 @gather_arrayptrs_alltrue() { ; OPAQUE: %gcframe = alloca ptr addrspace(10), i32 3 -; OPAQUE: %arrayptrs = call <2 x ptr addrspace(13)> @llvm.masked.gather.v2p13.v2p11(<2 x ptr addrspace(11)> %arrayptrptrs, i32 16, <2 x i1> , <2 x ptr addrspace(13)> zeroinitializer) +; OPAQUE: %arrayptrs = call <2 x ptr addrspace(13)> @llvm.masked.gather.v2p13.v2p11(<2 x ptr addrspace(11)> %arrayptrptrs, i32 16, <2 x i1> {{(|splat \(i1 true\))}}, <2 x ptr addrspace(13)> zeroinitializer) ; OPAQUE: [[GEP0:%.*]] = getelementptr inbounds ptr addrspace(10), ptr %gcframe, i32 2 ; OPAQUE: store ptr addrspace(10) %obj1, ptr [[GEP0]] ; diff --git a/test/llvmpasses/image-codegen.jl b/test/llvmpasses/image-codegen.jl index 35e5add2de601..d594c02a4392e 100644 --- a/test/llvmpasses/image-codegen.jl +++ b/test/llvmpasses/image-codegen.jl @@ -2,7 +2,7 @@ # RUN: export JULIA_LLVM_ARGS="--print-before=loop-vectorize --print-module-scope" # RUN: rm -rf %t # RUN: mkdir %t -# RUN: julia --image-codegen --startup-file=no %s 2> %t/output.txt +# RUN: julia --image-codegen -t1,0 --startup-file=no %s 2> %t/output.txt # RUN: FileCheck %s < %t/output.txt # COM: checks that global variables compiled in imaging codegen diff --git a/test/llvmpasses/late-lower-gc-sret.ll b/test/llvmpasses/late-lower-gc-sret.ll index d0ad94fcf8990..b8593f691bb6f 100644 --- a/test/llvmpasses/late-lower-gc-sret.ll +++ b/test/llvmpasses/late-lower-gc-sret.ll @@ -6,7 +6,7 @@ declare ptr @julia.get_pgcstack() declare swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]), ptr nonnull swiftself, ptr addrspace(10) nonnull) -define hidden swiftcc nonnull ptr addrspace(10) @sret_select(ptr nonnull swiftself %0, ptr addrspace(10) noundef nonnull align 8 dereferenceable(88) %1, i1 %unpredictable) { +define hidden swiftcc nonnull ptr addrspace(10) @sret_select(ptr nonnull swiftself "gcstack" %0, ptr addrspace(10) noundef nonnull align 8 dereferenceable(88) %1, i1 %unpredictable) { ; CHECK-LABEL: @sret_select ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 6) ; CHECK: call ptr @julia.get_gc_frame_slot(ptr %gcframe, i32 3) @@ -17,12 +17,12 @@ define hidden swiftcc nonnull ptr addrspace(10) @sret_select(ptr nonnull swiftse %3 = alloca [3 x i64], align 8 %4 = alloca [3 x i64], align 8 %5 = select i1 %unpredictable, ptr %3, ptr %4 - call swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]) %5, ptr nonnull swiftself %0, ptr addrspace(10) nonnull %1) + call swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]) %5, ptr nonnull swiftself "gcstack" %0, ptr addrspace(10) nonnull %1) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret ptr addrspace(10) %1 } -define hidden swiftcc nonnull ptr addrspace(10) @sret_phi(ptr nonnull swiftself %0, ptr addrspace(10) noundef nonnull align 8 dereferenceable(88) %1, i1 %unpredictable) { +define hidden swiftcc nonnull ptr addrspace(10) @sret_phi(ptr nonnull swiftself "gcstack" %0, ptr addrspace(10) noundef nonnull align 8 dereferenceable(88) %1, i1 %unpredictable) { top: ; CHECK-LABEL: @sret_phi ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 6) @@ -43,14 +43,14 @@ false: ; preds = %top ret: ; preds = %false, %true %4 = phi ptr [ %2, %true ], [ %3, %false ] - call swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]) %4, ptr nonnull swiftself %0, ptr addrspace(10) nonnull %1) + call swiftcc void @sret_call(ptr noalias nocapture noundef nonnull sret([3 x ptr addrspace(10)]) %4, ptr nonnull swiftself "gcstack" %0, ptr addrspace(10) nonnull %1) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret ptr addrspace(10) %1 } declare swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }), ptr noalias nocapture noundef, ptr nonnull swiftself) -define hidden swiftcc void @sret_gc_root_phi(ptr nonnull swiftself %0, i1 %unpredictable) { +define hidden swiftcc void @sret_gc_root_phi(ptr nonnull swiftself "gcstack" %0, i1 %unpredictable) { top: ; CHECK-LABEL: @sret_gc_root_phi ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 2) @@ -75,13 +75,13 @@ false: ; preds = %top ret: ; preds = %false, %true %4 = phi ptr [ %2, %true ], [ %3, %false ] - call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %4, ptr nonnull swiftself %0) + call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %4, ptr nonnull swiftself "gcstack" %0) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret void } -define hidden swiftcc void @sret_gc_root_phi_select(ptr nonnull swiftself %0, i1 %unpredictable, i1 %unpredictable2) { +define hidden swiftcc void @sret_gc_root_phi_select(ptr nonnull swiftself "gcstack" %0, i1 %unpredictable, i1 %unpredictable2) { top: ; CHECK-LABEL: @sret_gc_root_phi_select ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 3) @@ -110,12 +110,12 @@ false: ; preds = %top ret: ; preds = %false, %true %5 = phi ptr [ %2, %true ], [ %3, %false ] %6 = select i1 %unpredictable2, ptr %4, ptr %5 - call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %6, ptr nonnull swiftself %0) + call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %6, ptr nonnull swiftself "gcstack" %0) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret void } -define hidden swiftcc void @sret_gc_root_select_phi(ptr nonnull swiftself %0, i1 %unpredictable, i1 %unpredictable2) { +define hidden swiftcc void @sret_gc_root_select_phi(ptr nonnull swiftself "gcstack" %0, i1 %unpredictable, i1 %unpredictable2) { top: ; CHECK-LABEL: @sret_gc_root_select_phi ; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 3) @@ -145,7 +145,7 @@ false: ; preds = %top ret: ; preds = %false, %true %6 = phi ptr [ %2, %true ], [ %5, %false ] - call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %6, ptr nonnull swiftself %0) + call swiftcc void @sret_call_gc(ptr noalias nocapture noundef sret({ ptr addrspace(10), i64, i64 }) %1, ptr noalias nocapture noundef %6, ptr nonnull swiftself "gcstack" %0) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) ret void } diff --git a/test/llvmpasses/late-lower-gc.ll b/test/llvmpasses/late-lower-gc.ll index 4739fa186ffc7..346e19e537819 100644 --- a/test/llvmpasses/late-lower-gc.ll +++ b/test/llvmpasses/late-lower-gc.ll @@ -164,6 +164,21 @@ define {} addrspace(10)* @gclift_switch({} addrspace(13)* addrspace(10)* %input, ret {} addrspace(10)* %ret } +; Shouldn't hang +define void @vector_insert(<4 x {} addrspace(10)* > %0, <2 x {} addrspace(10)* > %1) { +top: + %pgcstack = call {}*** @julia.get_pgcstack() + %2 = call <4 x {} addrspace(10)*> @llvm.vector.insert.v4p10.v2p10(<4 x {} addrspace(10)*> %0, <2 x {} addrspace(10)*> %1, i64 2) + ret void +} + +define void @vector_extract(<4 x {} addrspace(10)* > %0, <2 x {} addrspace(10)* > %1) { +top: + %pgcstack = call {}*** @julia.get_pgcstack() + %2 = call <2 x {} addrspace(10)*> @llvm.vector.extract.v2p10.v4p10(<4 x {} addrspace(10)* > %0, i64 2) + ret void +} + define void @decayar([2 x {} addrspace(10)* addrspace(11)*] %ar) { %v2 = call {}*** @julia.get_pgcstack() %e0 = extractvalue [2 x {} addrspace(10)* addrspace(11)*] %ar, 0 @@ -184,6 +199,20 @@ define void @decayar([2 x {} addrspace(10)* addrspace(11)*] %ar) { ; CHECK: %r = call i32 @callee_root(ptr addrspace(10) %l0, ptr addrspace(10) %l1) ; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) +define swiftcc ptr addrspace(10) @insert_element(ptr swiftself "gcstack" %0) { +; CHECK-LABEL: @insert_element + %2 = alloca [10 x i64], i32 1, align 8 +; CHECK: %gcframe = call ptr @julia.new_gc_frame(i32 10) +; CHECK: [[gc_slot_addr_:%.*]] = call ptr @julia.get_gc_frame_slot(ptr %gcframe, i32 0) +; CHECK: call void @julia.push_gc_frame(ptr %gcframe, i32 10) + call void null(ptr sret([2 x [5 x ptr addrspace(10)]]) %2, ptr null, ptr addrspace(11) null, ptr null) + %4 = insertelement <4 x ptr> zeroinitializer, ptr %2, i32 0 +; CHECK: [[gc_slot_addr_:%.*]] = insertelement <4 x ptr> zeroinitializer, ptr [[gc_slot_addr_:%.*]], i32 0 +; CHECK: call void @julia.pop_gc_frame(ptr %gcframe) + ret ptr addrspace(10) null +} + + !0 = !{i64 0, i64 23} !1 = !{!1} !2 = !{!7} ; scope list diff --git a/test/llvmpasses/remove-addrspaces.ll b/test/llvmpasses/remove-addrspaces.ll index fbd84de85a4a3..99acd92b0e03b 100644 --- a/test/llvmpasses/remove-addrspaces.ll +++ b/test/llvmpasses/remove-addrspaces.ll @@ -2,6 +2,9 @@ ; RUN: opt --load-pass-plugin=libjulia-codegen%shlibext -passes='RemoveJuliaAddrspaces' -S %s | FileCheck %s --check-prefixes=CHECK,OPAQUE +; COM: check that the addrspace of the global itself is removed +; OPAQUE: @ejl_enz_runtime_exc = external global {} +@ejl_enz_runtime_exc = external addrspace(10) global {} ; COM: check that package image fptrs work @pjlsys_BoundsError_32 = internal global {} addrspace(10)* ({}***, {} addrspace(10)*, [1 x i64] addrspace(11)*)* null @@ -111,6 +114,13 @@ define void @byval_type([1 x {} addrspace(10)*] addrspace(11)* byval([1 x {} add } +define private fastcc void @diffejulia__mapreduce_97() { +L6: +; OPAQUE: store atomic ptr @ejl_enz_runtime_exc, ptr null unordered + store atomic {} addrspace(10)* @ejl_enz_runtime_exc, {} addrspace(10)* addrspace(10)* null unordered, align 8 + unreachable +} + ; COM: check that function attributes are preserved on declarations too declare void @convergent_function() #0 attributes #0 = { convergent } diff --git a/test/loading.jl b/test/loading.jl index d12cd2769ef1d..3a085277d96b8 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1495,7 +1495,7 @@ end # helper function to load a package and return the output function load_package(name, args=``) - code = "using $name" + code = "Base.disable_parallel_precompile = true; using $name" cmd = addenv(`$(Base.julia_cmd()) -e $code $args`, "JULIA_LOAD_PATH" => dir, "JULIA_DEPOT_PATH" => depot_path, @@ -1731,3 +1731,21 @@ end rm(depot_path, force=true, recursive=true) end end + +# Test `import Package as M` +module M57965 + import Random as R +end +@test M57965.R === Base.require(M57965, :Random) + +# #58272 - _eval_import accidentally reuses evaluated "from" path +module M58272_1 + const x = 1 + module M58272_2 + const y = 3 + const x = 2 + end +end +module M58272_to end +@eval M58272_to import ..M58272_1: M58272_2.y, x +@test @eval M58272_to x === 1 diff --git a/test/math.jl b/test/math.jl index a32d66edc30f8..dea1c8035d5eb 100644 --- a/test/math.jl +++ b/test/math.jl @@ -46,8 +46,7 @@ has_fma = Dict( @test clamp(100, Int8) === Int8(100) @test clamp(200, Int8) === typemax(Int8) - begin - x = [0.0, 1.0, 2.0, 3.0, 4.0] + let x = [0.0, 1.0, 2.0, 3.0, 4.0] clamp!(x, 1, 3) @test x == [1.0, 1.0, 2.0, 3.0, 3.0] end @@ -59,12 +58,14 @@ has_fma = Dict( @test clamp(typemax(UInt16), Int16) === Int16(32767) # clamp should not allocate a BigInt for typemax(Int16) - x = big(2) ^ 100 - @test (@allocated clamp(x, Int16)) == 0 + let x = big(2) ^ 100 + @test (@allocated clamp(x, Int16)) == 0 + end - x = clamp(2.0, BigInt) - @test x isa BigInt - @test x == big(2) + let x = clamp(2.0, BigInt) + @test x isa BigInt + @test x == big(2) + end end end @@ -447,7 +448,7 @@ end end @testset "deg2rad/rad2deg" begin - @testset "$T" for T in (Int, Float64, BigFloat) + @testset "$T" for T in (Int, Float16, Float32, Float64, BigFloat) @test deg2rad(T(180)) ≈ 1pi @test deg2rad.(T[45, 60]) ≈ [pi/T(4), pi/T(3)] @test rad2deg.([pi/T(4), pi/T(3)]) ≈ [45, 60] @@ -455,6 +456,16 @@ end @test rad2deg(T(1)) ≈ rad2deg(true) @test deg2rad(T(1)) ≈ deg2rad(true) end + @testset "accuracy" begin + @testset "$T" for T in (Float16, Float32, Float64) + @test rad2deg(T(1)) === setprecision(BigFloat, 500) do + T(180 / BigFloat(pi)) + end + @test deg2rad(T(1)) === setprecision(BigFloat, 500) do + T(BigFloat(pi) / 180) + end + end + end @test deg2rad(180 + 60im) ≈ pi + (pi/3)*im @test rad2deg(pi + (pi/3)*im) ≈ 180 + 60im end diff --git a/test/meta.jl b/test/meta.jl index e9e344bba2e22..3d5fc08ee24e9 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -234,7 +234,7 @@ let ex = Meta.parseall("@foo", filename=:bar) @test isa(arg2arg2, LineNumberNode) && arg2arg2.file === :bar end -_lower(m::Module, ex, world::UInt) = ccall(:jl_expand_in_world, Any, (Any, Ref{Module}, Cstring, Cint, Csize_t), ex, m, "none", 0, world) +_lower(m::Module, ex, world::UInt) = Base.fl_lower(ex, m, "none", 0, world, false)[1] module TestExpandInWorldModule macro m() 1 end diff --git a/test/misc.jl b/test/misc.jl index 6c8d76fa1cd1a..8782941adcc1e 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -237,6 +237,24 @@ end @test all(<=(sem_size), history) @test all(>=(0), history) @test history[end] == 0 + + # macro form + clock = Threads.Atomic{Int}(1) + occupied = Threads.Atomic{Int}(0) + history = fill!(Vector{Int}(undef, 2n), -1) + @sync for _ in 1:n + @async begin + @test Base.@acquire s begin + history[Threads.atomic_add!(clock, 1)] = Threads.atomic_add!(occupied, 1) + 1 + sleep(rand(0:0.01:0.1)) + history[Threads.atomic_add!(clock, 1)] = Threads.atomic_sub!(occupied, 1) - 1 + return :resultvalue + end === :resultvalue + end + end + @test all(<=(sem_size), history) + @test all(>=(0), history) + @test history[end] == 0 end # task switching @@ -481,12 +499,12 @@ begin local second = @capture_stdout @time @eval calldouble2(1.0) # these functions were not recompiled - local matches = collect(eachmatch(r"(\d+(?:\.\d+)?)%", first)) + local matches = collect(eachmatch(r"(\d+(?:\.\d+)?)% compilation", first)) @test length(matches) == 1 @test parse(Float64, matches[1][1]) > 0.0 @test parse(Float64, matches[1][1]) <= 100.0 - matches = collect(eachmatch(r"(\d+(?:\.\d+)?)%", second)) + matches = collect(eachmatch(r"(\d+(?:\.\d+)?)% compilation", second)) @test length(matches) == 1 @test parse(Float64, matches[1][1]) > 0.0 @test parse(Float64, matches[1][1]) <= 100.0 @@ -1614,10 +1632,10 @@ end let errs = IOBuffer() run(`$(Base.julia_cmd()) -e ' using Test - @test isdefined(DataType.name.mt, :backedges) + @test !isempty(Core.methodtable.backedges) Base.Experimental.disable_new_worlds() @test_throws "disable_new_worlds" @eval f() = 1 - @test !isdefined(DataType.name.mt, :backedges) + @test isempty(Core.methodtable.backedges) @test_throws "disable_new_worlds" Base.delete_method(which(+, (Int, Int))) @test 1+1 == 2 using Dates diff --git a/test/mpfr.jl b/test/mpfr.jl index 3bb8768b280d5..48477fc4dbcb7 100644 --- a/test/mpfr.jl +++ b/test/mpfr.jl @@ -35,6 +35,9 @@ import Base.MPFR @test typeof(BigFloat(1//1)) == BigFloat @test typeof(BigFloat(one(Rational{BigInt}))) == BigFloat + rat = 1 // (big(2)^300 + 1) + @test BigFloat(rat, RoundDown) < rat < BigFloat(rat, RoundUp) + @test BigFloat(-rat, RoundUp) < -rat < BigFloat(-rat, RoundDown) # BigFloat constructor respects global precision when not specified let prec = precision(BigFloat) < 16 ? 256 : precision(BigFloat) ÷ 2 @@ -667,16 +670,19 @@ end @test string(parse(BigFloat, "0.1")) == "0.10000002" @test string(parse(BigFloat, "0.5")) == "0.5" @test string(parse(BigFloat, "-9.9")) == "-9.9000015" + @test string(parse(BigFloat, "1e6")) == "1.0e6" end setprecision(40) do @test string(parse(BigFloat, "0.1")) == "0.10000000000002" @test string(parse(BigFloat, "0.5")) == "0.5" @test string(parse(BigFloat, "-9.9")) == "-9.8999999999942" + @test string(parse(BigFloat, "1e6")) == "1.0e6" end setprecision(123) do @test string(parse(BigFloat, "0.1")) == "0.0999999999999999999999999999999999999953" @test string(parse(BigFloat, "0.5")) == "0.5" @test string(parse(BigFloat, "-9.9")) == "-9.8999999999999999999999999999999999997" + @test string(parse(BigFloat, "1e6")) == "1.0e6" end end @testset "eps" begin @@ -998,7 +1004,7 @@ end test_show_bigfloat(big"1.23456789", contains_e=false, starts="1.23") test_show_bigfloat(big"-1.23456789", contains_e=false, starts="-1.23") - test_show_bigfloat(big"2.3457645687563543266576889678956787e10000", starts="2.345", ends="e+10000") + test_show_bigfloat(big"2.3457645687563543266576889678956787e10000", starts="2.345", ends="e10000") test_show_bigfloat(big"-2.3457645687563543266576889678956787e-10000", starts="-2.345", ends="e-10000") test_show_bigfloat(big"42.0", contains_e=false, starts="42.0") test_show_bigfloat(big"420.0", contains_e=false, starts="420.0") # '0's have to be added on the right before point @@ -1006,10 +1012,10 @@ end test_show_bigfloat(big"420000.0", contains_e=false, starts="420000.0") test_show_bigfloat(big"654321.0", contains_e=false, starts="654321.0") test_show_bigfloat(big"-654321.0", contains_e=false, starts="-654321.0") - test_show_bigfloat(big"6543210.0", contains_e=true, starts="6.5", ends="e+06") + test_show_bigfloat(big"6543210.0", contains_e=true, starts="6.5", ends="e6") test_show_bigfloat(big"0.000123", contains_e=false, starts="0.000123") test_show_bigfloat(big"-0.000123", contains_e=false, starts="-0.000123") - test_show_bigfloat(big"0.00001234", contains_e=true, starts="1.23", ends="e-05") + test_show_bigfloat(big"0.00001234", contains_e=true, starts="1.23", ends="e-5") for to_string in [string, x->sprint(show, x), diff --git a/test/numbers.jl b/test/numbers.jl index 510c630c9f4ba..0553ab342f7a2 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -154,7 +154,8 @@ end x = unorded[i], unorded[i] y = unorded[j], unorded[j] z = Base._extrema_rf(x, y) - @test z === x || z === y + @test (z[1] === x[1] || z[1] === y[1]) && + (z[2] === x[1] || z[2] === y[1]) end end end @@ -830,6 +831,28 @@ end @test cmp(isless, 1, NaN) == -1 @test cmp(isless, NaN, NaN) == 0 end +@testset "ispositive/isnegative" begin + for T in [Base.uniontypes(Base.BitInteger)..., Bool, Rational{Int}, BigInt, Base.uniontypes(Base.IEEEFloat)..., BigFloat, Missing] + values = T[zero(T), one(T)] + if T <: AbstractFloat + push!(values, Inf, NaN) # also check Infs and NaNs + elseif T <: Rational + push!(values, 1//0) # also check Infs + end + @testset "$T" begin + for value in values + # https://github.com/JuliaLang/julia/pull/53677#discussion_r1534044582 + # Use eval to explicitly show expressions when they fail + @eval begin + @test ispositive($value) === ($value > 0) + @test ispositive(-$value) === (-$value > 0) + @test isnegative($value) === ($value < 0) + @test isnegative(-$value) === (-$value < 0) + end + end + end + end +end @testset "Float vs Integer comparison" begin for x=-5:5, y=-5:5 @test (x==y)==(Float64(x)==Int64(y)) @@ -1595,36 +1618,44 @@ end end end - for x=0:5, y=1:5 - @test div(UInt(x),UInt(y)) == div(x,y) - @test div(UInt(x),y) == div(x,y) - @test div(x,UInt(y)) == div(x,y) - @test div(UInt(x),-y) == reinterpret(UInt,div(x,-y)) - @test div(-x,UInt(y)) == div(-x,y) - - @test fld(UInt(x),UInt(y)) == fld(x,y) - @test fld(UInt(x),y) == fld(x,y) - @test fld(x,UInt(y)) == fld(x,y) - @test fld(UInt(x),-y) == reinterpret(UInt,fld(x,-y)) - @test fld(-x,UInt(y)) == fld(-x,y) - - @test cld(UInt(x),UInt(y)) == cld(x,y) - @test cld(UInt(x),y) == cld(x,y) - @test cld(x,UInt(y)) == cld(x,y) - @test cld(UInt(x),-y) == reinterpret(UInt,cld(x,-y)) - @test cld(-x,UInt(y)) == cld(-x,y) - - @test rem(UInt(x),UInt(y)) == rem(x,y) - @test rem(UInt(x),y) == rem(x,y) - @test rem(x,UInt(y)) == rem(x,y) - @test rem(UInt(x),-y) == rem(x,-y) - @test rem(-x,UInt(y)) == rem(-x,y) - - @test mod(UInt(x),UInt(y)) == mod(x,y) - @test mod(UInt(x),y) == mod(x,y) - @test mod(x,UInt(y)) == mod(x,y) - @test mod(UInt(x),-y) == mod(x,-y) - @test mod(-x,UInt(y)) == mod(-x,y) + @test isnan(mod(NaN, Inf)) + @test isnan(mod(NaN, -Inf)) + for x=0:5 + @test mod(x, Inf) == x + @test mod(x, -Inf) == x + @test mod(-x, Inf) == -x + @test mod(-x, -Inf) == -x + for y=1:5 + @test div(UInt(x),UInt(y)) == div(x,y) + @test div(UInt(x),y) == div(x,y) + @test div(x,UInt(y)) == div(x,y) + @test div(UInt(x),-y) == reinterpret(UInt,div(x,-y)) + @test div(-x,UInt(y)) == div(-x,y) + + @test fld(UInt(x),UInt(y)) == fld(x,y) + @test fld(UInt(x),y) == fld(x,y) + @test fld(x,UInt(y)) == fld(x,y) + @test fld(UInt(x),-y) == reinterpret(UInt,fld(x,-y)) + @test fld(-x,UInt(y)) == fld(-x,y) + + @test cld(UInt(x),UInt(y)) == cld(x,y) + @test cld(UInt(x),y) == cld(x,y) + @test cld(x,UInt(y)) == cld(x,y) + @test cld(UInt(x),-y) == reinterpret(UInt,cld(x,-y)) + @test cld(-x,UInt(y)) == cld(-x,y) + + @test rem(UInt(x),UInt(y)) == rem(x,y) + @test rem(UInt(x),y) == rem(x,y) + @test rem(x,UInt(y)) == rem(x,y) + @test rem(UInt(x),-y) == rem(x,-y) + @test rem(-x,UInt(y)) == rem(-x,y) + + @test mod(UInt(x),UInt(y)) == mod(x,y) + @test mod(UInt(x),y) == mod(x,y) + @test mod(x,UInt(y)) == mod(x,y) + @test mod(UInt(x),-y) == mod(x,-y) + @test mod(-x,UInt(y)) == mod(-x,y) + end end @test div(typemax(UInt64) , 1) == typemax(UInt64) @@ -2389,8 +2420,8 @@ end function allsubtypes!(m::Module, x::DataType, sts::Set) for s in names(m, all = true) - if isdefined(m, s) && !Base.isdeprecated(m, s) - t = getfield(m, s) + if isdefinedglobal(m, s) && !Base.isdeprecated(m, s) + t = getglobal(m, s) if isa(t, Type) && t <: x && t != Union{} push!(sts, t) elseif isa(t, Module) && t !== m && nameof(t) === s && parentmodule(t) === m diff --git a/test/opaque_closure.jl b/test/opaque_closure.jl index 6c988b068a668..0dc2ed95b8872 100644 --- a/test/opaque_closure.jl +++ b/test/opaque_closure.jl @@ -390,3 +390,23 @@ let ir = first(only(Base.code_ircode(sin, (Int,)))) oc = Core.OpaqueClosure(ir; do_compile=false) @test oc(1) == sin(1) end + +function typed_add54236(::Type{T}) where T + return @opaque (x::Int)->T(x) + T(1) +end +let f = typed_add54236(Float64) + @test f isa Core.OpaqueClosure + @test f(32) === 33.0 +end + +f54357(g, ::Type{AT}) where {AT} = Base.Experimental.@opaque AT->_ (args...) -> g((args::AT)...) +let f = f54357(+, Tuple{Int,Int}) + @test f isa Core.OpaqueClosure + @test f(32, 34) === 66 + g = f54357(+, Tuple{Float64,Float64}) + @test g isa Core.OpaqueClosure + @test g(32.0, 34.0) === 66.0 +end + +# 49659: signature-scoped typevar shouldn't fail in lowering +@test_throws "must be a tuple type" @opaque ((x::T,y::T) where {T}) -> 123 diff --git a/test/precompile.jl b/test/precompile.jl index 07384e66927a4..8e091692b68fd 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -105,6 +105,11 @@ precompile_test_harness(false) do dir process_state_calls = 0 @assert process_state() === process_state() @assert process_state_calls === 0 + + const empty_state = Base.OncePerProcess{Nothing}() do + return nothing + end + @assert empty_state() === nothing end """) write(Foo2_file, @@ -681,13 +686,15 @@ precompile_test_harness(false) do dir error("break me") end """) - @test_warn r"LoadError: break me\nStacktrace:\n[ ]*\[1\] [\e01m\[]*error" try - Base.require(Main, :FooBar2) - error("the \"break me\" test failed") - catch exc - isa(exc, ErrorException) || rethrow() - occursin("ERROR: LoadError: break me", exc.msg) && rethrow() - end + try + Base.require(Main, :FooBar2) + error("the \"break me\" test failed") + catch exc + isa(exc, Base.Precompilation.PkgPrecompileError) || rethrow() + occursin("Failed to precompile FooBar2", exc.msg) || rethrow() + # The LoadError is printed to stderr in the precompilepkgs worker and captured in the PkgPrecompileError msg + occursin("LoadError: break me", exc.msg) || rethrow() + end # Test that trying to eval into closed modules during precompilation is an error FooBar3_file = joinpath(dir, "FooBar3.jl") @@ -699,11 +706,12 @@ precompile_test_harness(false) do dir $code end """) - @test_warn "Evaluation into the closed module `Base` breaks incremental compilation" try - Base.require(Main, :FooBar3) - catch exc - isa(exc, ErrorException) || rethrow() - end + try + Base.require(Main, :FooBar3) + catch exc + isa(exc, Base.Precompilation.PkgPrecompileError) || rethrow() + occursin("Evaluation into the closed module `Base` breaks incremental compilation", exc.msg) || rethrow() + end end # Test transitive dependency for #21266 @@ -773,12 +781,12 @@ precompile_test_harness("code caching") do dir Base.compilecache(pkgid) @test Base.isprecompiled(pkgid) @eval using $Cache_module - M = invokelatest(getfield, @__MODULE__, Cache_module) + M = invokelatest(getglobal, @__MODULE__, Cache_module) Mid = rootid(M) invokelatest() do # Test that this cache file "owns" all the roots for name in (:f, :fpush, :callboth) - func = getfield(M, name) + func = getglobal(M, name) m = only(collect(methods(func))) @test all(i -> root_provenance(m, i) == Mid, 1:length(m.roots)) end @@ -951,6 +959,23 @@ precompile_test_harness("code caching") do dir use_stale(c) = stale(c[1]) + not_stale("hello") build_stale(x) = use_stale(Any[x]) + # bindings + struct InvalidatedBinding + x::Int + end + struct Wrapper + ib::InvalidatedBinding + end + makewib(x) = Wrapper(InvalidatedBinding(x)) + const gib = makewib(1) + fib() = gib.ib.x + + struct LogBindingInvalidation + x::Int + end + const glbi = LogBindingInvalidation(1) + flbi() = @__MODULE__().glbi.x + # force precompilation build_stale(37) stale('c') @@ -975,11 +1000,15 @@ precompile_test_harness("code caching") do dir useA() = $StaleA.stale("hello") useA2() = useA() + useflbi() = $StaleA.flbi() + # force precompilation begin Base.Experimental.@force_compile useA2() + useflbi() end + precompile($StaleA.fib, ()) ## Reporting tests call_nbits(x::Integer) = $StaleA.nbits(x) @@ -1007,8 +1036,24 @@ precompile_test_harness("code caching") do dir Base.compilecache(Base.PkgId(string(pkg))) end @eval using $StaleA - MA = invokelatest(getfield, @__MODULE__, StaleA) + MA = invokelatest(getglobal, @__MODULE__, StaleA) Base.eval(MA, :(nbits(::UInt8) = 8)) + Base.eval(MA, quote + struct InvalidatedBinding + x::Float64 + end + struct Wrapper + ib::InvalidatedBinding + end + const gib = makewib(2.0) + end) + # TODO: test a "method_globalref" invalidation also + Base.eval(MA, quote + struct LogBindingInvalidation # binding invalidations can't be done during precompilation + x::Float64 + end + const glbi = LogBindingInvalidation(2.0) + end) @eval using $StaleC invalidations = Base.StaticData.debug_method_invalidation(true) @eval using $StaleB @@ -1039,6 +1084,10 @@ precompile_test_harness("code caching") do dir m = only(methods(MC.call_buildstale)) mi = m.specializations::Core.MethodInstance @test hasvalid(mi, world) # was compiled with the new method + m = only(methods(MA.fib)) + mi = m.specializations::Core.MethodInstance + @test !hasvalid(mi, world) # invalidated by redefining `gib` before loading StaleB + @test MA.fib() === 2.0 # Reporting test (ensure SnoopCompile works) @test all(i -> isassigned(invalidations, i), eachindex(invalidations)) @@ -1065,6 +1114,16 @@ precompile_test_harness("code caching") do dir @test !hasvalid(mi, world) @test any(x -> x isa Core.CodeInstance && x.def === mi, invalidations) + idxb = findfirst(x -> x isa Core.Binding, invalidations) + @test invalidations[idxb+1] == "insert_backedges_callee" + idxv = findnext(==("verify_methods"), invalidations, idxb) + if invalidations[idxv-1].def.def.name === :getproperty + idxv = findnext(==("verify_methods"), invalidations, idxv+1) + end + @test invalidations[idxv-1].def.def.name === :flbi + idxv = findnext(==("verify_methods"), invalidations, idxv+1) + @test invalidations[idxv-1].def.def.name === :useflbi + m = only(methods(MB.map_nbits)) @test !hasvalid(m.specializations::Core.MethodInstance, world+1) # insert_backedges invalidations also trigger their backedges end @@ -1097,7 +1156,7 @@ precompile_test_harness("precompiletools") do dir Base.compilecache(pkgid) @test Base.isprecompiled(pkgid) @eval using $PrecompileToolsModule - M = invokelatest(getfield, @__MODULE__, PrecompileToolsModule) + M = invokelatest(getglobal, @__MODULE__, PrecompileToolsModule) invokelatest() do m = which(Tuple{typeof(findfirst), Base.Fix2{typeof(==), T}, Vector{T}} where T) success = 0 @@ -1224,7 +1283,7 @@ precompile_test_harness("invoke") do dir """) Base.compilecache(Base.PkgId(string(CallerModule))) @eval using $InvokeModule: $InvokeModule - MI = invokelatest(getfield, @__MODULE__, InvokeModule) + MI = invokelatest(getglobal, @__MODULE__, InvokeModule) @eval $MI.getlast(a::UnitRange) = a.stop @eval using $CallerModule invokelatest() do @@ -1828,8 +1887,8 @@ end @testset "Precompile external abstract interpreter" begin dir = @__DIR__ - @test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint1.jl`; dir); stdout, stderr)) - @test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint2.jl`; dir); stdout, stderr)) + @test success(pipeline(Cmd(`$(Base.julia_cmd()) --startup-file=no precompile_absint1.jl`; dir); stdout, stderr)) + @test success(pipeline(Cmd(`$(Base.julia_cmd()) --startup-file=no precompile_absint2.jl`; dir); stdout, stderr)) end precompile_test_harness("Recursive types") do load_path @@ -1927,8 +1986,12 @@ precompile_test_harness("PkgCacheInspector") do load_path end modules, init_order, edges, new_ext_cis, external_methods, new_method_roots, cache_sizes = sv - m = only(external_methods).func::Method - @test m.name == :repl_cmd && m.nargs < 2 + for m in external_methods + m = m.func::Method + if m.name !== :f + @test m.name == :repl_cmd && m.nargs == 1 + end + end @test new_ext_cis === nothing || any(new_ext_cis) do ci mi = ci.def::Core.MethodInstance mi.specTypes == Tuple{typeof(Base.repl_cmd), Int, String} @@ -2114,6 +2177,29 @@ precompile_test_harness("No backedge precompile") do load_path end end +precompile_test_harness("Pre-compile Core methods") do load_path + # Core methods should support pre-compilation as external CI's like anything else + # https://github.com/JuliaLang/julia/issues/58497 + write(joinpath(load_path, "CorePrecompilation.jl"), + """ + module CorePrecompilation + struct Foo end + precompile(Tuple{Type{Vector{Foo}}, UndefInitializer, Tuple{Int}}) + end + """) + ji, ofile = Base.compilecache(Base.PkgId("CorePrecompilation")) + @eval using CorePrecompilation + invokelatest() do + let tt = Tuple{Type{Vector{CorePrecompilation.Foo}}, UndefInitializer, Tuple{Int}}, + match = first(Base._methods_by_ftype(tt, -1, Base.get_world_counter())), + mi = Base.specialize_method(match) + @test isdefined(mi, :cache) + @test mi.cache.max_world === typemax(UInt) + @test mi.cache.invoke != C_NULL + end + end +end + # Test precompilation of generated functions that return opaque closures # (with constprop marker set to false). precompile_test_harness("Generated Opaque") do load_path @@ -2337,7 +2423,7 @@ precompile_test_harness("llvmcall validation") do load_path using LLVMCall LLVMCall.do_llvmcall2() """ - @test readchomp(`$(Base.julia_cmd()) --pkgimages=no -E $(testcode)`) == repr(UInt32(0)) + @test readchomp(`$(Base.julia_cmd()) --startup-file=no --pkgimages=no -E $(testcode)`) == repr(UInt32(0)) # Now the regular way @eval using LLVMCall invokelatest() do @@ -2415,4 +2501,10 @@ let m = only(methods(Base.var"@big_str")) @test m.specializations === Core.svec() || !isdefined(m.specializations, :cache) end +# Issue #58841 - make sure we don't accidentally throw away code for inference +let io = IOBuffer() + run(pipeline(`$(Base.julia_cmd()) --startup-file=no --trace-compile=stderr -e 'f() = sin(1.) == 0. ? 1 : 0; exit(f())'`, stderr=io)) + @test isempty(String(take!(io))) +end + finish_precompile_test!() diff --git a/test/project/ScriptProject/Project.toml b/test/project/ScriptProject/Project.toml new file mode 100644 index 0000000000000..3301f2b79da83 --- /dev/null +++ b/test/project/ScriptProject/Project.toml @@ -0,0 +1,2 @@ +name = "ScriptProject" +uuid = "6646321a-c4de-46ad-9761-435e5bb1f223" diff --git a/test/project/ScriptProject/SubProject/Project.toml b/test/project/ScriptProject/SubProject/Project.toml new file mode 100644 index 0000000000000..e6c472c7a33f6 --- /dev/null +++ b/test/project/ScriptProject/SubProject/Project.toml @@ -0,0 +1,2 @@ +name = "SubProject" +uuid = "50d58d6a-5ae2-46f7-9677-83c51ca667d5" diff --git a/test/project/ScriptProject/bin/script.jl b/test/project/ScriptProject/bin/script.jl new file mode 100644 index 0000000000000..e38351c9ab9a8 --- /dev/null +++ b/test/project/ScriptProject/bin/script.jl @@ -0,0 +1 @@ +println(Base.active_project()) diff --git a/test/ranges.jl b/test/ranges.jl index 40dbf6a42dbd9..9bad12e6692d2 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -230,7 +230,7 @@ end @test cmp_sn2(Tw(xw+yw), astuple(x+y)..., slopbits) @test cmp_sn2(Tw(xw-yw), astuple(x-y)..., slopbits) @test cmp_sn2(Tw(xw*yw), astuple(x*y)..., slopbits) - @test cmp_sn2(Tw(xw/yw), astuple(x/y)..., slopbits) + @test cmp_sn2(Tw(xw/yw), astuple(x/y)..., slopbits+1) # extra bit because division is hard y = rand(T) yw = widen(widen(y)) @test cmp_sn2(Tw(xw+yw), astuple(x+y)..., slopbits) @@ -699,6 +699,18 @@ end @test Duck(4) ∈ Duck(1):Duck(5) @test Duck(0) ∉ Duck(1):Duck(5) end + @testset "unique" begin + struct MyStepRangeLen{T,R} <: AbstractRange{T} + x :: R + end + MyStepRangeLen(s::StepRangeLen{T}) where {T} = MyStepRangeLen{T,typeof(s)}(s) + Base.first(s::MyStepRangeLen) = first(s.x) + Base.last(s::MyStepRangeLen) = last(s.x) + Base.length(s::MyStepRangeLen) = length(s.x) + Base.step(s::MyStepRangeLen) = step(s.x) + sr = StepRangeLen(1,0,4) + @test unique(MyStepRangeLen(sr)) == unique(sr) + end end @testset "indexing range with empty range (#4309)" begin @test (@inferred (3:6)[5:4]) === 7:6 @@ -743,6 +755,8 @@ end @test length(typemin(T):typemax(T)) == T(0) @test length(zero(T):one(T):typemax(T)) == typemin(T) @test length(typemin(T):one(T):typemax(T)) == T(0) + @test length(StepRange{T,BigInt}(zero(T), 1, typemax(T))) == typemin(T) + @test length(StepRange{T,BigInt}(typemin(T), 1, typemax(T))) == T(0) @test_throws OverflowError checked_length(zero(T):typemax(T)) @test_throws OverflowError checked_length(typemin(T):typemax(T)) @test_throws OverflowError checked_length(zero(T):one(T):typemax(T)) diff --git a/test/rebinding.jl b/test/rebinding.jl index ab9696c7f0222..a54a7d833403b 100644 --- a/test/rebinding.jl +++ b/test/rebinding.jl @@ -285,7 +285,7 @@ module RangeMerge function get_llvm(@nospecialize(f), @nospecialize(t), raw=true, dump_module=false, optimize=true) params = Base.CodegenParams(safepoint_on_entry=false, gcstack_arg = false, debug_info_level=Cint(2)) - d = InteractiveUtils._dump_function(f, t, false, false, raw, dump_module, :att, optimize, :none, false, params) + d = InteractiveUtils._dump_function(InteractiveUtils.ArgInfo(f, t), false, false, raw, dump_module, :att, optimize, :none, false, params) sprint(print, d) end @@ -318,3 +318,77 @@ module UndefinedTransitions @test Base.Compiler.is_nothrow(Base.Compiler.decode_effects(ci.ipo_purity_bits)) end end + +# Identical implicit partitions should be merge (#57923) +for binding in (convert(Core.Binding, GlobalRef(Base, :Math)), + convert(Core.Binding, GlobalRef(Base, :Intrinsics))) + # Test that these both only have two partitions + @test isdefined(binding, :partitions) + @test isdefined(binding.partitions, :next) + @test !isdefined(binding.partitions.next, :next) +end + +# Test various scenarios for implicit partition merging +module MergeStress + for i = 1:5 + @eval module $(Symbol("M$i")) + export x, y + const x = 1 + const y = 2 + end + end + const before = Base.get_world_counter() + using .M1 + const afterM1 = Base.get_world_counter() + using .M2 + const afterM2 = Base.get_world_counter() + using .M3 + const afterM3 = Base.get_world_counter() + using .M4 + const afterM4 = Base.get_world_counter() + using .M5 + const afterM5 = Base.get_world_counter() +end + +function count_partitions(b::Core.Binding) + n = 0 + isdefined(b, :partitions) || return n + bpart = b.partitions + while true + n += 1 + isdefined(bpart, :next) || break + bpart = bpart.next + end + return n +end +using Base: invoke_in_world + +const xbinding = convert(Core.Binding, GlobalRef(MergeStress, :x)) +function access_and_count(point) + invoke_in_world(getglobal(MergeStress, point), getglobal, MergeStress, :x) + count_partitions(xbinding) +end + +@test count_partitions(xbinding) == 0 +@test access_and_count(:afterM1) == 1 +# M2 is the first change to the `usings` table after M1. The partitions +# can and should be merged +@test access_and_count(:afterM2) == 1 + +# There is a gap between M2 and M5 - the partitions should not be merged +@test access_and_count(:afterM5) == 2 + +# M4 and M5 are adjacent, these partitions should also be merged (in the opposite direction) +@test access_and_count(:afterM4) == 2 + +# M3 connects all, so we should have a single partition +@test access_and_count(:afterM3) == 1 + +# Test that delete_binding in an outdated world age works +module BindingTestModule; end +function create_and_delete_binding() + Core.eval(BindingTestModule, :(const x = 1)) + Base.delete_binding(BindingTestModule, :x) +end +create_and_delete_binding() +@test Base.binding_kind(BindingTestModule, :x) == Base.PARTITION_KIND_GUARD diff --git a/test/reduce.jl b/test/reduce.jl index f5140c8a34bd9..a1d66a03e2c42 100644 --- a/test/reduce.jl +++ b/test/reduce.jl @@ -299,19 +299,37 @@ end arr = zeros(N) @test minimum(arr) === 0.0 @test maximum(arr) === 0.0 + @test minimum(abs, arr) === 0.0 + @test maximum(abs, arr) === 0.0 + @test minimum(-, arr) === -0.0 + @test maximum(-, arr) === -0.0 arr[i] = -0.0 @test minimum(arr) === -0.0 @test maximum(arr) === 0.0 + @test minimum(abs, arr) === 0.0 + @test maximum(abs, arr) === 0.0 + @test minimum(-, arr) === -0.0 + @test maximum(-, arr) === 0.0 arr = -zeros(N) @test minimum(arr) === -0.0 @test maximum(arr) === -0.0 + @test minimum(abs, arr) === 0.0 + @test maximum(abs, arr) === 0.0 + @test minimum(-, arr) === 0.0 + @test maximum(-, arr) === 0.0 arr[i] = 0.0 @test minimum(arr) === -0.0 - @test maximum(arr) === 0.0 + @test maximum(arr) === 0.0 + @test minimum(abs, arr) === 0.0 + @test maximum(abs, arr) === 0.0 + @test minimum(-, arr) === -0.0 + @test maximum(-, arr) === 0.0 end end + + @test minimum(abs, fill(-0.0, 16)) === mapreduce(abs, (x,y)->min(x,y), fill(-0.0, 16)) === 0.0 end @testset "maximum works on generic order #30320" begin @@ -568,11 +586,11 @@ struct NonFunctionIsZero end @test count(NonFunctionIsZero(), [0]) == 1 @test count(NonFunctionIsZero(), [1]) == 0 -@test count(Iterators.repeated(true, 3), init=0x04) === 0x07 -@test count(!=(2), Iterators.take(1:7, 3), init=Int32(0)) === Int32(2) -@test count(identity, [true, false], init=Int8(5)) === Int8(6) -@test count(!, [true false; false true], dims=:, init=Int16(0)) === Int16(2) -@test isequal(count(identity, [true false; false true], dims=2, init=UInt(4)), reshape(UInt[5, 5], 2, 1)) +@test count(Iterators.repeated(true, 3), init=UInt(0)) === UInt(3) +@test count(!=(2), Iterators.take(1:7, 3), init=Int32(0)) === 2 +@test count(identity, [true, false], init=Int8(0)) === 1 +@test count(!, [true false; false true], dims=:, init=Int16(0)) === 2 +@test isequal(count(identity, [true false; false true], dims=2, init=UInt(0)), reshape(UInt[1, 1], 2, 1)) ## cumsum, cummin, cummax @@ -681,6 +699,27 @@ end end end +@testset "issue #45562" begin + @test all([true, true, true], dims = 1) == [true] + @test any([true, true, true], dims = 1) == [true] + @test_throws TypeError all([3, 3, 3], dims = 1) + @test_throws TypeError any([3, 3, 3], dims = 1) + @test_throws TypeError all(Any[true, 3, 3], dims = 1) + @test_throws TypeError any(Any[false, 3, 3], dims = 1) + @test_throws TypeError all([1, 1, 1], dims = 1) + @test_throws TypeError any([0, 0, 0], dims = 1) + @test_throws TypeError all!([false], [3, 3, 3]) + @test_throws TypeError any!([false], [3, 3, 3]) + @test_throws TypeError all!([false], Any[true, 3, 3]) + @test_throws TypeError any!([false], Any[false, 3, 3]) + @test_throws TypeError all!([false], [1, 1, 1]) + @test_throws TypeError any!([false], [0, 0, 0]) + @test reduce(|, Bool[]) == false + @test reduce(&, Bool[]) == true + @test reduce(|, Bool[], dims=1) == [false] + @test reduce(&, Bool[], dims=1) == [true] +end + # issue #45748 @testset "foldl's stability for nested Iterators" begin a = Iterators.flatten((1:3, 1:3)) @@ -732,3 +771,9 @@ end Val(any(in((:one,:two,:three)),(:four,:three))) end |> only == Val{true} end + +# `reduce(vcat, A)` should not alias the input for length-1 collections +let A=[1;;] + @test reduce(vcat, Any[A]) !== A + @test reduce(hcat, Any[A]) !== A +end diff --git a/test/reducedim.jl b/test/reducedim.jl index 6a6f20214058c..1664b2708d7e3 100644 --- a/test/reducedim.jl +++ b/test/reducedim.jl @@ -88,12 +88,12 @@ safe_minabs(A::Array{T}, region) where {T} = safe_mapslices(minimum, abs.(A), re @test @inferred(count(!, Breduc, dims=region)) ≈ safe_count(.!Breduc, region) @test isequal( - @inferred(count(Breduc, dims=region, init=0x02)), - safe_count(Breduc, region) .% UInt8 .+ 0x02, + @inferred(Array{UInt8,ndims(Breduc)}, count(Breduc, dims=region, init=0x00)), + safe_count(Breduc, region), ) @test isequal( - @inferred(count(!, Breduc, dims=region, init=Int16(0))), - safe_count(.!Breduc, region) .% Int16, + @inferred(Array{Int16,ndims(Breduc)}, count(!, Breduc, dims=region, init=Int16(0))), + safe_count(.!Breduc, region), ) end @@ -693,7 +693,7 @@ end @test_throws TypeError count!([1], [1]) end -@test @inferred(count(false:true, dims=:, init=0x0004)) === 0x0005 +@test @inferred(UInt16, count(false:true, dims=:, init=0x0000)) === 1 @test @inferred(count(isodd, reshape(1:9, 3, 3), dims=:, init=Int128(0))) === Int128(5) @testset "reduced_index for BigInt (issue #39995)" begin diff --git a/test/reflection.jl b/test/reflection.jl index 345e219b41a3e..5a0cd2e8645f7 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -16,6 +16,11 @@ function test_ir_reflection(freflect, f, types) nothing end +function test_ir_reflection(freflect, argtypes) + @test !isempty(freflect(argtypes)) + nothing +end + function test_bin_reflection(freflect, f, types) iob = IOBuffer() freflect(iob, f, types) @@ -27,6 +32,9 @@ end function test_code_reflection(freflect, f, types, tester) tester(freflect, f, types) tester(freflect, f, (types.parameters...,)) + tt = Base.signature_type(f, types) + tester(freflect, tt) + tester(freflect, (tt.parameters...,)) nothing end @@ -43,6 +51,7 @@ end test_code_reflections(test_ir_reflection, code_lowered) test_code_reflections(test_ir_reflection, code_typed) +test_code_reflections(test_ir_reflection, Base.code_ircode) io = IOBuffer() Base.print_statement_costs(io, map, (typeof(sqrt), Tuple{Int})) @@ -346,6 +355,7 @@ tlayout = TLayout(5,7,11) @test !hasproperty(tlayout, :p) @test [(fieldoffset(TLayout,i), fieldname(TLayout,i), fieldtype(TLayout,i)) for i = 1:fieldcount(TLayout)] == [(0, :x, Int8), (2, :y, Int16), (4, :z, Int32)] +@test [fieldoffset(TLayout, s) for s = (:x, :y, :z)] == [0, 2, 4] @test fieldnames(Complex) === (:re, :im) @test_throws BoundsError fieldtype(TLayout, 0) @test_throws ArgumentError fieldname(TLayout, 0) @@ -361,6 +371,10 @@ tlayout = TLayout(5,7,11) @test fieldtype(Union{Tuple{Char},Tuple{Char,Char}},2) === Char @test_throws BoundsError fieldtype(Union{Tuple{Char},Tuple{Char,Char}},3) +@test [fieldindex(TLayout, i) for i = (:x, :y, :z)] == [1, 2, 3] +@test fieldname(TLayout, fieldindex(TLayout, :z)) === :z +@test fieldindex(TLayout, fieldname(TLayout, 3)) === 3 + @test fieldnames(NTuple{3, Int}) == ntuple(i -> fieldname(NTuple{3, Int}, i), 3) == (1, 2, 3) @test_throws ArgumentError fieldnames(Union{}) @test_throws BoundsError fieldname(NTuple{3, Int}, 0) @@ -523,13 +537,13 @@ test_typed_ir_printing(g15714, Tuple{Vector{Float32}}, #@test used_dup_var_tested15715 @test used_unique_var_tested15714 -let li = typeof(fieldtype).name.mt.cache.func::Core.MethodInstance, +let li = only(methods(fieldtype)).unspecialized, lrepr = string(li), mrepr = string(li.def), lmime = repr("text/plain", li), mmime = repr("text/plain", li.def) - @test lrepr == lmime == "MethodInstance for fieldtype(...)" + @test lrepr == lmime == "MethodInstance for fieldtype(::Vararg{Any})" @test mrepr == "fieldtype(...) @ Core none:0" # simple print @test mmime == "fieldtype(...)\n @ Core none:0" # verbose print end @@ -572,6 +586,32 @@ fLargeTable(::Union, ::Union) = "b" @test length(methods(fLargeTable)) == 205 @test fLargeTable(Union{Int, Missing}, Union{Int, Missing}) == "b" +# issue #58479 +fLargeTable(::Type) = "Type" +fLargeTable(::Type{<:DataType}) = "DataType" +@test fLargeTable(Type) == "Type" +@test fLargeTable(DataType) == "DataType" +@test fLargeTable(Type{DataType}) == "DataType" +@test fLargeTable(Type{UnionAll}) == "DataType" +@test fLargeTable(Type{Int}) == "DataType" +@test fLargeTable(Type{Vector}) == "Type" +@test fLargeTable(Type{Type{Union{}}}) == "DataType" +@test fLargeTable(Type{Union{}}) == "Type" +@test fLargeTable(Union{}) == "DataType" +@test fLargeTable(Type{<:DataType}) == "Type" +fLargeTable(::Type{<:UnionAll}) = "UnionAll" +@test fLargeTable(UnionAll) == "UnionAll" +@test fLargeTable(Type{Vector}) == "UnionAll" +@test fLargeTable(Type{Int}) == "DataType" +@test fLargeTable(Type{Type{Union{}}}) == "DataType" +@test fLargeTable(Type{Union{}}) == "Type" +@test_throws MethodError fLargeTable(Union{}) +@test fLargeTable(Type{<:DataType}) == "Type" +@test fLargeTable(Type{Vector{T}} where T) == "DataType" +@test fLargeTable(Union{DataType,Type{Vector{T}} where T}) == "DataType" +@test fLargeTable(Union{DataType,UnionAll,Type{Vector{T}} where T}) == "Type" +@test fLargeTable(Union{Type{Vector},Type{Vector{T}} where T}) == "Type" + # issue #15280 function f15280(x) end @test functionloc(f15280)[2] > 0 @@ -586,9 +626,9 @@ function module_depth(from::Module, to::Module) end function has_backslashes(mod::Module) for n in names(mod, all = true, imported = true) - isdefined(mod, n) || continue + isdefinedglobal(mod, n) || continue Base.isdeprecated(mod, n) && continue - f = getfield(mod, n) + f = getglobal(mod, n) if isa(f, Module) && module_depth(Main, f) <= module_depth(Main, mod) continue end @@ -651,6 +691,10 @@ end @test Base.code_typed_by_type(Tuple{Type{<:Val}})[2][2] == Val @test Base.code_typed_by_type(Tuple{typeof(sin), Float64})[1][2] === Float64 +# functor-like code_typed(...) +@test Base.code_typed((Type{<:Val},))[2][2] == Val +@test Base.code_typed((typeof(sin), Float64))[1][2] === Float64 + # New reflection methods in 0.6 struct ReflectionExample{T<:AbstractFloat, N} x::Tuple{T, N} @@ -931,6 +975,7 @@ f(x::Int; y=3) = x + y @test hasmethod(f, Tuple{Int}) @test hasmethod(f, Tuple{Int}, ()) @test hasmethod(f, Tuple{Int}, (:y,)) +@test !hasmethod(f, Tuple{Int}, (:x,)) @test !hasmethod(f, Tuple{Int}, (:jeff,)) @test !hasmethod(f, Tuple{Int}, (:y,), world=typemin(UInt)) g(; b, c, a) = a + b + c @@ -1006,11 +1051,12 @@ _test_at_locals2(1,1,0.5f0) @testset "issue #31687" begin import InteractiveUtils._dump_function + import InteractiveUtils.ArgInfo @noinline f31687_child(i) = f31687_nonexistent(i) f31687_parent() = f31687_child(0) params = Base.CodegenParams() - _dump_function(f31687_parent, Tuple{}, + _dump_function(ArgInfo(f31687_parent, Tuple{}), #=native=#false, #=wrapper=#false, #=raw=#true, #=dump_module=#true, #=syntax=#:att, #=optimize=#false, :none, #=binary=#false) @@ -1099,9 +1145,12 @@ end @test 1+1 == 2 mi1 = Base.method_instance(+, (Int, Int)) @test mi1.def.name == :+ - # Note `jl_method_lookup` doesn't returns CNull if not found - mi2 = @ccall jl_method_lookup(Any[+, 1, 1]::Ptr{Any}, 3::Csize_t, Base.get_world_counter()::Csize_t)::Ref{Core.MethodInstance} - @test mi1 == mi2 + mi2 = Base.method_instance((typeof(+), Int, Int)) + @test mi2.def.name == :+ + # Note `jl_method_lookup` doesn't return CNull if not found + mi3 = @ccall jl_method_lookup(Any[+, 1, 1]::Ptr{Any}, 3::Csize_t, Base.get_world_counter()::Csize_t)::Ref{Core.MethodInstance} + @test mi1 == mi3 + @test mi2 == mi3 end Base.@assume_effects :terminates_locally function issue41694(x::Int) @@ -1211,6 +1260,8 @@ end @test Base.ismutationfree(Type{Union{}}) +@test !Base.ismutationfree(Core.SimpleVector) + module TestNames public publicized diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index e6381329e4ec6..f6269792ef0a6 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -10,17 +10,24 @@ tslow(a::AbstractArray) = TSlow(a) wrapper(a::AbstractArray) = WrapperArray(a) fcviews(a::AbstractArray) = view(a, ntuple(Returns(:),ndims(a)-1)..., axes(a)[end]) fcviews(a::AbstractArray{<:Any, 0}) = view(a) +offset_nominal(a::AbstractArray) = OffsetArray(a) +offset_maybe(a::AbstractArray) = (eltype(a) <: Real) ? a : OffsetArray(a, (1-ndims(A)):2:(ndims(A)-1)...) tslow(t::Tuple) = map(tslow, t) wrapper(t::Tuple) = map(wrapper, t) fcviews(t::Tuple) = map(fcviews, t) +offset_nominal(t::Tuple) = map(offset_nominal, t) +offset_maybe(t::Tuple) = map(offset_maybe, t) test_many_wrappers(testf, A, wrappers) = foreach(w -> testf(w(A)), wrappers) -test_many_wrappers(testf, A) = test_many_wrappers(testf, A, (identity, tslow, wrapper, fcviews)) +test_many_wrappers(testf, A) = test_many_wrappers( + testf, A, (identity, tslow, wrapper, fcviews, offset_nominal, offset_maybe) +) A = Int64[1, 2, 3, 4] Ars = Int64[1 3; 2 4] B = Complex{Int64}[5+6im, 7+8im, 9+10im] Av = [Int32[1,2], Int32[3,4]] +C = view([1,1], [1,2]) test_many_wrappers(Ars, (identity, tslow)) do Ar @test @inferred(ndims(reinterpret(reshape, Complex{Int64}, Ar))) == 1 @@ -64,6 +71,44 @@ test_many_wrappers(B) do _B @test reinterpret(reshape, Int64, _B) == [5 7 9; 6 8 10] end +@testset "setindex! converts before reinterpreting" begin + for dims in ((), 1) + z = reinterpret(UInt64, fill(1.0, dims)) + @test z[] == z[1] == 0x3ff0000000000000 + z[] = Int32(1)//Int32(1) + @test z[] == z[1] == 0x0000000000000001 + z[1] = Int32(2)//Int32(1) + @test z[] == z[1] == 0x0000000000000002 + z[1] = 3//1 + @test z[] == z[1] == 0x0000000000000003 + @test_throws InexactError z[] = 3//2 + @test_throws InexactError z[] = 1.5 + @test_throws InexactError z[1] = 3//2 + @test_throws InexactError z[1] = 1.5 + + z = reinterpret(UInt64, fill(Int32(16)//Int32(1), dims)) + @test z[] == z[1] == 0x0000000100000010 + z[] = Int32(1)//Int32(1) + @test z[] == z[1] == 0x0000000000000001 + z[1] = Int32(2)//Int32(1) + @test z[] == z[1] == 0x0000000000000002 + z[1] = 3//1 + @test z[] == z[1] == 0x0000000000000003 + @test_throws InexactError z[] = 3//2 + @test_throws InexactError z[] = 1.5 + @test_throws InexactError z[1] = 3//2 + @test_throws InexactError z[1] = 1.5 + + z = reinterpret(Missing, fill(nothing, dims)) + @test z[] === missing + @test z[1] === missing + @test_throws "cannot convert" z[] = nothing + @test_throws "cannot convert" z[1] = nothing + @test z[] === missing + @test z[1] === missing + end +end + # setindex test_many_wrappers((A, Ars, B)) do (A, Ars, B) _A, Ar, _B = deepcopy(A), deepcopy(Ars), deepcopy(B) @@ -117,6 +162,18 @@ test_many_wrappers(A3) do A3_ @test A3[2,1,2] == 400 end +test_many_wrappers(C) do Cr_ + Cr = deepcopy(Cr_) + r = reinterpret(reshape, Tuple{Int, Int}, Cr) + @test r == fill((1,1)) + r[] = (2,2) + @test r[] === (2,2) + r[1] = (3,3) + @test r[1] === (3,3) + r[1,1] = (4,4) + @test r[1,1] === (4,4) +end + # same-size reinterpret where one of the types is non-primitive let a = NTuple{4,UInt8}[(0x01,0x02,0x03,0x04)] test_many_wrappers(a, (identity, wrapper, fcviews)) do a_ @@ -272,7 +329,7 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ fill!(r, 2) @test all(a .=== reinterpret(Float64, [Int64(2)])[1]) @test all(r .=== Int64(2)) - for badinds in (0, 16, (0,1), (1,0), (6,3), (5,4)) + for badinds in ((), 0, 16, (0,1), (1,0), (6,3), (5,4)) @test_throws BoundsError r[badinds...] @test_throws BoundsError r[badinds...] = -2 end @@ -285,7 +342,7 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ fill!(r, 3) @test all(a .=== reinterpret(Float64, [(Int32(3), Int32(3))])[1]) @test all(r .=== Int32(3)) - for badinds in (0, 31, (0,1), (1,0), (11,3), (10,4)) + for badinds in ((), 0, 31, (0,1), (1,0), (11,3), (10,4)) @test_throws BoundsError r[badinds...] @test_throws BoundsError r[badinds...] = -3 end @@ -298,7 +355,7 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ fill!(r, 4) @test all(a[1:2:5,:] .=== reinterpret(Float64, [Int64(4)])[1]) @test all(r .=== Int64(4)) - for badinds in (0, 10, (0,1), (1,0), (4,3), (3,4)) + for badinds in ((), 0, 10, (0,1), (1,0), (4,3), (3,4)) @test_throws BoundsError r[badinds...] @test_throws BoundsError r[badinds...] = -4 end @@ -311,7 +368,7 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ fill!(r, 5) @test all(a[1:2:5,:] .=== reinterpret(Float64, [(Int32(5), Int32(5))])[1]) @test all(r .=== Int32(5)) - for badinds in (0, 19, (0,1), (1,0), (7,3), (6,4)) + for badinds in ((), 0, 19, (0,1), (1,0), (7,3), (6,4)) @test_throws BoundsError r[badinds...] @test_throws BoundsError r[badinds...] = -5 end @@ -320,6 +377,25 @@ test_many_wrappers(fill(1.0, 5, 3), (identity, wrapper)) do a_ @test r[goodinds...] == -5 end end + +let a = rand(ComplexF32, 5) + r = reinterpret(reshape, Float32, a) + ref = Array(r) + + @test all(r .== OffsetArray(r)[:, :, :]) + + @test r[1, :, 1] == ref[1, :] + @test r[1, :, 1, 1, 1] == ref[1, :] + @test r[1, :, UInt8(1)] == ref[1, :] + + r[2, :, 1] .= 0f0 + ref[2, :] .= 0f0 + @test r[2, :, 1] == ref[2, :] + + @test r[4] == ref[4] + @test_throws BoundsError r[1, :, 2] +end + let ar = [(1,2), (3,4)] arr = reinterpret(reshape, Int, ar) @test @inferred(IndexStyle(arr)) == Base.IndexSCartesian2{2}() @@ -607,3 +683,9 @@ let R = reinterpret(reshape, Float32, ComplexF32[1.0f0+2.0f0*im, 4.0f0+3.0f0*im] @test !isassigned(R, 5) @test Array(R)::Matrix{Float32} == [1.0f0 4.0f0; 2.0f0 3.0f0] end + +@testset "issue #54623" begin + x = 0xabcdef01234567 + @test reinterpret(reshape, UInt8, fill(x)) == [0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x00] + @test reinterpret(reshape, UInt8, [x]) == [0x67; 0x45; 0x23; 0x01; 0xef; 0xcd; 0xab; 0x00;;] +end diff --git a/test/runtests.jl b/test/runtests.jl index 60139a4691bd7..ba64c66172126 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -13,7 +13,7 @@ include("choosetests.jl") include("testenv.jl") include("buildkitetestjson.jl") -(; tests, net_on, exit_on_error, use_revise, seed) = choosetests(ARGS) +(; tests, net_on, exit_on_error, use_revise, buildroot, seed) = choosetests(ARGS) tests = unique(tests) if Sys.islinux() @@ -26,8 +26,15 @@ else end if use_revise + # First put this at the top of the DEPOT PATH to install revise if necessary. + # Once it's loaded, we swizzle it to the end, to avoid confusing any tests. + pushfirst!(DEPOT_PATH, joinpath(buildroot, "deps", "jlutilities", "depot")) + using Pkg + Pkg.activate(joinpath(@__DIR__, "..", "deps", "jlutilities", "revise")) + Pkg.instantiate() using Revise union!(Revise.stdlib_names, Symbol.(STDLIBS)) + push!(DEPOT_PATH, popfirst!(DEPOT_PATH)) # Remote-eval the following to initialize Revise in workers const revise_init_expr = quote using Revise @@ -37,6 +44,11 @@ if use_revise end end +if isempty(tests) + println("No tests selected. Exiting.") + exit() +end + const max_worker_rss = if haskey(ENV, "JULIA_TEST_MAXRSS_MB") parse(Int, ENV["JULIA_TEST_MAXRSS_MB"]) * 2^20 else @@ -123,6 +135,7 @@ cd(@__DIR__) do Sys.CPU_THREADS = $(Sys.CPU_THREADS) Sys.total_memory() = $(Base.format_bytes(Sys.total_memory())) Sys.free_memory() = $(Base.format_bytes(Sys.free_memory())) + Sys.uptime() = $(Sys.uptime()) ($(round(Sys.uptime() / (60 * 60), digits=1)) hours) """) #pretty print the information about gc and mem usage @@ -342,8 +355,6 @@ cd(@__DIR__) do end end - BuildkiteTestJSON.write_testset_json_files(@__DIR__) - #= ` Construct a testset on the master node which will hold results from all the test files run on workers and on node1. The loop goes through the results, @@ -369,6 +380,7 @@ cd(@__DIR__) do Test.TESTSET_PRINT_ENABLE[] = false o_ts = Test.DefaultTestSet("Overall") o_ts.time_end = o_ts.time_start + o_ts_duration # manually populate the timing + BuildkiteTestJSON.write_testset_json_files(@__DIR__, o_ts) Test.push_testset(o_ts) completed_tests = Set{String}() for (testname, (resp,), duration) in results @@ -403,7 +415,7 @@ cd(@__DIR__) do # deserialization errors or something similar. Record this testset as Errored. fake = Test.DefaultTestSet(testname) fake.time_end = fake.time_start + duration - Test.record(fake, Test.Error(:nontest_error, testname, nothing, Any[(resp, [])], LineNumberNode(1))) + Test.record(fake, Test.Error(:nontest_error, testname, nothing, Base.ExceptionStack(NamedTuple[(;exception = resp, backtrace = [])]), LineNumberNode(1), nothing)) Test.push_testset(fake) Test.record(o_ts, fake) Test.pop_testset() @@ -412,7 +424,7 @@ cd(@__DIR__) do for test in all_tests (test in completed_tests) && continue fake = Test.DefaultTestSet(test) - Test.record(fake, Test.Error(:test_interrupted, test, nothing, [("skipped", [])], LineNumberNode(1))) + Test.record(fake, Test.Error(:test_interrupted, test, nothing, Base.ExceptionStack(NamedTuple[(;exception = "skipped", backtrace = [])]), LineNumberNode(1), nothing)) Test.push_testset(fake) Test.record(o_ts, fake) Test.pop_testset() diff --git a/test/ryu.jl b/test/ryu.jl index 05eedef9a0da2..e885d6c10838f 100644 --- a/test/ryu.jl +++ b/test/ryu.jl @@ -19,6 +19,34 @@ todouble(sign, exp, mant) = Core.bitcast(Float64, (UInt64(sign) << 63) | (UInt64 @test Ryu.writeshortest(-Inf) == "-Inf" end +@testset "OutputOptions" begin + # plus + @test "+1" == Base.Ryu.writeshortest(1.0, true, false, false) + @test "-1" == Base.Ryu.writeshortest(-1.0, true, false, false) + + # space + @test " 1" == Ryu.writeshortest(1.0, false, true, false) + + # hash + @test "0" == Ryu.writeshortest(0.0, false, false, false) + + # precision + @test "9.9900" == Ryu.writeshortest(9.99, false, false, true, 5) + @test "1." == Ryu.writeshortest(1.0, false, false, true, 1) + + # expchar + @test "1.0d6" == Ryu.writeshortest(1e6, false, false, true, -1, UInt8('d')) + + # padexp + @test "3.0e+08" == Ryu.writeshortest(3e8, false, false, true, -1, UInt8('e'), true) + + # decchar + @test "3,14" == Ryu.writeshortest(3.14, false, false, true, -1, UInt8('e'), false, UInt8(',')) + + # compact + @test "0.333333" == Ryu.writeshortest(1/3, false, false, true, -1, UInt8('e'), false, UInt8('.'), false, true) +end + @testset "SwitchToSubnormal" begin @test "2.2250738585072014e-308" == Ryu.writeshortest(2.2250738585072014e-308) end @@ -241,6 +269,17 @@ end # Float64 @test "-Inf" == Ryu.writeshortest(Float32(-Inf)) end +@testset "OutputOptions" begin + # typed + @test "1.0f0" == Ryu.writeshortest(Float32(1.0), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "Inf32" == Ryu.writeshortest(Float32(Inf), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "NaN32" == Ryu.writeshortest(Float32(NaN), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "3.14f0" == Ryu.writeshortest(Float32(3.14), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + + # typed and no-hash + @test "1f0" == Ryu.writeshortest(1.0f0, false, false, false, -1, UInt8('e'), false, UInt8('.'), true) +end + @testset "SwitchToSubnormal" begin @test "1.1754944e-38" == Ryu.writeshortest(1.1754944f-38) end @@ -341,6 +380,17 @@ end # Float32 @test "-Inf" == Ryu.writeshortest(Float16(-Inf)) end +@testset "OutputOptions" begin + # typed + @test "Float16(1.0)" == Ryu.writeshortest(Float16(1.0), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "Inf16" == Ryu.writeshortest(Float16(Inf), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "NaN16" == Ryu.writeshortest(Float16(NaN), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + @test "Float16(3.14)" == Ryu.writeshortest(Float16(3.14), false, false, true, -1, UInt8('e'), false, UInt8('.'), true) + + # typed and no-hash + @test "Float16(1)" == Ryu.writeshortest(Float16(1.0), false, false, false, -1, UInt8('e'), false, UInt8('.'), true) +end + let x=floatmin(Float16) while x <= floatmax(Float16) @test parse(Float16, Ryu.writeshortest(x)) == x diff --git a/test/show.jl b/test/show.jl index 24a7b574f759c..60d0538e71a07 100644 --- a/test/show.jl +++ b/test/show.jl @@ -8,6 +8,8 @@ include("testenv.jl") replstr(x, kv::Pair...) = sprint((io,x) -> show(IOContext(io, :limit => true, :displaysize => (24, 80), kv...), MIME("text/plain"), x), x) showstr(x, kv::Pair...) = sprint((io,x) -> show(IOContext(io, :limit => true, :displaysize => (24, 80), kv...), x), x) +const IRShow = Base.Compiler.IRShow + @testset "IOContext" begin io = IOBuffer() ioc = IOContext(io) @@ -703,7 +705,7 @@ let oldout = stdout, olderr = stderr redirect_stderr(olderr) close(wrout) close(wrerr) - @test fetch(out) == "primitive type Int64 <: Signed\nTESTA\nTESTB\nΑ1Β2\"A\"\nA\n123\"C\"\n" + @test fetch(out) == "primitive type Int64 <: Signed\nTESTA\nTESTB\nΑ1Β2\"A\"\nA\n123.0000000000000000\"C\"\n" @test fetch(err) == "TESTA\nTESTB\nΑ1Β2\"A\"\n" finally redirect_stdout(oldout) @@ -858,7 +860,7 @@ struct S45879{P} end let ms = methods(S45879) @test ms isa Base.MethodList @test length(ms) == 0 - @test sprint(show, Base.MethodList(Method[], typeof(S45879).name.mt)) isa String + @test sprint(show, Base.MethodList(Method[], typeof(S45879).name)) isa String end function f49475(a=12.0; b) end @@ -1570,8 +1572,61 @@ struct var"%X%" end # Invalid name without '#' typeof(+), var"#f#", typeof(var"#f#"), + + # Integers should round-trip (#52677) + 1, UInt(1), + Int8(1), Int16(1), Int32(1), Int64(1), + UInt8(1), UInt16(1), UInt32(1), UInt64(1), + + # Float round-trip + Float16(1), Float32(1), Float64(1), + Float16(1.5), Float32(1.5), Float64(1.5), + Float16(0.4893243538921085), Float32(0.4893243538921085), Float64(0.4893243538921085), + # Examples that require the full 5, 9, and 17 digits of precision + Float16(0.00010014), Float32(1.00000075f-36), Float64(-1.561051336605761e-182), + floatmax(Float16), floatmax(Float32), floatmax(Float64), + floatmin(Float16), floatmin(Float32), floatmin(Float64), + Float16(0.0), 0.0f0, 0.0, + Float16(-0.0), -0.0f0, -0.0, + Inf16, Inf32, Inf, + -Inf16, -Inf32, -Inf, + nextfloat(Float16(0)), nextfloat(Float32(0)), nextfloat(Float64(0)), + NaN16, NaN32, NaN, + Float16(1e3), 1f7, 1e16, + Float16(-1e3), -1f7, -1e16, + Float16(1e4), 1f8, 1e17, + Float16(-1e4), -1f8, -1e17, + + # Pointers should round-trip + Ptr{Cvoid}(0), Ptr{Cvoid}(typemax(UInt)), Ptr{Any}(0), Ptr{Any}(typemax(UInt)), + + # :var"" escaping rules differ from strings (#58484) + :foo, + :var"bar baz", + :var"a $b", # No escaping for $ in raw string + :var"a\b", # No escaping for backslashes in middle + :var"a\\", # Backslashes must be escaped at the end + :var"a\\\\", + :var"a\"b", + :var"a\"", + :var"\\\"", + :+, :var"+-", + :(=), :(:), :(::), # Requires quoting + Symbol("a\nb"), + + Val(Float16(1.0)), Val(1f0), Val(1.0), + Val(:abc), Val(:(=)), Val(:var"a\b"), + + Val(1), Val(Int8(1)), Val(Int16(1)), Val(Int32(1)), Val(Int64(1)), Val(Int128(1)), + Val(UInt(1)), Val(UInt8(1)), Val(UInt16(1)), Val(UInt32(1)), Val(UInt64(1)), Val(UInt128(1)), + + # BROKEN + # Symbol("a\xffb"), + # User-defined primitive types + # Non-canonical NaNs + # BFloat16 ) - @test v == eval(Meta.parse(static_shown(v))) + @test v === eval(Meta.parse(static_shown(v))) end end @@ -1598,7 +1653,7 @@ struct f_with_params{t} <: Function end end let io = IOBuffer() - show(io, MIME"text/html"(), ModFWithParams.f_with_params.body.name.mt) + show(io, MIME"text/html"(), methods(ModFWithParams.f_with_params{Int}())) @test occursin("ModFWithParams.f_with_params", String(take!(io))) end @@ -1708,6 +1763,13 @@ end "[3.141592653589793 3.141592653589793; 3.141592653589793 3.141592653589793]" end +@testset "`displaysize` return type inference" begin + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{}) + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{IO}) + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{IOContext}) + @test Tuple{Int, Int} === Base.infer_return_type(displaysize, Tuple{Base.TTY}) +end + @testset "Array printing with limited rows" begin arrstr = let buf = IOBuffer() function (A, rows) @@ -1773,10 +1835,10 @@ end anonfn_type_repr = "$modname.var\"$(typeof(anonfn).name.name)\"" @test repr(typeof(anonfn)) == anonfn_type_repr @test repr(anonfn) == anonfn_type_repr * "()" - @test repr("text/plain", anonfn) == "$(typeof(anonfn).name.mt.name) (generic function with 1 method)" + @test repr("text/plain", anonfn) == "$(typeof(anonfn).name.singletonname) (generic function with 1 method)" mkclosure = x->y->x+y clo = mkclosure(10) - @test repr("text/plain", clo) == "$(typeof(clo).name.mt.name) (generic function with 1 method)" + @test repr("text/plain", clo) == "$(typeof(clo).name.singletonname) (generic function with 1 method)" @test repr(UnionAll) == "UnionAll" end @@ -1950,9 +2012,9 @@ end @test replstr(view(A, [1], :)) == "1×1 view(::Matrix{Float64}, [1], :) with eltype Float64:\n 0.0" # issue #27680 - @test showstr(Set([(1.0,1.0), (2.0,2.0), (3.0, 3.0)])) == (sizeof(Int) == 8 ? - "Set([(1.0, 1.0), (3.0, 3.0), (2.0, 2.0)])" : - "Set([(1.0, 1.0), (2.0, 2.0), (3.0, 3.0)])") + @test showstr(Set([(1.0, 1.0), (2.0, 2.0), (3.0, 3.0)])) == (sizeof(Int) == 8 ? + "Set([(2.0, 2.0), (1.0, 1.0), (3.0, 3.0)])" : + "Set([(2.0, 2.0), (1.0, 1.0), (3.0, 3.0)])") # issue #27747 let t = (x = Integer[1, 2],) @@ -2101,7 +2163,7 @@ end function compute_annotations(f, types) src = code_typed(f, types, debuginfo=:source)[1][1] ir = Core.Compiler.inflate_ir(src) - la, lb, ll = Base.IRShow.compute_ir_line_annotations(ir) + la, lb, ll = IRShow.compute_ir_line_annotations(ir) max_loc_method = maximum(length(s) for s in la) return join((strip(string(a, " "^(max_loc_method-length(a)), b)) for (a, b) in zip(la, lb)), '\n') end @@ -2156,6 +2218,8 @@ eval(Meta._parse_string("""function my_fun28173(x) return y end""", "a"^80, 1, 1, :statement)[1]) # use parse to control the line numbers let src = code_typed(my_fun28173, (Int,), debuginfo=:source)[1][1] + @test_throws "must be one of the following" sprint(IRShow.show_ir, src; context = :debuginfo => :_) + @test !contains(sprint(IRShow.show_ir, src; context = :debuginfo => :source_inline), "a"^80) ir = Core.Compiler.inflate_ir(src) src.debuginfo = Core.DebugInfo(src.debuginfo.def) # IRCode printing defaults to incomplete line info printing, so turn it off completely for CodeInfo too let source_slotnames = String["my_fun28173", "x"], @@ -2185,18 +2249,16 @@ let src = code_typed(my_fun28173, (Int,), debuginfo=:source)[1][1] @test pop!(lines2) == "18 │ \$(QuoteNode(3))" @test lines1 == lines2 - # verbose linetable - io = IOBuffer() - Base.IRShow.show_ir(io, ir, Base.IRShow.default_config(ir; verbose_linetable=true)) - seekstart(io) - @test count(contains(r"@ a{80}:\d+ within `my_fun28173"), eachline(io)) == 10 + # debuginfo = :source + output = sprint(Base.IRShow.show_ir, ir, Base.IRShow.default_config(ir; debuginfo=:source)) + @test count(contains(r"@ a{80}:\d+ within `my_fun28173"), split(output, '\n')) == 10 + @test output == sprint(show, ir; context = :debuginfo => :source) + @test output != sprint(show, ir) + @test_throws "must be one of the following" sprint(show, ir; context = :debuginfo => :_) # Test that a bad :invoke doesn't cause an error during printing Core.Compiler.insert_node!(ir, 1, Core.Compiler.NewInstruction(Expr(:invoke, nothing, sin), Any), false) - io = IOBuffer() - Base.IRShow.show_ir(io, ir) - seekstart(io) - @test contains(String(take!(io)), "Expr(:invoke, nothing") + @test contains(string(ir), "Expr(:invoke, nothing") end # Verify that extra instructions at the end of the IR @@ -2428,6 +2490,7 @@ end @test string(Union{M37012.SimpleU, Nothing, T} where T) == "Union{Nothing, $(curmod_prefix)M37012.SimpleU, T} where T" @test string(Union{AbstractVector{T}, T} where T) == "Union{AbstractVector{T}, T} where T" @test string(Union{AbstractVector, T} where T) == "Union{AbstractVector, T} where T" +@test string(Union{Array, Memory}) == "Union{Array, Memory}" @test sprint(show, :(./)) == ":((./))" @test sprint(show, :((.|).(.&, b))) == ":((.|).((.&), b))" @@ -2556,7 +2619,7 @@ end mktemp() do f, io redirect_stdout(io) do let io = IOBuffer() - for i = 1:10 + for i = 1:length(Base.Compiler.ALL_PASS_NAMES) # make sure we don't error on printing IRs at any optimization level ir = only(Base.code_ircode(sin, (Float64,); optimize_until=i))[1] @test try; show(io, ir); true; catch; false; end @@ -2825,3 +2888,15 @@ end @test_repr """:(var"\a\b\t\n\v\f\r\e" = 1)""" @test_repr """:(var"\x01\u03c0\U03c0" = 1)""" end + +# test `print_signature_only::Bool` argument of `Base.show_method` +f_show_method(x::T) where T<:Integer = :integer +let m = only(methods(f_show_method)) + let io = IOBuffer() + Base.show_method(io, m; print_signature_only=true) + @test "f_show_method(x::T) where T<:Integer" == String(take!(io)) + end + let s = sprint(show, m; context=:print_method_signature_only=>true) + @test "f_show_method(x::T) where T<:Integer" == s + end +end diff --git a/test/sorting.jl b/test/sorting.jl index ab4250fef411b..da5392a8ff884 100644 --- a/test/sorting.jl +++ b/test/sorting.jl @@ -808,6 +808,16 @@ end end end +@testset "partialsort(x; scratch)" begin + for n in [1,10,100,1000] + v = rand(n) + scratch = [0.0] + k = n ÷ 2 + 1 + @test partialsort(v, k) == partialsort(v, k; scratch) + @test partialsort!(copy(v), k) == partialsort!(copy(v), k; scratch) + end +end + @testset "sorting preserves identity" begin a = BigInt.([2, 2, 2, 1, 1, 1]) # issue #39620 sort!(a) diff --git a/test/spawn.jl b/test/spawn.jl index 099f0670ce5f7..b20200d5ffa2b 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -25,7 +25,7 @@ busybox_hash_correct(file) = bytes2hex(open(SHA.sha256, file)) == "ed2f95da95552 function _tryonce_download_from_cache(desired_url::AbstractString) cache_url = "https://cache.julialang.org/$(desired_url)" - cache_output_filename = joinpath(mktempdir(), "busybox") + cache_output_filename = joinpath(mktempdir(), "busybox" * (Sys.iswindows() ? ".exe" : "")) cache_response = Downloads.request( cache_url; output = cache_output_filename, @@ -620,7 +620,9 @@ end @test reduce(&, [`$echocmd abc`, `$echocmd def`, `$echocmd hij`]) == `$echocmd abc` & `$echocmd def` & `$echocmd hij` # readlines(::Cmd), accidentally broken in #20203 -@test sort(readlines(`$lscmd -A`)) == sort(readdir()) +let str = "foo\nbar" + @test readlines(`$echocmd $str`) == split(str) +end # issue #19864 (PR #20497) let c19864 = readchomp(pipeline(ignorestatus( diff --git a/test/stacktraces.jl b/test/stacktraces.jl index ca553c2a2e801..116e8dd3fe031 100644 --- a/test/stacktraces.jl +++ b/test/stacktraces.jl @@ -90,7 +90,7 @@ f(x) = (y = h(x); y) trace = (try; f(3); catch; stacktrace(catch_backtrace()); end)[1:3] can_inline = Bool(Base.JLOptions().can_inline) for (frame, func, inlined) in zip(trace, [g,h,f], (can_inline, can_inline, false)) - @test frame.func === typeof(func).name.mt.name + @test frame.func === typeof(func).name.singletonname # broken until #50082 can be addressed mi = isa(frame.linfo, Core.CodeInstance) ? frame.linfo.def : frame.linfo @test mi.def.module === which(func, (Any,)).module broken=inlined @@ -109,10 +109,10 @@ let src = Meta.lower(Main, quote let x = 1 end end).args[1]::Core.CodeInfo repr = string(sf) @test repr == "Toplevel MethodInstance thunk at b:3" end -let li = typeof(fieldtype).name.mt.cache.func::Core.MethodInstance, +let li = only(methods(fieldtype)).unspecialized, sf = StackFrame(:a, :b, 3, li, false, false, 0), repr = string(sf) - @test repr == "fieldtype(...) at b:3" + @test repr == "fieldtype(::Vararg{Any}) at b:3" end let ctestptr = cglobal((:ctest, "libccalltest")), @@ -248,7 +248,7 @@ struct F49231{a,b,c,d,e,f,g} end stacktrace(catch_backtrace()) end str = sprint(Base.show_backtrace, st, context = (:limit=>true, :stacktrace_types_limited => Ref(false), :color=>true, :displaysize=>(50,105))) - @test contains(str, "[5] \e[0m\e[1mcollect_to!\e[22m\e[0m\e[1m(\e[22m\e[90mdest\e[39m::\e[0mVector\e[90m{…}\e[39m, \e[90mitr\e[39m::\e[0mBase.Generator\e[90m{…}\e[39m, \e[90moffs\e[39m::\e[0m$Int, \e[90mst\e[39m::\e[0mTuple\e[90m{…}\e[39m\e[0m\e[1m)\e[22m\n\e[90m") + @test contains(str, "[5] \e[0m\e[1mcollect_to!\e[22m\e[0m\e[1m(\e[22m\e[90mdest\e[39m::\e[0mVector\e[90m{…}\e[39m, \e[90mitr\e[39m::\e[0mBase.Generator\e[90m{…}\e[39m, \e[90moffs\e[39m::\e[0m$Int, \e[90mst\e[39m::\e[0m$Int\e[0m\e[1m)\e[22m\n") st = try F49231{Vector,Val{'}'},Vector{Vector{Vector{Vector}}},Tuple{Int,Int,Int,Int,Int,Int,Int},Int,Int,Int}()(1,2,3) @@ -256,7 +256,7 @@ struct F49231{a,b,c,d,e,f,g} end stacktrace(catch_backtrace()) end str = sprint(Base.show_backtrace, st, context = (:limit=>true, :stacktrace_types_limited => Ref(false), :color=>true, :displaysize=>(50,132))) - @test contains(str, "[2] \e[0m\e[1m(::$F49231{Vector, Val{…}, Vector{…}, NTuple{…}, $Int, $Int, $Int})\e[22m\e[0m\e[1m(\e[22m\e[90ma\e[39m::\e[0m$Int, \e[90mb\e[39m::\e[0m$Int, \e[90mc\e[39m::\e[0m$Int\e[0m\e[1m)\e[22m\n\e[90m") + @test contains(str, "[2] \e[0m\e[1m(::$F49231{Vector, Val{…}, Vector{…}, NTuple{…}, $Int, $Int, $Int})\e[22m\e[0m\e[1m(\e[22m\e[90ma\e[39m::\e[0m$Int, \e[90mb\e[39m::\e[0m$Int, \e[90mc\e[39m::\e[0m$Int\e[0m\e[1m)\e[22m\n") end @testset "Base.StackTraces docstrings" begin diff --git a/test/staged.jl b/test/staged.jl index d416b0f9a22f0..4fb5d03331711 100644 --- a/test/staged.jl +++ b/test/staged.jl @@ -477,3 +477,5 @@ module GeneratedScope57417 end @test g() == 1 end + +@test_throws "syntax: expression too large" code_lowered(ntuple, (Returns{Nothing}, Val{1000000})) diff --git a/test/stdlib_dependencies.jl b/test/stdlib_dependencies.jl new file mode 100644 index 0000000000000..b1529b7daebca --- /dev/null +++ b/test/stdlib_dependencies.jl @@ -0,0 +1,324 @@ +using Libdl +using Pkg +using Test +prev_env = Base.active_project() +Pkg.activate(temp=true) +Pkg.add(Pkg.PackageSpec(name="ObjectFile", uuid="d8793406-e978-5875-9003-1fc021f44a92", version="0.4")) +using ObjectFile +try + + strip_soversion(lib::AbstractString) = Base.BinaryPlatforms.parse_dl_name_version(lib)[1] + + function get_deps_objectfile_macos(lib_path::String) + open(lib_path, "r") do io + obj_handles = readmeta(io) + obj = only(obj_handles) # If more than one its unclear what to do + raw_libs = String[] + + # For Mach-O files, get load commands + if isa(obj, ObjectFile.MachOHandle) + for lc in ObjectFile.MachOLoadCmds(obj) + if lc isa ObjectFile.MachO.MachOLoadDylibCmd + # Extract the library name from the load command + lib_name = ObjectFile.dylib_name(lc) + if lib_name !== nothing + # Remove @rpath/ prefix if present + lib_name = last(split(lib_name, "@rpath/")) + # Get basename + lib_name = basename(lib_name) + isempty(splitext(lib_name)[2]) && continue # skip frameworks + push!(raw_libs, lib_name) + end + end + end + end + libs = strip_soversion.(raw_libs) + # Get rid of any self-referential links + self_lib = strip_soversion(basename(lib_path)) + libs = filter(!=(self_lib), libs) + return libs + end + end + + function get_deps_objectfile_linux_freebsd(lib_path::String) + open(lib_path, "r") do io + obj_handles = readmeta(io) + obj = first(obj_handles) # Take the first handle from the vector + raw_libs = String[] + + # For ELF files, get dynamic dependencies + if isa(obj, ObjectFile.ELFHandle) + # Get all dynamic entries + dyn_entries = ObjectFile.ELFDynEntries(obj) + for entry in dyn_entries + # Check if the entry is of type DT_NEEDED + if ObjectFile.dyn_entry_type(entry) == ObjectFile.ELF.DT_NEEDED + lib_name = ObjectFile.strtab_lookup(entry) + if lib_name !== nothing && !isempty(lib_name) + push!(raw_libs, basename(lib_name)) + end + end + end + end + + libs = strip_soversion.(raw_libs) + # Self-reference is typically not listed in NEEDED for ELF, so no explicit filter here. + return libs + end + end + + function get_deps_objectfile_windows(lib_path::String) + open(lib_path, "r") do io + obj_handles = readmeta(io) + obj = first(obj_handles) # Take the first handle from the vector + raw_libs_set = Set{String}() # Use Set for uniqueness of DLL names + + # For COFF/PE files, get import table + if isa(obj, ObjectFile.COFFHandle) + # Get dynamic links + dls = ObjectFile.DynamicLinks(obj) + for link in dls + lib_name = ObjectFile.path(link) + if lib_name !== nothing && !isempty(lib_name) + # COFF library names are case-insensitive + push!(raw_libs_set, lowercase(lib_name)) + end + end + end + + libs = strip_soversion.(collect(raw_libs_set)) + # Get rid of any self-referential links + self_lib = strip_soversion(lowercase(basename(lib_path))) + libs = filter(!=(self_lib), libs) + return libs + end + end + + function get_deps_objectfile(lib_path::String) + if Sys.isapple() + return get_deps_objectfile_macos(lib_path) + elseif Sys.islinux() || Sys.isfreebsd() + return get_deps_objectfile_linux_freebsd(lib_path) + elseif Sys.iswindows() + return get_deps_objectfile_windows(lib_path) + else + error("Unsupported platform for ObjectFile.jl dependency extraction") + end + end + + function is_system_lib_macos(lib) + system_libs = [ + "libSystem.B", + "libc++", # While we package libstdc++, we do NOT package libc++. + "libiconv", # some things (like git) link against system libiconv + + # macOS frameworks used by things like LibCurl + "CoreFoundation", + "CoreServices", + "Security", + "SystemConfiguration" + ] + return lib ∈ system_libs + end + + function is_system_lib_linux(lib) + system_libs = [ + "libdl", + "libc", + "libm", + "librt", + "libpthread", + "ld-linux", + "ld-linux-x86-64", + "ld-linux-x86", + "ld-linux-aarch64", + "ld-linux-armhf", + "ld-linux-i386", + ] + return lib ∈ system_libs + end + + function is_system_lib_freebsd(lib) + system_libs = [ + "libdl", + "libc", + "libm", + "libthr", # primary threading library + "libpthread", # alias kept for compatibility + "librt", + "libutil", + "libexecinfo", + "libc++", + "libcxxrt", + ] + return lib ∈ system_libs + end + + function is_system_lib_windows(lib) + system_libs = [ + "kernel32", + "user32", + "gdi32", + "advapi32", + "ole32", + "oleaut32", + "shell32", + "ws2_32", + "comdlg32", + "shlwapi", + "rpcrt4", + "msvcrt", + "comctl32", + "ucrtbase", + "vcruntime140", + "msvcp140", + "libwinpthread", + "ntdll", + "crypt32", + "bcrypt", + "winhttp", + "secur32", + "iphlpapi", + ] + return any(syslib -> lowercase(lib) == syslib, system_libs) + end + + # Set up platform-specific functions + if Sys.islinux() || Sys.isfreebsd() + is_system_lib = Sys.islinux() ? is_system_lib_linux : is_system_lib_freebsd + elseif Sys.isapple() + is_system_lib = is_system_lib_macos + elseif Sys.iswindows() + is_system_lib = is_system_lib_windows + else + error("Unsupported platform for `stdlib_dependencies.jl`. Only Linux, FreeBSD, macOS, and Windows are supported.") + end + + # Iterate over all JLL stdlibs, check their lazy libraries to ensure + # that they list all valid library dependencies, avoiding a situation + # where the JLL wrapper code has fallen out of sync with the binaries + # themselves. + @testset "Stdlib JLL dependency check" begin + for (_, (stdlib_name, _)) in Pkg.Types.stdlibs() + if !endswith(stdlib_name, "_jll") + continue + end + + # Import the stdlib, skip it if it's not available on this platform + m = eval(Meta.parse("import $(stdlib_name); $(stdlib_name)")) + if !Base.invokelatest(getproperty(m, :is_available)) + continue + end + + for prop_name in names(m) + prop = getproperty(m, prop_name) + if isa(prop, Libdl.LazyLibrary) + lib_path = dlpath(prop) + lazy_lib_deps = strip_soversion.(basename.(dlpath.(prop.dependencies))) + real_lib_deps = filter(!is_system_lib, get_deps_objectfile(lib_path)) + + # See if there are missing dependencies in the lazy library deps + missing_deps = setdiff(real_lib_deps, lazy_lib_deps) + extraneous_deps = setdiff(lazy_lib_deps, real_lib_deps) + + # The library name is `libpcre2-8`, with a dash in + # its name. That works fine on Unix. On Windows, a + # suffix starting with a dash denotes the + # library's soversion. So we think (on Windows) + # that this is the library `libpcre2`, soversion + # 8, and things don't match. + if Sys.iswindows() + if "libpcre2-8" in missing_deps && "libpcre2" in extraneous_deps + missing_deps = setdiff(missing_deps, ["libpcre2-8"]) + extraneous_deps = setdiff(extraneous_deps, ["libpcre2"]) + end + end + + if prop_name == :libspqr + # Allow libstdc++ to not be linked - spqr only uses std::complex, + # which may be header-only, so doesn't get linked on as-needed distributions. + # However, in general, we can't assume that, so we need to take the dependency + # and just allow this here. + extraneous_deps = setdiff(extraneous_deps, ["libstdc++"]) + end + + # We expect there to be no missing or extraneous deps + deps_mismatch = !isempty(missing_deps) || !isempty(extraneous_deps) + + # This is a manually-managed special case + if stdlib_name == "libblastrampoline_jll" && + prop_name == :libblastrampoline && + extraneous_deps in (["libopenblas64_"], ["libopenblas"]) + deps_mismatch = false + end + + @test !deps_mismatch + + # Print out the deps mismatch if we find one + if deps_mismatch + @warn("Dependency mismatch", + jll = stdlib_name, + library = string(prop_name), + missing_deps = join(missing_deps, ", "), + extraneous_deps = join(extraneous_deps, ", "), + actual_deps = join(real_lib_deps, ", "), + ) + end + end + end + if isdefined(m, :eager_mode) + # If the JLL has an eager_mode function, call it + Base.invokelatest(getproperty(m, :eager_mode)) + end + end + end + + # Check that any JLL stdlib that defines executables also provides corresponding *_path variables + @testset "Stdlib JLL executable path variables" begin + for (_, (stdlib_name, _)) in Pkg.Types.stdlibs() + if !endswith(stdlib_name, "_jll") + continue + end + + # Import the stdlib, skip it if it's not available on this platform + m = eval(Meta.parse("import $(stdlib_name); $(stdlib_name)")) + if !Base.invokelatest(getproperty(m, :is_available)) + continue + end + + # Look for *_exe constants that indicate executable definitions + exe_constants = Symbol[] + for name in names(m, all=true) + name_str = string(name) + if endswith(name_str, "_exe") && isdefined(m, name) + push!(exe_constants, name) + end + end + + # For each *_exe constant, check if there's a corresponding *_path variable + for exe_const in exe_constants + exe_name_str = string(exe_const) + # Convert from *_exe to *_path (e.g., zstd_exe -> zstd_path) + expected_path_var = Symbol(replace(exe_name_str, "_exe" => "_path")) + + @test isdefined(m, expected_path_var) + + if !isdefined(m, expected_path_var) + @warn("Missing path variable", + jll = stdlib_name, + exe_constant = exe_const, + expected_path_var = expected_path_var + ) + end + end + end + end + +finally + if prev_env !== nothing + Pkg.activate(prev_env) + else + # If no previous environment, activate the default one + Pkg.activate() + end +end diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 8658c1b52a2ab..7f53740b9eec1 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -258,3 +258,51 @@ end write(aio, Base.AnnotatedString("hello", [(1:5, :tag, 1)])) @test sprint(show, aio) == "Base.AnnotatedIOBuffer(5 bytes, 1 annotation)" end + +@testset "Eachregion" begin + annregions(str::String, annots::Vector{<:Tuple{UnitRange{Int}, Symbol, <:Any}}) = + [(s, Tuple.(a)) for (s, a) in Base.eachregion(Base.AnnotatedString(str, annots))] + # Regions that do/don't extend to the left/right edges + @test annregions(" abc ", [(2:4, :face, :bold)]) == + [(" ", []), + ("abc", [(:face, :bold)]), + (" ", [])] + @test annregions(" x ", [(2:2, :face, :bold)]) == + [(" ", []), + ("x", [(:face, :bold)]), + (" ", [])] + @test annregions(" x", [(2:2, :face, :bold)]) == + [(" ", []), + ("x", [(:face, :bold)])] + @test annregions("x ", [(1:1, :face, :bold)]) == + [("x", [(:face, :bold)]), + (" ", [])] + @test annregions("x", [(1:1, :face, :bold)]) == + [("x", [(:face, :bold)])] + # Overlapping/nested regions + @test annregions(" abc ", [(2:4, :face, :bold), (3:3, :face, :italic)]) == + [(" ", []), + ("a", [(:face, :bold)]), + ("b", [(:face, :bold), (:face, :italic)]), + ("c", [(:face, :bold)]), + (" ", [])] + @test annregions("abc-xyz", [(1:7, :face, :bold), (1:3, :face, :green), (4:4, :face, :yellow), (4:7, :face, :italic)]) == + [("abc", [(:face, :bold), (:face, :green)]), + ("-", [(:face, :bold), (:face, :yellow), (:face, :italic)]), + ("xyz", [(:face, :bold), (:face, :italic)])] + # Preserving annotation order + @test annregions("abcd", [(1:3, :face, :red), (2:2, :face, :yellow), (2:3, :face, :green), (2:4, :face, :blue)]) == + [("a", [(:face, :red)]), + ("b", [(:face, :red), (:face, :yellow), (:face, :green), (:face, :blue)]), + ("c", [(:face, :red), (:face, :green), (:face, :blue)]), + ("d", [(:face, :blue)])] + @test annregions("abcd", [(2:4, :face, :blue), (1:3, :face, :red), (2:3, :face, :green), (2:2, :face, :yellow)]) == + [("a", [(:face, :red)]), + ("b", [(:face, :blue), (:face, :red), (:face, :green), (:face, :yellow)]), + ("c", [(:face, :blue), (:face, :red), (:face, :green)]), + ("d", [(:face, :blue)])] + # Region starting after a character spanning multiple codepoints. + @test annregions("𝟏x", [(1:4, :face, :red)]) == + [("𝟏", [(:face, :red)]), + ("x", [])] +end diff --git a/test/strings/basic.jl b/test/strings/basic.jl index c3e0bcc501070..214a14ed2443f 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -49,6 +49,24 @@ using Random end end +@testset "takestring!" begin + v = [0x61, 0x62, 0x63] + old_mem = v.ref.mem + @test takestring!(v) == "abc" + @test isempty(v) + @test v.ref.mem !== old_mem # memory is changed + for v in [ + UInt8[], + [0x01, 0x02, 0x03], + collect(codeunits("æøå")) + ] + cp = copy(v) + s = takestring!(v) + @test isempty(v) + @test codeunits(s) == cp + end +end + @testset "{starts,ends}with" begin @test startswith("abcd", 'a') @test startswith('a')("abcd") @@ -1078,6 +1096,7 @@ let s = "∀x∃y", u = codeunits(s) @test_throws Base.CanonicalIndexError (u[1] = 0x00) @test collect(u) == b"∀x∃y" @test Base.elsize(u) == Base.elsize(typeof(u)) == 1 + @test similar(typeof(u), 3) isa Vector{UInt8} end @testset "issue #24388" begin @@ -1445,3 +1464,12 @@ if Sys.iswindows() @test Base.cwstring(str_3) == UInt16[0x0061, 0x0723, 0xd808, 0xdc00, 0x0000] end end + + +@testset "eltype for AbstractString subtypes" begin + @test eltype(String) == Char + @test eltype(SubString{String}) == Char + + u = b"hello" + @test eltype(u) === UInt8 +end diff --git a/test/strings/util.jl b/test/strings/util.jl index bb87881bbaa1d..9ced27ee3f8d0 100644 --- a/test/strings/util.jl +++ b/test/strings/util.jl @@ -8,6 +8,8 @@ SubStr(s) = SubString("abc$(s)de", firstindex(s) + 3, lastindex(s) + 3) @test textwidth(c^3) == w*3 @test w == @invoke textwidth(c::AbstractChar) end + @test textwidth('\xc0\xa0') == 1 # overlong + @test textwidth('\xf0\x80\x80') == 1 # malformed for i in 0x00:0x7f # test all ASCII chars (which have fast path) w = Int(ccall(:utf8proc_charwidth, Cint, (UInt32,), i)) c = Char(i) diff --git a/test/subtype.jl b/test/subtype.jl index 979746bd626dc..dbd761c7f5867 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -2087,8 +2087,7 @@ let A = Tuple{Any, Type{Ref{_A}} where _A}, I = typeintersect(A, B) @test I != Union{} @test Tuple{Type{Ref{Integer}}, Type{Ref{Integer}}} <: I - # TODO: this intersection result seems too wide (I == B) ? - @test_broken !<:(Tuple{Type{Int}, Type{Int}}, I) + @test !<:(Tuple{Type{Int}, Type{Int}}, I) end @testintersect(Tuple{Type{T}, T} where T<:(Tuple{Vararg{_A, _B}} where _B where _A), @@ -2757,3 +2756,33 @@ end Pair{N, T} where {N,NTuple{N,Int}<:T<:Tuple{Int,Vararg{Int}}}, !Union{} ) + +#issue 57852 +@testintersect( + Tuple{Type{T}, Type{<:F}, Type{<:F}} where {T, F<:Union{String, T}}, + Tuple{Type{Complex{T}} where T, Type{Complex{T}} where T, Type{String}}, + Tuple{Type{Complex{T}}, Type{Complex{T}}, Type{String}} where T +) +@testintersect( + Tuple{Type{T}, Type{<:Union{F, Nothing}}, Type{<:Union{F, Nothing}}} where {T, F<:Union{String, T}}, + Tuple{Type{Complex{T}} where T, Type{Complex{T}} where T, Type{String}}, + Tuple{Type{Complex{T}}, Type{Complex{T}}, Type{String}} where T +) + +#issue 58129 +for k in 1:500 + @eval struct $(Symbol(:T58129, k)){T} end +end +let Tvar = TypeVar(:Tvar) + V = UnionAll(Tvar, Union{(@eval($(Symbol(:T58129, k)){$Tvar}) for k in 1:500)...}) + @test Set{<:V} <: AbstractSet{<:V} +end +let Tvar1 = TypeVar(:Tvar1), Tvar2 = TypeVar(:Tvar2) + V1 = UnionAll(Tvar1, Union{(@eval($(Symbol(:T58129, k)){$Tvar1}) for k in 1:100)...}) + V2 = UnionAll(Tvar2, Union{(@eval($(Symbol(:T58129, k)){$Tvar2}) for k in 1:100)...}) + @test Set{<:V2} <: AbstractSet{<:V1} +end + +#issue 58115 +@test Tuple{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{ Union{Tuple{}, Tuple{Tuple{}}}}}}}}}}}}} , Tuple{}} <: + Tuple{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Tuple{Vararg{Union{Tuple{}, Tuple{Tuple{}}}}}}}}}}}}}}}, Tuple{}} diff --git a/test/syntax.jl b/test/syntax.jl index 107bf42ba7267..35fe05c9c425d 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -596,10 +596,9 @@ let thismodule = @__MODULE__, @test !isdefined(M16096, :foo16096) @test !isdefined(M16096, :it) @test typeof(local_foo16096).name.module === thismodule - @test typeof(local_foo16096).name.mt.module === thismodule - @test getfield(thismodule, typeof(local_foo16096).name.mt.name) === local_foo16096 + @test getfield(thismodule, typeof(local_foo16096).name.singletonname) === local_foo16096 @test getfield(thismodule, typeof(local_foo16096).name.name) === typeof(local_foo16096) - @test !isdefined(M16096, typeof(local_foo16096).name.mt.name) + @test !isdefined(M16096, typeof(local_foo16096).name.singletonname) @test !isdefined(M16096, typeof(local_foo16096).name.name) end @@ -1527,8 +1526,11 @@ end @test Meta.lower(@__MODULE__, :(return 0 for i=1:2)) == Expr(:error, "\"return\" not allowed inside comprehension or generator") @test Meta.lower(@__MODULE__, :([ return 0 for i=1:2 ])) == Expr(:error, "\"return\" not allowed inside comprehension or generator") @test Meta.lower(@__MODULE__, :(Int[ return 0 for i=1:2 ])) == Expr(:error, "\"return\" not allowed inside comprehension or generator") +@test Meta.lower(@__MODULE__, :([ $(Expr(:thisfunction)) for i=1:2 ])) == Expr(:error, "\"@__FUNCTION__\" not allowed inside comprehension or generator") +@test Meta.lower(@__MODULE__, :($(Expr(:thisfunction)) for i=1:2)) == Expr(:error, "\"@__FUNCTION__\" not allowed inside comprehension or generator") @test [ ()->return 42 for i = 1:1 ][1]() == 42 @test Function[ identity() do x; return 2x; end for i = 1:1 ][1](21) == 42 +@test @eval let f=[ ()->$(Expr(:thisfunction)) for i = 1:1 ][1]; f() === f; end # issue #27155 macro test27155() @@ -1939,7 +1941,7 @@ end # eval'ing :const exprs eval(Expr(:const, :_var_30877)) @test !isdefined(@__MODULE__, :_var_30877) -@test isconst(@__MODULE__, :_var_30877) +@test !isconst(@__MODULE__, :_var_30877) # anonymous kw function in value position at top level f30926 = function (;k=0) @@ -2287,6 +2289,11 @@ end @test Meta.parse("a ⥷ b") == Expr(:call, :⥷, :a, :b) end +# issue 57143 +@testset "binary 🢲" begin + @test Meta.parse("a 🢲 b") == Expr(:call, :🢲, :a, :b) +end + # only allow certain characters after interpolated vars (#25231) @test_parseerror("\"\$x෴ \"", "interpolated variable \$x ends with invalid character \"෴\"; use \"\$(x)\" instead.") @@ -2417,15 +2424,6 @@ macro a35391(b) end @test @a35391(0) === (0,) -# global declarations from the top level are not inherited by functions. -# don't allow such a declaration to override an outer local, since it's not -# clear what it should do. -@test Meta.lower(Main, :(let - x = 1 - let - global x - end - end)) == Expr(:error, "`global x`: x is a local variable in its enclosing scope") # note: this `begin` block must be at the top level _temp_33553 = begin global _x_this_remains_undefined @@ -2437,6 +2435,70 @@ end @test _temp_33553 == 2 @test !@isdefined(_x_this_remains_undefined) +module GlobalContainment +using Test +@testset "scope of global declarations" begin + + # global declarations from the top level are not inherited by functions. + # don't allow such a declaration to override an outer local, since it's not + # clear what it should do. + @test Meta.lower( + Main, + :(let + x = 1 + let + global x + end + end)) == Expr(:error, "`global x`: x is a local variable in its enclosing scope") + + # a declared global can shadow a local in an outer scope + @test let + function f() + g0 = 2 + let; global g0 = 1; end + a = () -> (global g0 = 1); a(); + return g0 + end + (f(), g0); + end === (2, 1) + @test let + function f() + let; global g2 = 1; end; + let; try; g2 = 2; catch _; end; end; + end + (f(), g2) + end === (2, 1) + + # an inner global declaration should not interfere with the closure (#57547) + @test let + g3 = 1 + function f() + let; global g3 = 2; end; + return g3 + end + f() + end === 1 + @test_throws UndefVarError let + function returns_global() + for i in 1 + global ge = 2 + end + return ge # local declared below + end + ge = returns_global() + end + @test let + function f(x::T) where T + function g(x) + let; global T = 1; end + x::T + end; g(x) + end; f(1) + end === 1 + +end +end + # lowering of adjoint @test (1 + im)' == 1 - im x = let var"'"(x) = 2x @@ -2838,6 +2900,9 @@ end @test Meta.isexpr(Meta.lower(Main, :(for _ in 1:2; 1; end)), :thunk) @test (try; throw(1); catch _; 2; end) === 2 @test (let _ = 1; 2; end) === 2 + @test (function f(_, _); 2; end)(0,0) === 2 + @test (function f(_, _=1); 2; end)(0,0) === 2 + @test (function f(_, _; kw1=2); kw1; end)(0,0) === 2 # ERROR: syntax: all-underscore identifiers are write-only and their values cannot be used in expressions @test Meta.isexpr(Meta.lower(Main, :(_ = 1; a = _)), :error) @test Meta.isexpr(Meta.lower(Main, :(let; function f(); _; end; end)), :error) @@ -3518,6 +3583,8 @@ end # issue #45162 f45162(f) = f(x=1) @test first(methods(f45162)).called != 0 +f45162_2(f) = f([]...) +@test first(methods(f45162_2)).called != 0 # issue #45024 @test_parseerror "const x" "expected assignment after \"const\"" @@ -4238,6 +4305,34 @@ end @test letf_57470(3) == 5 @test letT_57470 === Int64 +# Closure conversion should happen on const assignment rhs +module M59128 +using Test +const x0::Int = (()->1)() +global x1::Int = (()->1)() +global const x2::Int = (()->1)() +const global x3::Int = (()->1)() +@test x0 === x1 === x2 === x3 === 1 +let g = 1 + global x4::Vector{T} where {T<:Number} = let; (()->[g])(); end + const global x5::Vector{T} where {T<:Number} = let; (()->[g])(); end + global const x6::Vector{T} where {T<:Number} = let; (()->[g])(); end +end +@test x4 == x5 == x6 == [1] +const letT_57470{T} = (()->Int64)() +@test letT_57470 == Int64 +end + +end # M57470_sub + +# lowering globaldecl with complex type +module M58609 +using Test +global x::T where T +global y::Type{<:Number} + +@test Core.get_binding_type(M58609, :x) === Any +@test Core.get_binding_type(M58609, :y) == Type{<:Number} end # #57574 @@ -4260,3 +4355,196 @@ module DoubleImport import Random end @test DoubleImport.Random === Test.Random + +# Expr(:method) returns the method +let ex = @Meta.lower function return_my_method(); 1; end + code = ex.args[1].code + idx = findfirst(ex->Meta.isexpr(ex, :method) && length(ex.args) > 1, code) + code[end] = Core.ReturnNode(Core.SSAValue(idx)) + @test isa(Core.eval(@__MODULE__, ex), Method) +end + +# Capturing a @nospecialize argument should result in an Any field in the closure +module NoSpecClosure + K(@nospecialize(x)) = y -> x +end +let f = NoSpecClosure.K(1) + @test f(2) == 1 + @test typeof(f).parameters == Core.svec() +end + +@testset "@__FUNCTION__ and Expr(:thisfunction)" begin + @testset "Basic usage" begin + # @__FUNCTION__ in regular functions + test_function_basic() = @__FUNCTION__ + @test test_function_basic() === test_function_basic + + # Expr(:thisfunction) in regular functions + @eval regular_func() = $(Expr(:thisfunction)) + @test regular_func() === regular_func + end + + @testset "Recursion" begin + # Factorial with @__FUNCTION__ + factorial_function(n) = n <= 1 ? 1 : n * (@__FUNCTION__)(n - 1) + @test factorial_function(5) == 120 + + # Fibonacci with Expr(:thisfunction) + struct RecursiveCallableStruct; end + @eval (::RecursiveCallableStruct)(n) = n <= 1 ? n : $(Expr(:thisfunction))(n-1) + $(Expr(:thisfunction))(n-2) + @test RecursiveCallableStruct()(10) === 55 + + # Anonymous function recursion + @test (n -> n <= 1 ? 1 : n * (@__FUNCTION__)(n - 1))(5) == 120 + end + + @testset "Closures and nested functions" begin + # Prevents boxed closures + function make_closure() + fib(n) = n <= 1 ? 1 : (@__FUNCTION__)(n - 1) + (@__FUNCTION__)(n - 2) + return fib + end + Test.@inferred make_closure() + closure = make_closure() + @test closure(5) == 8 + Test.@inferred closure(5) + + # Complex closure of closures + function f1() + function f2() + function f3() + return @__FUNCTION__ + end + return (@__FUNCTION__), f3() + end + return (@__FUNCTION__), f2()... + end + Test.@inferred f1() + @test f1()[1] === f1 + @test f1()[2] !== f1 + @test f1()[3] !== f1 + @test f1()[3]() === f1()[3] + @test f1()[2]()[2]() === f1()[3] + end + + @testset "Do blocks" begin + function test_do_block() + result = map([1, 2, 3]) do x + return (@__FUNCTION__, x) + end + # All should refer to the same do-block function + @test all(r -> r[1] === result[1][1], result) + # Values should be different + @test [r[2] for r in result] == [1, 2, 3] + # It should be different than `test_do_block` + @test result[1][1] !== test_do_block + end + test_do_block() + end + + @testset "Keyword arguments" begin + # @__FUNCTION__ with kwargs + foo(; n) = n <= 1 ? 1 : n * (@__FUNCTION__)(; n = n - 1) + @test foo(n = 5) == 120 + + # Expr(:thisfunction) with kwargs + let + @eval f2(; n=1) = n <= 1 ? n : n * $(Expr(:thisfunction))(; n=n-1) + result = f2(n=5) + @test result == 120 + end + end + + @testset "Callable structs" begin + # @__FUNCTION__ in callable structs + @gensym A + @eval module $A + struct CallableStruct{T}; val::T; end + (c::CallableStruct)() = @__FUNCTION__ + end + @eval using .$A: CallableStruct + c = CallableStruct(5) + @test c() === c + + # In closures, var"#self#" should refer to the enclosing function, + # NOT the enclosing struct instance + struct CallableStruct2; end + @eval function (obj::CallableStruct2)() + function inner_func() + $(Expr(:thisfunction)) + end + inner_func + end + + let cs = CallableStruct2() + @test cs()() === cs() + @test cs()() !== cs + end + + # Accessing values via self-reference + struct CallableStruct3 + value::Int + end + @eval (obj::CallableStruct3)() = $(Expr(:thisfunction)) + @eval (obj::CallableStruct3)(x) = $(Expr(:thisfunction)).value + x + + let cs = CallableStruct3(42) + @test cs() === cs + @test cs(10) === 52 + end + + # Callable struct with args and kwargs + struct CallableStruct4 + end + @eval function (obj::CallableStruct4)(x, args...; y=2, kws...) + return (; func=(@__FUNCTION__), x, args, y, kws) + end + c = CallableStruct4() + @test c(1).func === c + @test c(2, 3).args == (3,) + @test c(2; y=4).y == 4 + @test c(2; y=4, a=5, b=6, c=7).kws[:c] == 7 + end + + @testset "Special cases" begin + # Generated functions + let @generated foo2() = Expr(:thisfunction) + @test foo2() === foo2 + end + + # Struct constructors + let + @eval struct Cols{T<:Tuple} + cols::T + operator + Cols(args...; operator=union) = (new{typeof(args)}(args, operator); string($(Expr(:thisfunction)))) + end + result = Cols(1, 2, 3) + @test occursin("Cols", result) + end + + # Should not access arg-map for local variables + @gensym f + @eval begin + function $f end + function ($f::typeof($f))() + $f = 1 + $(Expr(:thisfunction)) + end + end + @test @eval($f() === $f) + end + + @testset "Error upon misuse" begin + @gensym B + @test_throws( + "\"@__FUNCTION__\" can only be used inside a function", + @eval(module $B; @__FUNCTION__; end) + ) + + @test_throws( + "\"@__FUNCTION__\" not allowed inside comprehension or generator", + @eval([(@__FUNCTION__) for _ in 1:10]) + ) + end +end diff --git a/test/testhelpers/InfiniteArrays.jl b/test/testhelpers/InfiniteArrays.jl index cec3c94aaa296..a5f77356900e9 100644 --- a/test/testhelpers/InfiniteArrays.jl +++ b/test/testhelpers/InfiniteArrays.jl @@ -37,7 +37,7 @@ Define an `AbstractInfUnitRange` that behaves like `1:∞`, with the added distinction that the limits are guaranteed (by the type system) to be 1 and ∞. """ -struct OneToInf{T<:Integer} <: AbstractUnitRange{T} end +struct OneToInf{T<:Integer} <: Base.AbstractOneTo{T} end OneToInf() = OneToInf{Int}() diff --git a/test/testhelpers/SizedArrays.jl b/test/testhelpers/SizedArrays.jl index e52e965a64859..bd0272d78987d 100644 --- a/test/testhelpers/SizedArrays.jl +++ b/test/testhelpers/SizedArrays.jl @@ -14,7 +14,7 @@ import LinearAlgebra: mul! export SizedArray -struct SOneTo{N} <: AbstractUnitRange{Int} end +struct SOneTo{N} <: Base.AbstractOneTo{Int} end SOneTo(N) = SOneTo{N}() Base.length(::SOneTo{N}) where {N} = N Base.size(r::SOneTo) = (length(r),) @@ -54,18 +54,15 @@ Base.axes(a::SizedArray) = map(SOneTo, size(a)) Base.getindex(A::SizedArray, i...) = getindex(A.data, i...) Base.setindex!(A::SizedArray, v, i...) = setindex!(A.data, v, i...) Base.zero(::Type{T}) where T <: SizedArray = SizedArray{size(T)}(zeros(eltype(T), size(T))) +function Base.one(::Type{SizedMatrix{SZ,T,A}}) where {SZ,T,A} + allequal(SZ) || throw(DimensionMismatch("multiplicative identity defined only for square matrices")) + D = diagm(fill(one(T), SZ[1])) + SizedArray{SZ}(convert(A, D)) +end Base.parent(S::SizedArray) = S.data +(S1::SizedArray{SZ}, S2::SizedArray{SZ}) where {SZ} = SizedArray{SZ}(S1.data + S2.data) ==(S1::SizedArray{SZ}, S2::SizedArray{SZ}) where {SZ} = S1.data == S2.data -homogenize_shape(t::Tuple) = (_homogenize_shape(first(t)), homogenize_shape(Base.tail(t))...) -homogenize_shape(::Tuple{}) = () -_homogenize_shape(x::Integer) = x -_homogenize_shape(x::AbstractUnitRange) = length(x) -const Dims = Union{Integer, Base.OneTo, SOneTo} -function Base.similar(::Type{A}, shape::Tuple{Dims, Vararg{Dims}}) where {A<:AbstractArray} - similar(A, homogenize_shape(shape)) -end function Base.similar(::Type{A}, shape::Tuple{SOneTo, Vararg{SOneTo}}) where {A<:AbstractArray} R = similar(A, length.(shape)) SizedArray{length.(shape)}(R) @@ -74,6 +71,10 @@ function Base.similar(x::SizedArray, ::Type{T}, shape::Tuple{SOneTo, Vararg{SOne sz = map(length, shape) SizedArray{sz}(similar(parent(x), T, sz)) end +function Base.reshape(x::AbstractArray, shape::Tuple{SOneTo, Vararg{SOneTo}}) + sz = map(length, shape) + SizedArray{length.(sz)}(reshape(x, length.(sz))) +end const SizedMatrixLike = Union{SizedMatrix, Transpose{<:Any, <:SizedMatrix}, Adjoint{<:Any, <:SizedMatrix}} diff --git a/test/testhelpers/coverage_file.info b/test/testhelpers/coverage_file.info index b03b0e07e6977..61410a72bc849 100644 --- a/test/testhelpers/coverage_file.info +++ b/test/testhelpers/coverage_file.info @@ -1,6 +1,6 @@ SF: DA:3,1 -DA:4,1 +DA:4,2 DA:5,0 DA:7,1 DA:8,1 diff --git a/test/threads.jl b/test/threads.jl index 52d0546f0e31b..fa0b33a6352f3 100644 --- a/test/threads.jl +++ b/test/threads.jl @@ -447,6 +447,12 @@ let once = OncePerProcess(() -> return [nothing]) @atomic once.state = 0x01 @test x === once() end +let once1 = OncePerProcess(BigFloat), once2 = OncePerProcess{BigFloat}(BigFloat) + # Using a type as a constructor should create a OncePerProcess with + # Type{...} as its initializer (rather than DataType) + @test typeof(once1) <: OncePerProcess{BigFloat,Type{BigFloat}} + @test typeof(once2) <: OncePerProcess{BigFloat,Type{BigFloat}} +end let once = OncePerProcess{Int}(() -> error("expected")) @test_throws ErrorException("expected") once() @test_throws ErrorException("OncePerProcess initializer failed previously") once() @@ -551,6 +557,12 @@ let e = Base.Event(true), @test_throws ArgumentError once[-1] end +let once1 = OncePerThread(BigFloat), once2 = OncePerThread{BigFloat}(BigFloat) + # Using a type as a constructor should create a OncePerThread with + # Type{...} as its initializer (rather than DataType) + @test typeof(once1) <: OncePerThread{BigFloat,Type{BigFloat}} + @test typeof(once2) <: OncePerThread{BigFloat,Type{BigFloat}} +end let once = OncePerThread{Int}(() -> error("expected")) @test_throws ErrorException("expected") once() @test_throws ErrorException("OncePerThread initializer failed previously") once() @@ -563,6 +575,12 @@ let once = OncePerTask(() -> return [nothing]) delete!(task_local_storage(), once) @test x !== once() === once() end +let once1 = OncePerTask(BigFloat), once2 = OncePerTask{BigFloat}(BigFloat) + # Using a type as a constructor should create a OncePerTask with + # Type{...} as its initializer (rather than DataType) + @test typeof(once1) <: OncePerTask{BigFloat,Type{BigFloat}} + @test typeof(once2) <: OncePerTask{BigFloat,Type{BigFloat}} +end let once = OncePerTask{Int}(() -> error("expected")) @test_throws ErrorException("expected") once() @test_throws ErrorException("expected") once() diff --git a/test/threads_exec.jl b/test/threads_exec.jl index 629f474f53a38..66191f77459fa 100644 --- a/test/threads_exec.jl +++ b/test/threads_exec.jl @@ -110,6 +110,26 @@ if threadpoolsize(:default) > 1 end end +if threadpoolsize() > 1 + let lk = Base.Threads.PaddedSpinLock() + c1 = Base.Event() + c2 = Base.Event() + @test trylock(lk) + @test !trylock(lk) + t1 = Threads.@spawn (notify(c1); lock(lk); unlock(lk); trylock(lk)) + t2 = Threads.@spawn (notify(c2); trylock(lk)) + Libc.systemsleep(0.1) # block our thread from scheduling for a bit + wait(c1) + wait(c2) + @test !fetch(t2) + @test istaskdone(t2) + @test !istaskdone(t1) + unlock(lk) + @test fetch(t1) + @test istaskdone(t1) + end +end + # threading constructs @testset "@threads and @spawn threadpools" begin @@ -334,29 +354,12 @@ using Base.Threads end end -# Ensure only LLVM-supported types can be atomic -@test_throws TypeError Atomic{BigInt} -@test_throws TypeError Atomic{ComplexF64} - -if Sys.ARCH === :i686 || startswith(string(Sys.ARCH), "arm") || - Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le - - @test_throws TypeError Atomic{Int128}() - @test_throws TypeError Atomic{UInt128}() -end - -if Sys.ARCH === :powerpc64le || Sys.ARCH === :ppc64le - @test_throws TypeError Atomic{Float16}() - @test_throws TypeError Atomic{Float32}() - @test_throws TypeError Atomic{Float64}() -end - function test_atomic_bools() x = Atomic{Bool}(false) - # Arithmetic functions are not defined. - @test_throws MethodError atomic_add!(x, true) - @test_throws MethodError atomic_sub!(x, true) - # All the rest are: + # Arithmetic functions such as true+true returns Int + @test_throws TypeError atomic_add!(x, true) + @test_throws TypeError atomic_sub!(x, true) + # All the rest are supported: for v in [true, false] @test x[] == atomic_xchg!(x, v) @test v == atomic_cas!(x, v, !v) @@ -462,10 +465,9 @@ end test_fence() # Test load / store with various types -let atomictypes = intersect((Int8, Int16, Int32, Int64, Int128, - UInt8, UInt16, UInt32, UInt64, UInt128, - Float16, Float32, Float64), - Base.Threads.atomictypes) +let atomictypes = (Int8, Int16, Int32, Int64, Int128, + UInt8, UInt16, UInt32, UInt64, UInt128, + Float16, Float32, Float64) for T in atomictypes var = Atomic{T}() var[] = 42 @@ -493,7 +495,7 @@ function test_atomic_cas!(var::Atomic{T}, range::StepRange{Int,Int}) where T end end end -for T in intersect((Int32, Int64, Float32, Float64), Base.Threads.atomictypes) +for T in (Int32, Int64, Float32, Float64) var = Atomic{T}() nloops = 1000 di = threadpoolsize(:default) @@ -507,7 +509,7 @@ function test_atomic_xchg!(var::Atomic{T}, i::Int, accum::Atomic{Int}) where T old = atomic_xchg!(var, T(i)) atomic_add!(accum, Int(old)) end -for T in intersect((Int32, Int64, Float32, Float64), Base.Threads.atomictypes) +for T in (Int32, Int64, Float32, Float64) accum = Atomic{Int}() var = Atomic{T}() nloops = 1000 @@ -522,7 +524,7 @@ function test_atomic_float(varadd::Atomic{T}, varmax::Atomic{T}, varmin::Atomic{ atomic_max!(varmax, T(i)) atomic_min!(varmin, T(i)) end -for T in intersect((Int32, Int64, Float16, Float32, Float64), Base.Threads.atomictypes) +for T in (Int32, Int64, Float16, Float32, Float64) varadd = Atomic{T}() varmax = Atomic{T}() varmin = Atomic{T}() @@ -898,7 +900,7 @@ function _atthreads_greedy_dynamic_schedule() end @test _atthreads_greedy_dynamic_schedule() == threadpoolsize(:default) * threadpoolsize(:default) -function _atthreads_dymamic_greedy_schedule() +function _atthreads_dynamic_greedy_schedule() inc = Threads.Atomic{Int}(0) Threads.@threads :dynamic for _ = 1:threadpoolsize(:default) Threads.@threads :greedy for _ = 1:threadpoolsize(:default) @@ -907,7 +909,7 @@ function _atthreads_dymamic_greedy_schedule() end return inc[] end -@test _atthreads_dymamic_greedy_schedule() == threadpoolsize(:default) * threadpoolsize(:default) +@test _atthreads_dynamic_greedy_schedule() == threadpoolsize(:default) * threadpoolsize(:default) function _atthreads_static_greedy_schedule() ids = zeros(Int, threadpoolsize(:default)) @@ -1593,7 +1595,7 @@ end @sync begin for i in 1:n_tasks start_time_i = time_ns() - task_i = Threads.@spawn peakflops() + task_i = Threads.@spawn peakflops(1024) Threads.@spawn begin wait(task_i) end_time_i = time_ns() diff --git a/test/trimming/Makefile b/test/trimming/Makefile index 02e9fb5bdb257..c3145765655e7 100644 --- a/test/trimming/Makefile +++ b/test/trimming/Makefile @@ -16,7 +16,6 @@ endif # location of test source SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) JULIAHOME := $(abspath $(SRCDIR)/../..) -BUILDSCRIPT := $(BIN)/../share/julia/juliac-buildscript.jl include $(JULIAHOME)/Make.inc # get the executable suffix, if any @@ -24,35 +23,36 @@ EXE := $(suffix $(abspath $(JULIA))) # get compiler and linker flags. (see: `contrib/julia-config.jl`) JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "julia-config.jl"))' -- +JULIA_LIBDIR := $(shell $(JULIA) -e 'println(joinpath(Sys.BINDIR, "..", "lib"))' --) CPPFLAGS_ADD := CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags) LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) -ljulia-internal -#============================================================================= +# get the JuliaC build script +JULIAC_BUILDSCRIPT := $(shell $(JULIA) -e 'print(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "juliac", "juliac-buildscript.jl"))') -release: hello$(EXE) basic_jll$(EXE) +#============================================================================= -hello.o: $(SRCDIR)/hello.jl $(BUILDSCRIPT) - $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $(SRCDIR)/hello.jl --output-exe true +release: $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) -init.o: $(SRCDIR)/init.c - $(CC) -c -o $@ $< $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) +$(BIN)/hello-o.a: $(SRCDIR)/hello.jl $(JULIAC_BUILDSCRIPT) + $(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(JULIAC_BUILDSCRIPT) $< --output-exe true -basic_jll.o: $(SRCDIR)/basic_jll.jl $(BUILDSCRIPT) - $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) -e "using Pkg; Pkg.instantiate()" - $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $(SRCDIR)/basic_jll.jl --output-exe true +$(BIN)/basic_jll-o.a: $(SRCDIR)/basic_jll.jl $(JULIAC_BUILDSCRIPT) + $(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) -e "using Pkg; Pkg.instantiate()" + $(JULIA) -t 1 -J $(JULIA_LIBDIR)/julia/sys.$(SHLIB_EXT) --startup-file=no --history-file=no --project=$(SRCDIR) --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(JULIAC_BUILDSCRIPT) $< --output-exe true -hello$(EXE): hello.o init.o - $(CC) -o $@ $(WHOLE_ARCHIVE) hello.o $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) +$(BIN)/hello$(EXE): $(BIN)/hello-o.a + $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) -basic_jll$(EXE): basic_jll.o init.o - $(CC) -o $@ $(WHOLE_ARCHIVE) basic_jll.o $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) +$(BIN)/basic_jll$(EXE): $(BIN)/basic_jll-o.a + $(CC) -o $@ $(WHOLE_ARCHIVE) $< $(NO_WHOLE_ARCHIVE) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) -check: hello$(EXE) basic_jll$(EXE) - $(JULIA) --depwarn=error $(SRCDIR)/../runtests.jl $(SRCDIR)/trimming +check: $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) + $(JULIA) --depwarn=error $(SRCDIR)/trimming.jl $< clean: - -rm -f hello$(EXE) basic_jll$(EXE) init.o hello.o basic_jll.o + -rm -f $(BIN)/hello$(EXE) $(BIN)/basic_jll$(EXE) $(BIN)/hello-o.a $(BIN)/basic_jll-o.a .PHONY: release clean check diff --git a/test/trimming/Project.toml b/test/trimming/Project.toml index a0bf6688d9dd4..86c75e7d24639 100644 --- a/test/trimming/Project.toml +++ b/test/trimming/Project.toml @@ -1,3 +1,7 @@ [deps] JLLWrappers = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" + +[sources] +Zstd_jll = {path = "Zstd_jll"} diff --git a/test/trimming/Zstd_jll/Project.toml b/test/trimming/Zstd_jll/Project.toml new file mode 100644 index 0000000000000..467516843390a --- /dev/null +++ b/test/trimming/Zstd_jll/Project.toml @@ -0,0 +1,15 @@ +name = "Zstd_jll" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[deps] +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[compat] +julia = "1.6" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/test/trimming/Zstd_jll/src/Zstd_jll.jl b/test/trimming/Zstd_jll/src/Zstd_jll.jl new file mode 100644 index 0000000000000..c16413f963d0b --- /dev/null +++ b/test/trimming/Zstd_jll/src/Zstd_jll.jl @@ -0,0 +1,73 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## dummy stub for https://github.com/JuliaBinaryWrappers/Zstd_jll.j: +# +baremodule Zstd_jll +using Base, Libdl + +export libzstd, zstd, zstdmt + +# These get calculated in __init__() +libzstd_handle::Ptr{Cvoid} = C_NULL + +if Sys.iswindows() + const libzstd = "libzstd-1.dll" +elseif Sys.isapple() + const libzstd = "@rpath/libzstd.1.dylib" +else + const libzstd = "libzstd.so.1" +end + +if Sys.iswindows() + const zstd_exe = "zstd.exe" + const zstdmt_exe = "zstdmt.exe" +else + const zstd_exe = "zstd" + const zstdmt_exe = "zstdmt" +end + +if Sys.iswindows() + const pathsep = ';' +elseif Sys.isapple() + const pathsep = ':' +else + const pathsep = ':' +end + +if Sys.iswindows() +function adjust_ENV(cmd::Cmd) + dllPATH = Sys.BINDIR + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? dllPATH : "$dllPATH$pathsep$oldPATH" + return addenv(cmd, "PATH"=>newPATH) +end +else +adjust_ENV(cmd::Cmd) = cmd +end + +function adjust_ENV() + addPATH = joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR) + oldPATH = get(ENV, "PATH", "") + newPATH = isempty(oldPATH) ? addPATH : "$addPATH$pathsep$oldPATH" + return ("PATH"=>newPATH,) +end + +function zstd(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstd()) + end +end +function zstdmt(f::Function; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) # deprecated, for compat only + withenv((adjust_PATH ? adjust_ENV() : ())...) do + f(zstdmt()) + end +end +zstd() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstd_exe))`) +zstdmt() = adjust_ENV(`$(joinpath(Sys.BINDIR, Base.PRIVATE_LIBEXECDIR, zstdmt_exe))`) + +function __init__() + global libzstd_handle = dlopen(libzstd) + nothing +end + +end # module Zstd_jll diff --git a/test/trimming/Zstd_jll/test/runtests.jl b/test/trimming/Zstd_jll/test/runtests.jl new file mode 100644 index 0000000000000..5cfa2a1375c73 --- /dev/null +++ b/test/trimming/Zstd_jll/test/runtests.jl @@ -0,0 +1,7 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +using Test, Zstd_jll + +@testset "Zstd_jll" begin + @test ccall((:ZSTD_versionNumber, libzstd), Cuint, ()) == 1_05_07 +end diff --git a/test/trimming/basic_jll.jl b/test/trimming/basic_jll.jl index fc0137dd4eab2..a99e3f7692d27 100644 --- a/test/trimming/basic_jll.jl +++ b/test/trimming/basic_jll.jl @@ -1,14 +1,39 @@ -module MyApp - using Libdl -using Zstd_jll +using Zstd_jll # Note this uses the vendored older non-LazyLibrary version of Zstd_jll -Base.@ccallable function main()::Cint - println(Core.stdout, "Julia! Hello, world!") - fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString) +# JLL usage at build-time should function as expected +Zstd_jll.__init__() +const build_ver = unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ())) + +function print_string(fptr::Ptr{Cvoid}) println(Core.stdout, unsafe_string(ccall(fptr, Cstring, ()))) - println(Core.stdout, unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ()))) - return 0 end +function @main(args::Vector{String})::Cint + # Test the basic "Hello, world!" + println(Core.stdout, "Julia! Hello, world!") + + # JLL usage at run-time should function as expected + ver = unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ())) + println(Core.stdout, ver) + @assert ver == build_ver + + sleep(0.01) + + # Add an indirection via `@cfunction` / 1-arg ccall + cfunc = @cfunction(print_string, Cvoid, (Ptr{Cvoid},)) + fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString) + ccall(cfunc, Cvoid, (Ptr{Cvoid},), fptr) + + # map/mapreduce should work but relies on inlining and other optimizations + arr = rand(10) + sorted_arr = sort(arr) + tot = sum(sorted_arr) + tot = prod(sorted_arr) + a = any(x -> x > 0, sorted_arr) + b = all(x -> x >= 0, sorted_arr) + c = map(x -> x^2, sorted_arr) + # d = mapreduce(x -> x^2, +, sorted_arr) this doesn't work because of mapreduce_empty_iter having F specialized + # e = reduce(xor, rand(Int, 10)) + return 0 end diff --git a/test/trimming/hello.jl b/test/trimming/hello.jl index fef25f9e8558f..579ef4e18de38 100644 --- a/test/trimming/hello.jl +++ b/test/trimming/hello.jl @@ -1,13 +1,10 @@ -module MyApp - world::String = "world!" const str = OncePerProcess{String}() do return "Hello, " * world end -Base.@ccallable function main()::Cint +function @main(args::Vector{String})::Cint println(Core.stdout, str()) + foreach(x->println(Core.stdout, x), args) return 0 end - -end diff --git a/test/trimming/init.c b/test/trimming/init.c deleted file mode 100644 index ea1b02f8e5c8f..0000000000000 --- a/test/trimming/init.c +++ /dev/null @@ -1,9 +0,0 @@ -#include - -__attribute__((constructor)) void static_init(void) -{ - if (jl_is_initialized()) - return; - julia_init(JL_IMAGE_IN_MEMORY); - jl_exception_clear(); -} diff --git a/test/trimming/trimming.jl b/test/trimming/trimming.jl index a752c69460ad4..5d55ed62b03a8 100644 --- a/test/trimming/trimming.jl +++ b/test/trimming/trimming.jl @@ -1,12 +1,15 @@ using Test +@test length(ARGS) == 1 +bindir = dirname(ARGS[1]) + let exe_suffix = splitext(Base.julia_exename())[2] - hello_exe = joinpath(@__DIR__, "hello" * exe_suffix) - @test readchomp(`$hello_exe`) == "Hello, world!" - @test filesize(hello_exe) < 2000000 + hello_exe = joinpath(bindir, "hello" * exe_suffix) + @test readchomp(`$hello_exe arg1 arg2`) == "Hello, world!\n$hello_exe\narg1\narg2" + @test filesize(hello_exe) < 2_000_000 - basic_jll_exe = joinpath(@__DIR__, "basic_jll" * exe_suffix) + basic_jll_exe = joinpath(bindir, "basic_jll" * exe_suffix) lines = split(readchomp(`$basic_jll_exe`), "\n") @test lines[1] == "Julia! Hello, world!" @test lines[2] == lines[3] diff --git a/test/tuple.jl b/test/tuple.jl index 13af5ac992434..30782367803c5 100644 --- a/test/tuple.jl +++ b/test/tuple.jl @@ -208,7 +208,7 @@ end @test iterate(t, y3[2]) === nothing @test eachindex((2,5,"foo")) === Base.OneTo(3) - @test eachindex((2,5,"foo"), (1,2,5,7)) === Base.OneTo(4) + @test_throws DimensionMismatch eachindex((2,5,"foo"), (1,2,5,7)) @test Core.Compiler.is_nothrow(Base.infer_effects(iterate, (Tuple{Int,Int,Int}, Int))) end @@ -369,9 +369,9 @@ end @test !isless((1,2), (1,2)) @test !isless((2,1), (1,2)) - @test hash(()) === Base.tuplehash_seed - @test hash((1,)) === hash(1, Base.tuplehash_seed) - @test hash((1,2)) === hash(1, hash(2, Base.tuplehash_seed)) + @test hash(()) === Base.tuplehash_seed ⊻ Base.HASH_SEED + @test hash((1,)) === hash(1, Base.tuplehash_seed ⊻ Base.HASH_SEED) + @test hash((1,2)) === hash(1, hash(2, Base.tuplehash_seed ⊻ Base.HASH_SEED)) # Test Any32 methods t = ntuple(identity, 32) @@ -393,7 +393,7 @@ end @test !isless((t...,1,2), (t...,1,2)) @test !isless((t...,2,1), (t...,1,2)) - @test hash(t) === foldr(hash, [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,(),UInt(0)]) + @test hash(t) === foldr(hash, vcat(1:32, (), Base.HASH_SEED)) end @testset "functions" begin diff --git a/test/worlds.jl b/test/worlds.jl index 686708c5efd27..caededd329eff 100644 --- a/test/worlds.jl +++ b/test/worlds.jl @@ -5,6 +5,9 @@ using Base: get_world_counter, tls_world_age @test typemax(UInt) > get_world_counter() == tls_world_age() > 0 +# issue #58013 +@test_throws ArgumentError invokelatest() + # test simple method replacement begin g265a() = f265a(0) @@ -191,31 +194,26 @@ f_gen265(x::Type{Int}) = 3 # would have capped those specializations if they were still valid f26506(@nospecialize(x)) = 1 g26506(x) = Base.inferencebarrier(f26506)(x[1]) -z = Any["ABC"] +z26506 = Any["ABC"] f26506(x::Int) = 2 -g26506(z) # Places an entry for f26506(::String) in mt.name.cache +g26506(z26506) # Places an entry for f26506(::String) in MethodTable cache +w26506 = Base.get_world_counter() +cache26506 = ccall(:jl_mt_find_cache_entry, Any, (Any, Any, UInt), Core.methodtable.cache, Tuple{typeof(f26506),String}, w26506)::Core.TypeMapEntry +@test cache26506.max_world === typemax(UInt) +w26506 = Base.get_world_counter() f26506(x::String) = 3 -let cache = typeof(f26506).name.mt.cache - # The entry we created above should have been truncated - @test cache.min_world == cache.max_world -end -c26506_1, c26506_2 = Condition(), Condition() -# Captures the world age -result26506 = Any[] -t = Task(()->begin - wait(c26506_1) - push!(result26506, g26506(z)) - notify(c26506_2) -end) -yield(t) +@test w26506+1 === Base.get_world_counter() +# The entry we created above should have been truncated +@test cache26506.max_world == w26506 +# Captures the world age on creation +t26506 = @task g26506(z26506) f26506(x::Float64) = 4 -let cache = typeof(f26506).name.mt.cache - # The entry we created above should have been truncated - @test cache.min_world == cache.max_world -end -notify(c26506_1) -wait(c26506_2) -@test result26506[1] == 3 +@test cache26506.max_world == w26506 +f26506(x::String) = 5 +# The entry we created above should not have been changed +@test cache26506.max_world == w26506 +@test fetch(schedule(t26506)) === 3 +@test g26506(z26506) === 5 # issue #38435 f38435(::Int, ::Any) = 1 @@ -417,6 +415,55 @@ ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) "jl_method_table_insert" ] +# logging issue #58080 +f58080(::Integer) = 1 +callsf58080rts(x) = f58080(Base.inferencebarrier(x)::Signed) +invokesf58080s(x) = invoke(f58080, Tuple{Signed}, x) +# compilation +invokesf58080s(1) # invoked callee +callsf58080rts(1) # runtime-dispatched callee +# invalidation +logmeths = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1); +f58080(::Int) = 2 +f58080(::Signed) = 4 +ccall(:jl_debug_method_invalidation, Any, (Cint,), 0); +@test logmeths[1].def.name === :callsf58080rts +m58080i = which(f58080, (Int,)) +m58080s = which(f58080, (Signed,)) +idxi = findfirst(==(m58080i), logmeths) +@test logmeths[idxi+1] == "jl_method_table_insert" +@test logmeths[idxi+2].def.name === :invokesf58080s +@test logmeths[end-1] == m58080s +@test logmeths[end] == "jl_method_table_insert" + +# logging binding invalidations +struct LogBindingInvalidation + x::Int +end +makelbi(x) = LogBindingInvalidation(x) +const glbi = makelbi(1) +oLBI, oglbi = LogBindingInvalidation, glbi +flbi() = @__MODULE__().glbi.x +flbi() +milbi1 = only(Base.specializations(only(methods(makelbi)))) +milbi2 = only(Base.specializations(only(methods(flbi)))) +logmeths = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) +struct LogBindingInvalidation + x::Float64 +end +const glbi = makelbi(2.0) +@test flbi() === 2.0 +ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) +@test milbi1.cache.def ∈ logmeths +@test milbi2.cache.next.def ∈ logmeths +i = findfirst(x -> isa(x, Core.BindingPartition), logmeths) +T = logmeths[i].restriction +@test T === oLBI +@test logmeths[i+1] == "jl_maybe_log_binding_invalidation" +T = logmeths[end-1].restriction +@test T === oglbi +@test logmeths[end] == "jl_maybe_log_binding_invalidation" + # issue #50091 -- missing invoke edge affecting nospecialized dispatch module ExceptionUnwrapping @nospecialize @@ -478,6 +525,8 @@ Base.delete_method(fshadow_m2) @test Base.morespecific(fshadow_m3, fshadow_m1) @test !Base.morespecific(fshadow_m2, fshadow_m3) +@test_throws "Method of fshadow already disabled" Base.delete_method(fshadow_m2) + # Generated functions without edges must have min_world = 1. # N.B.: If changing this, move this test to precompile and make sure # that the specialization survives revalidation. @@ -508,7 +557,13 @@ struct FooBackdated FooBackdated() = new(FooBackdated[]) end -@test Base.invoke_in_world(before_backdate_age, isdefined, @__MODULE__, :FooBackdated) +# For depwarn == 1, this throws a warning on access, for depwarn == 2, it throws an error. +# `isdefinedglobal` changes with that, but doesn't error. +if Base.JLOptions().depwarn <= 1 + @test Base.invoke_in_world(before_backdate_age, isdefinedglobal, @__MODULE__, :FooBackdated) +else + @test !Base.invoke_in_world(before_backdate_age, isdefinedglobal, @__MODULE__, :FooBackdated) +end # Test that ambiguous binding intersect the using'd binding's world ranges module AmbigWorldTest @@ -534,3 +589,12 @@ module C57316; import ..X57316.Y57316 as Z, .Z.Y57316 as W; end @test !isdefined(B57316, :X57316) @test !isdefined(C57316, :X57316) @test !isdefined(C57316, :Y57316) + +# jl_module_import should always manipulate the latest world +module M57965 +function f() + @eval Random = 1 + Core._eval_import(true, @__MODULE__, nothing, Expr(:., :Random)) +end +end +@test_throws ErrorException("importing Random into M57965 conflicts with an existing global") M57965.f()s