Broken rails UJS loading when use arrow functions

When you have form with remote: true and add some js function, which use arrow function (just have it somewhere, no even need to call it) and submit such form in feature test then you get an unknown format error while in browser form works fine. Using standard function instead of arrow do not lead to such behavior.
phantomJS version: 2.1.1
poltergeist version: 1.15.0
system:  Ubuntu 14.04.5 LTS
debug output and stacktrace:
vagrant@vagrant-ubuntu-trusty-64:/vagrant/poltergeist_bug$ rspec spec/features/submit.rb
{"id":"8c523b10-0493-4f9d-a188-034bf1fad9ed","name":"set_debug","args":[true]}
{"command_id":"8c523b10-0493-4f9d-a188-034bf1fad9ed","response":true}
{"id":"8a56cbad-53bc-4428-bed6-f1df09df42be","name":"visit","args":["http://<IP_ADDRESS>:33684/"]}
{"command_id":"8a56cbad-53bc-4428-bed6-f1df09df42be","response":{"status":"success"}}
{"id":"af32d7ce-0c14-4ed2-96bb-6178e009b821","name":"find","args":["xpath",".//input[((((./@type = 'submit') or (./@type = 'reset')) or (./@type = 'image')) or (./@type = 'button'))][((./@id = 'Save') or ((./@value = 'Save') or (./@title = 'Save')))] | .//input[(./@type = 'image')][(./@alt = 'Save')] | .//button[(((./@id = 'Save') or ((./@value = 'Save') or (./@title = 'Save'))) or ((normalize-space(string(.)) = 'Save') or .//img[(./@alt = 'Save')]))] | .//input[(./@type = 'image')][(./@alt = 'Save')]"]}
{"command_id":"af32d7ce-0c14-4ed2-96bb-6178e009b821","response":{"page_id":1,"ids":[0]}}
{"id":"5d8110c2-99db-46ec-b411-4d74746b6ebf","name":"visible","args":[1,0]}
{"command_id":"5d8110c2-99db-46ec-b411-4d74746b6ebf","response":true}
{"id":"77eae35b-869f-4676-8b3d-343af33b321d","name":"disabled","args":[1,0]}
{"command_id":"77eae35b-869f-4676-8b3d-343af33b321d","response":false}
{"id":"ef6ac9a2-e386-4bd3-8758-6511ed01aa76","name":"click","args":[1,0]}
{"command_id":"ef6ac9a2-e386-4bd3-8758-6511ed01aa76","response":{"position":{"x":33.5,"y":42.5}}}
{"id":"41b4a5e0-bce7-45a6-b3f7-ca01358e9dc8","name":"find","args":["xpath","/html"]}
{"command_id":"41b4a5e0-bce7-45a6-b3f7-ca01358e9dc8","response":{"page_id":2,"ids":[0]}}
{"id":"f7062392-1d40-4beb-aab0-acaf2271a54b","name":"visible","args":[2,0]}
{"command_id":"f7062392-1d40-4beb-aab0-acaf2271a54b","response":true}
{"id":"9a403d5a-4bc6-4e54-8737-275454fef778","name":"visible_text","args":[2,0]}
{"command_id":"9a403d5a-4bc6-4e54-8737-275454fef778","response":"Internal Server Error\n\nBaseController#create is missing a template for this request format and variant. request.formats: [\"text/html\"] request.variant: []\nWEBrick/1.3.1 (Ruby/2.3.1/2016-04-26) at <IP_ADDRESS>:33684"}
{"id":"a178fdd3-20ea-42d6-aaec-f57359098a0e","name":"find","args":["xpath","/html"]}
{"command_id":"a178fdd3-20ea-42d6-aaec-f57359098a0e","response":{"page_id":2,"ids":[1]}}
{"id":"cf860fbe-ecef-46fe-b378-840e43c3ae34","name":"visible","args":[2,1]}
{"command_id":"cf860fbe-ecef-46fe-b378-840e43c3ae34","response":true}
{"id":"47859a23-5e3a-4880-b3b0-32f38319ef81","name":"all_text","args":[2,1]}
{"command_id":"47859a23-5e3a-4880-b3b0-32f38319ef81","response":"Internal Server Error\n  \n    Internal Server Error\n    BaseController#create is missing a template for this request format and variant.\n\nrequest.formats: [\"text/html\"]\nrequest.variant: []\n    \n    \n     WEBrick/1.3.1 (Ruby/2.3.1/2016-04-26) at\n     <IP_ADDRESS>:33684\n    \n  \n\n"}
{"id":"52d65b8b-be30-43f9-b0a0-40896a235e7e","name":"reset","args":[]}
{"command_id":"52d65b8b-be30-43f9-b0a0-40896a235e7e","response":true}
F

Failures:

  1) Submit form via AJAX submit
     Failure/Error: raise ActionController::UnknownFormat, message

     ActionController::UnknownFormat:
       BaseController#create is missing a template for this request format and variant.

       request.formats: ["text/html"]
       request.variant: []
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/etag.rb:25:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/conditional_get.rb:38:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/head.rb:12:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/session/abstract/id.rb:232:in `context'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/session/abstract/id.rb:226:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/railties-5.0.5/lib/rails/rack/logger.rb:36:in `call_app'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/railties-5.0.5/lib/rails/rack/logger.rb:24:in `block in call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/railties-5.0.5/lib/rails/rack/logger.rb:24:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/method_override.rb:22:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/runtime.rb:22:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/sendfile.rb:111:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/railties-5.0.5/lib/rails/engine.rb:522:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/urlmap.rb:68:in `block in call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/urlmap.rb:53:in `each'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/urlmap.rb:53:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/capybara-2.15.1/lib/capybara/server.rb:44:in `call'
     # /home/vagrant/.rvm/gems/ruby-2.3.1/gems/rack-2.0.3/lib/rack/handler/webrick.rb:86:in `service'
     # ------------------
     # --- Caused by: ---
     # Capybara::ExpectationNotMet:
     #   expected to find text "form submitted succesfully" in "Internal Server Error BaseController#create is missing a template for this request format and variant. request.formats: [\"text/html\"] request.variant: [] WEBrick/1.3.1 (Ruby/2.3.1/2016-04-26) at <IP_ADDRESS>:33684"
     #   /home/vagrant/.rvm/gems/ruby-2.3.1/gems/capybara-2.15.1/lib/capybara/node/matchers.rb:617:in `block in assert_text'

Repository to reproduce with more verbose description - https://github.com/sd-kin/poltergeist_bug/tree/master
P.S. Sorry for my english

PhantomJS 2.1.1 only supports up to ES5 - therefore it doesn’t support arrow functions. Either try the PhantomJS 2.5 beta or transpile your code to ES5


Incorrect lifetime deposit warning on Console

The Chore
I am trying to deposit funds from a brand new metamask key but getting the below on testnet

The key is 0x93e24583590464040188e5573f7Be660E1a396D2
I'm on Arbitrum sepolia.
This is the asset details so in theory lifetime deposit should be 100,000
{
  "asset": {
    "id": "520c3d2c121ba80186425d5c81d00c332d7bac7948835abced657c7faa6b8f4c",
    "details": {
      "name": "Tether USD",
      "symbol": "USDT",
      "decimals": "6",
      "quantum": "1000000",
      "erc20": {
        "contractAddress": "0x33Fdc12616Aa4E7901eDA4D17a73Dd4f573f24E8",
        "lifetimeLimit": "100000000000",
        "withdrawThreshold": "1",
        "chainId": "421614"
      }
    },
    "status": "STATUS_ENABLED"
  }
}

UPDATE
I just used the arbitrum contract to make myself an exempt depositor ... and I still cant get past here. ... something is definitely broken.

Additional details / background info

This working as expected ... the bridge is broken

Closing this .... pretty confident it is the bridge on testnet specifically .... will monitor to see if any other instances occur.


Enhancement: Step animations through custom easings

Hi,
I wanted to animate a background image (sprite sheet) to replace a css3 animation. The current CSS equivalent is:
@keyframes play {
  from { background-position:    0px; }
  to { background-position: -600px; }
}

.loader {
    width: 20px;
    height: 20px;
    background-image: url("../img/loader.png");
    @include animation(play 1s steps(10) infinite);
}

My current implementation with velocity is:
window.setInterval(function() {
    $element.velocity({
        'background-position-x': ['-600px', '0px']
    }, {
        duration: 1000
    });
}, 1000);

But it animates the background position with a step that I can't control. I would like to increment it by 20px each frame (the width of the loader), otherwhise it won't work as expected.
Is this possible with velocity ? Maybe with the easing function ?

Another use of steps :)
http://cssanimation.rocks/post/twitter-fave/

thanks, @ocombe! awesome use


Don't break on invalid .pcap file

When an invalid .pcap file was POSTed and parsing failed, the bogous pcap would stay in the queue and get reparsed over and over again. Now we just ignore the bad pcap and move on.
The pcaps are still kept on disk and ignored on startup.
Fixes #18

i'm concerned with silently ignoring errors - some errors might be transient (when free memory is low or something else)
i don't know what a good solution is we should discuss this...

i think i covered the needed changes from this pr in my change, i'll close this pr, please re-open if i missed something.


Counting RMB clicks

Needs to be done accurately. Current branch is WIP

also might want to include keboard presses. Either individual or all


Big File Upload

Big File Upload for OS practice

!!

zhangwentao

zhangwentao

Some notes after reviewing code:
https://github.com/nsknojj/notebook_task/tree/master/notes

@nsknojj
I recommend you writing notes using markdown format, which could be automatically rendered as HTML format.

I just figure out what I should do. I need to change the php server "Jquery-file-upload" provided  to a new server side with "tornado" framework. Big file upload uses post method with "Content-Type: multipart/form-data". Luckily, it is well-supported by tornado.

Maybe implementing the backend in tornado is more complex.
You can use one apache/php server for all users. The server puts every file to the correct user's directory. In this way, you just need to pass username to server from browser and then the server gets the username and puts the file in the right path.
But maybe implementing the backend in tornado is not so complex. I noticed there is a GAE python example of backend in source code. And it is not complex. So this method is also ok.
Which way ? Up to you.
Go ahead.

I prefer using tornado.
The problem of using Apache server is that we have to set up the server just for file uploading. However, the other functions, like downloading, previewing, are all supported by tornado. That will confuse developers and administrators.
If uploading function is provided by tornado, then we do not have to bother setting up a third apache server.

Today, I have implemented the basic Multi-Blob upload function in tornado server side,
https://github.com/nsknojj/notebook_task/blob/master/handlers.py
In this version, the upload file is saved in current directory, and the response of POST is incorrect.

Almost completed
https://github.com/nsknojj/notebook/tree/big-upload


Bug Fix Request: Re-enable 3 checks in cron job

Describe the bug
Currently, the CRON pushes scores of 16 checks to bigQuery and does not include scores for the following 3 checks CI_TEST, SAST, and CONTRIBUTOR.
Expected behavior
We expect all the scores to be available in the processed data dump. If the scorecards application was unable to compute it then possibly assigning a score value like -1 or any other pre-defined value can help us ignore those scores.
Additional context
I can think of the following 2 reasons -
A) There was a design decision taken to exclude these metrics. That's might be one of the reasons why previous issues that asked to re-enable this  https://github.com/ossf/scorecard/issues/1302 were put to backlog.
B) This has been missed out to include somehow. If that is the case, can we enable those now?

Some of these checks were taken out because they are API intensive so rate limiting can be a bottleneck.
@azeemsgoogle any plan to add these checks?

Disabling these checks is a conscious decision for being able to scale the weekly job (see https://github.com/ossf/scorecard/blob/main/cron/config/config.yaml#L27). We need to optimize these checks for API usage before we enable them in the cron job. As of now, I don't plan to work on adding these back. Happy if someone wants to take this on.
We should probably explicitly call out in our documentation though that not all checks are run during the weekly cron job.

We could add an extra column in the https://github.com/ossf/scorecard#scorecard-checks. @Parth59 interested in sending a quick PR for it?

I can take a look at this

My plan is to expand the graphQL API call to fetch all the check info in the initial API call. Something like this:
query {
  repository(owner:"ossf", name:"scorecard") {
    object(expression: "heads/main") {
      ... on Commit {
        checkSuites(first: 10) {
          edges {
            node {
              app{
                slug
              }
              conclusion
              status
            }
          }
        }
      }
    }
  }
}

which will produced the same info used by Client.Checks.ListCheckRunsForRef
{
  "node": {
    "app": {
      "slug": "dependabot"
    },
    "conclusion": null,
    "status": "QUEUED"
  }
}

In CI-Tests, the issue comes from this for loop - https://github.com/ossf/scorecard/blob/2b206dc3654ab96b88310625c5f0a1a3902723e7/checks/ci_tests.go#L53-L54
We basically need to make the ListChecksRunsForRef call for all commits returned by ListCommits which is what makes this check expensive.
The other issue is - we call ListCheckRunsForRef and ListStatuses on pr.HeadSHA, not on the merge commitSHA. If we were to look for the statuses on the merge commit it changes the meaning of the check - instead of checking whether CI-Tests ran before a PR got merged, we look for whether CI-Tests ran after the PR got merged.

We basically need to make the ListChecksRunsForRef call for all commits returned by ListCommits which is what makes this check expensive

Yep. That's what I was hoping to avoid by expanding graphqlData to include checkSuites so we can get it all in one call instead of 30 (or whatever commitsToAnalyze is set) API calls.  I think this would still increase the cost of the query though. I'll play around with if this is an improvement or not.

instead of checking whether CI-Tests ran before a PR got merged, we look for whether CI-Tests ran after the PR got merged

I'll keep this in mind to make sure to preserve existing behavior

I think this would still increase the cost of the query though. I'll play around with if this is an improvement or not.

Yes, although graphQL cost might still be better than 30x REST API calls. You can use the e2e tests to understand how a change affects the query cost - https://github.com/ossf/scorecard/blob/main/clients/githubrepo/graphql_e2e_test.go. Note that some increase in cost is ok and expected.

The SAST check has been enabled as part of #2223.
The next BigQuery cron job to pick it up will be the 9/5/22 run and the data should be available by 9/12/22

SAST fixed; still need to address Contributor, CI Test, Dependency Update


bear (build tool): 2.2.0 -> 2.2.1 + ignore wrapper

Motivation for this change
This fixes #20056.
Things done

[x] Tested using sandboxing
(nix.useSandbox on NixOS,
or option build-use-sandbox in nix.conf
on non-NixOS)
Built on platform(s)

[x] NixOS
[ ] OS X
[ ] Linux

[x] Tested compilation of all pkgs that depend on this change using nix-shell -p nox --run "nox-review wip" (no package depends on bear)
[x] Tested execution of all binary files (usually in ./result/bin/)
[x] Fits CONTRIBUTING.md.

Hmm, the build also fails on hydra.
Am I correct that Darwin stdenv does not support i386, and, thus, set(CMAKE_OSX_ARCHITERCTURES ...) should be removed from CMakeLists.txt?

Oh, I haven't noticed they set it explicitly. IIRC we once did support powerpc for it but probably never i386.

That patch seems not to do what it's meant for, at least in my setup.

I picked the update at least.

Could you, please, explain your setup? I tested it on an example from the issue and on the bear itself. For the former I get only `:
[
    {
        "command": "c++ -c -o CMakeFiles/bear_test.dir/test.cpp.o -B/nix/store/9nw5abnvdw91vrgv1z9x06560nxhyka8-gcc-5.4.0-lib/lib -B/nix/store/1jwkjzrivp541v5ibgzb4wc2avla5zx9-glibc-2.24/lib/ -idirafter /nix/store/s222vly64qrr3l4mqm6n3fsrkprhrnmm-glibc-2.24-dev/include -idirafter /nix/store/psmdlfqys1031hhyjhky4qphgyscmgdg-gcc-5.4.0/lib/gcc/x86_64-unknown-linux-gnu/5.4.0/include-fixed -B/nix/store/wckiwf1m333akbm3d7pyrj57g3i39367-gcc-wrapper-5.4.0/bin/ -O2 -D_FORTIFY_SOURCE=2 -fstack-protector-strong --param ssp-buffer-size=4 -fPIC -fno-strict-overflow -Wformat -Wformat-security -Werror=format-security /run/user/1111/tmp.i2CRzRzekp/test.cpp",
        "directory": "/run/user/1111/tmp.i2CRzRzekp",
        "file": "/run/user/1111/tmp.i2CRzRzekp/test.cpp"
    }
]

Ok, I have tested it on an another autotools-based package and it does not work.

Now it works with an autotools-based project that I've been using for testing. @vcunat could you, please, check if it works on your setup.

I expect this will work. I can probably check tomorrow.

Yes, thanks a lot! The mysteriously missing symbols in YCM are now gone.

I picked the fix to 16.09, too.


Would it be possible to match the song length from Gaana/Saavn?

So It seems to download the audio from the music video and the video is longer than the actual song and they have people talking in the beginning or in the middle before the actual song starts. If there was a way to match the length maybe it will get the actual song how it is in the website.

We can definitely try this!
For now, we match songs that have duration equal or less than 15 minutes (so as to avoid downloading compilation videos).
Thanks for the input. I will surely think about this.


Update the Blazor WebAssembly debugging instructions to include steps for Edge(ium) and (soon) Firefox

Currently the debug instructions for Blazor WebAssembly apps is specific to Chrome, but debugging works works with Edge(ium) today and will soon with Firefox.

Closing this as we have a separate issue tracking this.


'Show icon in dock' checkbox for non-admin users

(Related to issue #356.)
Now that (as of ß64?) hiding/showing the dock icon appears to be controlled by a user-level preference, rather than a setting in Quicksilver's Info.plist file, could the 'Show icon in dock' checkbox in the preferences be enabled again for non-admin users?
I run in a non-admin account, and had previously set Quicksilver not to show an icon in the dock (by editing the Info.plist or using Dockless). When I updated from ß63 to ß64, however, Quicksilver appeared in the dock, no matter how many times I edited the Info.plist file and relaunched it. Eventually I realised (by seeing what happened when I ran Quicksilver as an admin user) that the dock icon visibility is now controlled by the 'Hide Dock Icon' key in the user-level preferences file (i.e. ~/Library/Preferences/com.blacktree.quicksilver.plist).
Therefore, in order to make the dock icon go away, I had to manually edit that file and add the necessary key/value pair. Clearly it would have been a lot simpler if the checkbox hadn't been greyed out, and I could just click it instead!

I think the problem here is that the icon is greyed out. It is right that
Quicksilver now correctly respects the user settings (as opposed to those
in the Info.plist) so we will work on fixing this.
Thank you for notifying us of this
On 5 February 2012 20:54, David Shepherdson <
<EMAIL_ADDRESS>
wrote:

(Related to issue #356.)
Now that (as of 64?) hiding/showing the dock icon appears to be
controlled by a user-level preference, rather than a setting in
Quicksilver's Info.plist file, could the 'Show icon in dock' checkbox in
the preferences be enabled again for non-admin users?
I run in a non-admin account, and had previously set Quicksilver not to
show an icon in the dock (by editing the Info.plist or using Dockless).
When I updated from 63 to 64, however, Quicksilver appeared in the dock,
no matter how many times I edited the Info.plist file and relaunched it.
Eventually I realised (by seeing what happened when I ran Quicksilver as an
admin user) that the dock icon visibility is now controlled by the 'Hide
Dock Icon' key in the user-level preferences file (i.e.
~/Library/Preferences/com.blacktree.quicksilver.plist).
Therefore, in order to make the dock icon go away, I had to manually edit
that file and add the necessary key/value pair. Clearly it would have been
a lot simpler if the checkbox hadn't been greyed out, and I could just
click it instead!

Reply to this email directly or view it on GitHub:
https://github.com/quicksilver/Quicksilver/issues/686

Perfect -- all seems to be working well in ß65. Thanks to all involved -- Quicksilver really seems to be going from strength to strength these days!


How to create the schema.graphql file ?

Hi,
I wonder if anyone can help ? I know this format is used by other programs but I am just wanting to create a schema file from an endpoint.
I have the following
projects:
  blog:
    schemaPath: schema.graphql
    extensions:
      endpoints:
        default: "http://localhost:3000/___graphql"

Is there some kind of util to create this schema.graphql file ?

I found the following https://github.com/Urigo/graphql-cli


Update global vars with env values.

Update the global vars value with the environment variable values.

@andypham I added some comments on your PR. Could you please take a look?
Thanks for your contribution!

@andypham I am closing this PR since your changes have been manually included on the CLI.
Thanks so much for your contribution!

@guarinogabriel Sorry for the slow respond as I was away for the 4th July holiday. I added the comment for the EDITOR env variable.


Eng 1488 add breadcrumbs to the app & update nav layout

Describe your changes and why you are making these changes
Update to match the Figma mockup.
Related issue number (if any)
Eng 1488
Loom demo (if any)
https://user-images.githubusercontent.com/30596854/194920965-705407ad-9f5b-44e2-bf3d-ddbd71602b2c.mov
Checklist before requesting a review

[x] I have created a descriptive PR title. The PR title should complete the sentence "This PR...".
[x] I have performed a self-review of my code.
[x] I have included a small demo of the changes. For the UI, this would be a screenshot or a Loom video.
[N/A] If this is a new feature, I have added unit tests and integration tests.
[N/A] I have run the integration tests locally and they are passing.
[x] I have run the linter script locally (See python3 scripts/run_linters.py -h for usage).
[x] All features on the UI continue to work correctly.
[x] Added one of the following CI labels:

run_integration_test: Runs integration tests
skip_integration_test: Skips integration tests (Should be used when changes are ONLY documentation/UI)

Currently using a placeholder for the aqueduct logo. Would appreciate a link to a transparent version I can use.

Also, a demo showing that the account & notification buttons still work:
https://user-images.githubusercontent.com/30596854/194922046-9a4a49f5-5052-4b51-ac53-8e349cc87d86.mov

Code here looks good to me so far.
Going to do some testing on my machine later this afternoon and I can give some more detailed design feedback.

@agiron123 The logo size should be 64x64.

@agiron123 The logo size should be 64x64.

Okay great, try using this logo:
https://aqueduct-public-assets-bucket.s3.us-east-2.amazonaws.com/webapp/logos/aqueduct-logo-light/1x/logo_light_blue.png
If it's too blurry, I'll export it again and upload it to a file of the same name, so there shouldn't be any changes on your part other than setting the path to the image above.


[feat] Support Toml language

This pure java library is supporting Toml: https://github.com/tomlj/tomlj

Looks like a good (and easy) candidate for another subproject.
If anyone would like to contribute, the implementation should be quite small similar to https://github.com/keykey7/confij/blob/master/confij-yaml/src/main/java/ch/kk7/confij/source/format/YamlFormat.java .


Feature request: syntax highlighting for Pipfile and Pipfile.lock

It's great if this feature is integrated into vscode-python. Thanks

So a pipfile is TOML and a pipfile.lock is JSON, so we just need to add the appropriate associations. The latter is included in VS Code out of the box, but TOML isn't. The most popular TOML extension doesn't recoginize Pipfile, but I will send a PR for them to add it.

But in both instances we can add an association on our side as well to at least make it work when a TOML extension that has syntax highlighting is available.

Or if you want to submit a PR to add the association, @hujianxin , that would be awesome 😄


Builds initiated via workflow dispatch on main deploy inappropriately

Overview
Yesterday I ran a workflow_dispatch build on the main branch to verify that the many accumulated changes and bugfixes were actually going to pass tonight. It seems that this resulted in actual data deployment which we did not intend. See the /main paths on both our GCS and S3 buckets:
gsutil ls gs://pudl.catalyst.coop/main/
aws s3 ls --no-sign-request s3://pudl.catalyst.coop/main/

Seems like this is a result of some faulty logic in our altogether too complex conditionals for deciding when to deploy build outputs.
Logs from the misbehaving build can be obtained by:
gsutil cp gs://builds.catalyst.coop/2024-05-15-1655-cb73d689b-main/2024-05-15-1655-cb73d689b-main-pudl-etl.log .

Success Criteria

[ ] We have removed the spurious /main paths from S3 and GCS
[ ] A workflow_dispatch run of build-deploy-pudl on main does not deploy data to production destinations.
[ ] workflow_dispatch on other branches also do not deploy data to production destinations.
[ ] Tagged versions on main DO deploy to production destinations.

* [ ] Remove `s3://pudl.catalyst.coop/main/`
* [ ] Remove `gs://pudl.catalyst.coop/main/`
* [ ] Verify that we don't have a spurious data release on production Zenodo
* [ ] Make sure that the deployed Datasette is based on the currently `/nightly` outputs.
* [ ] Identify the faulty logic in our nightly build script or `build-deploy-pudl` workflow and fix it.

Punting this until next sprint since we're not focusing on the build script refactor/quality for this sprint.

Putting this into next sprint, since we'll be doing yet another release in November and this seems to be a problem again.


Adding New address data

Hi!
Supposing I add new data in OSM how long would it take it to re-train the model?
In fact, if you can give info on the below list - there will be a lot of good contributions / improvements that can happen.

What all things to add in OSM - for a single address or for a bunch of addresses
Where to add the data - openstreetmap or opencage .. which is the correct URL that can be used
In spite of your repeated remarks that custom data is not welcome.. still I think it has a strong use case - especially when I want to achieve 99% accuracy for a single country; also the consistency of the result - the same address should not produce two different results. this is happening in libpostal now.
And at last my encounter with an error in address-parser-training.exe
I created a sampletsv.tsv that ha sjust one line as below

en in 123/housenumber /FSEP SPRoad/road /FSEP gandhicolony/locality /FSEP city/navimumbai /FSEP mh/state /FSEP 400001/pincode
I get the error as below (see last few lines)
PS C:\apps\libpostal\src> .\address_parser_train.exe sampletsv.tsv trained                                              INFO  address dictionary module loaded
at main (address_parser_train.c:1245)
INFO  transliteration module loaded
at main (address_parser_train.c:1253)
INFO  Done with vocab, total size=5
at address_parser_init (address_parser_train.c:665)
INFO  After pruning vocab size=0
at address_parser_init (address_parser_train.c:679)
INFO  Creating phrases trie
at address_parser_init (address_parser_train.c:682)
INFO  hash_size=0
at trie_new_from_hash (trie_utils.c:14)
INFO  Calculating phrase types
at address_parser_init (address_parser_train.c:687)
INFO  num_classes = 5
at address_parser_init (address_parser_train.c:690)
INFO  Creating vocab trie
at address_parser_init (address_parser_train.c:693)
INFO  hash_size=0
at trie_new_from_hash (trie_utils.c:14)
INFO  Creating phrase_types trie
at address_parser_init (address_parser_train.c:725)
INFO  Creating phrases trie
at address_parser_init (address_parser_train.c:799)
INFO  hash_size=0
at trie_new_from_hash (trie_utils.c:14)
INFO  Creating postal codes trie
at address_parser_init (address_parser_train.c:827)
INFO  hash_size=0
at trie_new_from_hash (trie_utils.c:14)
INFO  Building postal code contexts
at address_parser_init (address_parser_train.c:856)
INFO  Freeing memory from initialization
at address_parser_init (address_parser_train.c:898)
INFO  Finished initialization
at main (address_parser_train.c:1262)
INFO  num_classes 5
at crf_averaged_perceptron_trainer_new (crf_trainer_averaged_perceptron.c:77)
INFO  Doing epoch 0
at address_parser_train (address_parser_train.c:1116)
INFO  Shuffling
at address_parser_train (address_parser_train.c:1121)
shuf: sampletsv.tsv: Device or resource busy
ERR   Error in shuffle
at address_parser_train (address_parser_train.c:1124) errno:Result too large
ERR   Error in training
at main (address_parser_train.c:1265) errno:Result too large
best regards
NK

OK. Updated..
Made progress. Now no error. Following @rinigus steps as in #132:
.\address_parser_train.exe sampletsv.tsv c:/apps/libpostal/src/nk --iterations 2 --model crf
but nothing happened in /nk directory - no .dat files created..
Any help?
regards
NK

@rinigus
If you can spare some time on writing clear step by step instructions with full examples of dir paths, filenames etc on how to train with custom dataset, it will be of great help.
Also how the custom dataset should look like - say how the data in the .tsv file should be.
best regards
NK

my scripts are at https://github.com/rinigus/geocoder-nlp/tree/master/scripts/postal . please see
In your command, I don't know whether windows path can be put that way. but I don't work on windows and have only occasional experience with it.

@rinigus , first of all thank you for responding.  I will study your scripts and commands.
I have generated the .DAT files using address_parser_training.exe. these files are generated in
/c/apps/libpostal/src/nk
Also ran the configure for the datadir like this:
./configure datadir=/c/apps/libpostal/src/nk
Ran  without any error messages.
How to make the address_parser use these .dat files?
Should I run MAKE again?
Or copy these files into some folder where address_parser is looking?
Can both sets co-exist - old .dat files originally downloaded from libpostal + .dat files of my own?
How do we know that address_parse is using the new .dat files??
best regards
NK

You can use setup functions for it, such as libpostal_setup_parser_datadir . That's what I use in the geocoder-nlp written around libpostal. See https://github.com/rinigus/geocoder-nlp/blob/master/src/postal.cpp for corresponding part.


cleanup contrib ansible task :Indent yaml

cleanup contrib ansible task :Indent yaml

=== Failed
=== FAIL: pkg/imageverifier/bindir TestBinDirVerifyImage/max_verifiers_=_-1,_with_timeout (5.40s)
time="2023-12-21T22:11:19Z" level=error msg="error reading stdout" error="read |0: i/o timeout"
time="2023-12-21T22:11:19Z" level=error msg="error flushing stdout" error="read |0: i/o timeout"
time="2023-12-21T22:11:19Z" level=debug msg="error logging image verifier stderr" error="read |0: i/o timeout" image_verifier=verifier-2 stream=stderr
time="2023-12-21T22:11:19Z" level=error msg="error flushing stderr" error="read |0: i/o timeout"

This pr changed may have not relation with  CI / MacOS unit tests (pull_request) Failing after 4m

What's going on about contrib/ansible/tasks/bootstrap_ubuntu.yaml.
Is there a new plan for that?


cli segfaults on power with rhel & centos

This is kind of a question-issue. I set up the runtime on a RHEL 7.5 system, but then when I tried using it via docker commands, I got a segfault in the cli. So, running just the cli, at one time (I thought) I at least got some usage instructions. Now just a segfault. Could it be that a prereq is checked for first? It doesn't look like that's the case.
$ /usr/bin/nvidia-container-cli --help
Segmentation fault

I built a debug version of the cli and attached gdb to it. But putting my first breakpoint at main hit this too, so it's pretty immediate, which is weird.
If this is not expected, I can provide more detail of course.

Is that ppc64le?

Did you install it from the repository?
Can you check that the CLI and the library come from the same build:
run ldd /usr/bin/nvidia-container-cli, then execute the path to libnvidia-container.so.1 and check that the build revision matches /usr/bin/nvidia-container-cli -V (or  strings /usr/bin/nvidia-container-cli | grep <BUILD_ID_HASH> if this segfaults too)

@3XX0 yah, I did.
for a centos recreate i installed a new vm and installed everything from repos.
and yes, running libnvidia-container.so.1 segfaults too, so here's the build info via strings for those two files:
nvidia-container-cli:
version: %s
1.0.0
build date: %s
2018-04-30T10:06+0000
build revision: %s
163054a04b21c4455c8cae7e47873d9f2a091f55
build compiler: %s
gcc 4.8.5 20150623 (Red Hat 4.8.5-16)
build platform: %s
ppc64le
'strings /lib64/libnvidia-container.so.1 `
version: %s
1.0.0
build date: %s
2018-04-30T10:06+0000
build revision: %s
163054a04b21c4455c8cae7e47873d9f2a091f55
build compiler: %s
gcc 4.8.5 20150623 (Red Hat 4.8.5-16)
build platform: %s
ppc64le
so looks like they're from the same build.

Weird, can you check that the path to the interpreter is actually correct:
objdump -j .interp -d /lib64/libnvidia-container.so.1
I'll try to get my hands on a PPC machine see if I can reproduce

$ objdump -j .interp -d  /lib64/libnvidia-container.so.1

/lib64/libnvidia-container.so.1:     file format elf64-powerpcle

Disassembly of section .interp:

0000000000030a78 <.interp>:
   30a78:       2f 6c 69 62     ori     r9,r19,27695
   30a7c:       36 34 2f 6c     xoris   r15,r1,13366
   30a80:       64 36 34 2e     cmpdi   cr4,r20,13924
   30a84:       73 6f 2e 32     addic   r17,r14,28531
        ...

:/

Looks like on power you need -s instead of -d.
The hexa translates to /lib64/ld64.so.2

:D
Well, then, that looks sane:
$ ls -lah /lib64/ld64.so.2
lrwxrwxrwx. 1 root root 10 May 29 18:14 /lib64/ld64.so.2 -> ld-2.17.so
]$ ls -lah /lib64/ld-2.17.so
-rwxr-xr-x. 1 root root 217K Apr 10 04:39 /lib64/ld-2.17.so

@clnperez Any way I can get my hands on a VM somewhere?
I suppose I don't need GPUs to reproduce, so it would speed up the process rather than waiting for one internally.

Hi,
I confirm, I have exactly the same problem on my power with RHEL 7.5. Running libnvidia-container.so.1 segfaults too...

@3XX0 here are some options: https://developer.ibm.com/linuxonpower/cloud-resources/
The fastest would be SiteOx. The most cost-effective (free!) would be OSU or PDP. LMK if you have any questions. I can set you up with another OSU VM, or, if you request one yourself, put me as the contact/sponsor.

Here is what dmesg returns:
 kernel: libnvidia-conta[22745]: unhandled signal 11 at<PHONE_NUMBER>2f0000 nip 00003fff78511214 lr 00003fff78505648 code 30002

So I managed to reproduce with this base which seems to indicate there has been an ABI break since then (the new one being here.
@clnperez any idea?

@clnperez  Is there another way to build a container with cuda and nvidia drivers for this architecture (ppc64le) with RHEL 7.5?

Any solution guys ?

This should be fixed in RC 2. Feel free to reopen if that's not the case.


CompareImage: Feature a consistent autoscale

Feature a consistent autoscale for the CompareImage.
Now

the autoscale statistics are computer globally for the 2 images
it is also computed on the raw image (instead of in the sliced left/right image)
This makes the autoscale usable, from the ColormapDialog, including 3stddev
subsequently when you are about to slice, the statistics are not recomputed -> that's faster

Changelog:

CompareImage: Colormap autoscale is now global to the 2 raw images to compare
CompareImage: updateColormap= was dropped, this can be set with getColormap()

I have committed an alternative implementation to drop the custom colormap class.

The virtual item is used to hold the concatenation of the 2 images
When the colormap is changed it update a sealed version of the colormap for the displayed images

That maybe a bit less efficient than the previous implementation, but this could be improved, and for now who cares.
The good thing is that the displayed histogram is properly computed from the concatenation (previously the histogram was computed from the crop of one of the images)

Can you rebase to include commit https://github.com/silx-kit/silx/commit/ccdc13bbd4b8aa5097958ad8f3e0fd6e20b93cf6 from main to fix CI on Windows?

While i was here, i have used the auto scale to improve the contrast on the composite image comparison.
The result is probably not extactly the same for the B&W composition. @payno would you have some times to check if it's fine for you?

@payno thinking about that a bit more, i think it is better to revert my change on the compare image for the B&W composition, it would be better to talk about together before merging.

I'd be +1 to merge this PR as it was and work on additional feature/fix in another PR.

@t20100 i am fine with it, let's merge it as it is


Feat: add an option to disable provisioner

Description
Fix https://github.com/aws/karpenter/issues/2491
To stop provisioner from provision more nodes when things going wrong, like there are pending pods.
How was this change tested?
Not tested yet. The test currently failed.
By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

I think we likely need a more holistic solution to control different actions that karpenter takes, e.g.

provisioning
deletion
replacement
drift
interruption
expiration

Why not set limits to 0? Do you want to disable scale down, too?

Hey @yizhang-zen, wanted to give you an update. Like https://github.com/aws/karpenter-core/pull/66#issuecomment-1315975575 says we're working on designing a solution to see if we'll need more granular controls.

Hey @yizhang-zen, unfortunately I'm going to be closing the PR. For the remaining deprovisioning scope for v1.0.0, I've created a design doc that touches on how we'll be opting for maintenance windows in place of this, and reasons this alternative shouldn't be done.
More info on that here. https://github.com/aws/karpenter-core/pull/207/files. Feel free to leave some comments and give your feedback.


FormTagHelper asp-all-route-data does not accept RouteValueDictionary

Exception:
Cannot implicitly convert type 'Microsoft.AspNetCore.Routing.RouteValueDictionary' to 'System.Collections.Generic.IDictionary<string, string>'

RouteValueDictionary implements IDictionary<string, object>, but not IDictionary<string, string>, so that's what causes this error.
I believe the tag helper that processes the asp-all-route-data attribute specifically uses string/string so that the most common cases are supported.
If you want to pass in a RouteValueDictionary, you'll need to convert it to a string/string dictionary, which you could do by writing a custom extension method (or other method) that does the conversion.
E.g. something like:
    public static class MyExtensions
    {
        public static IDictionary<string, string> ToStringDictionary(this RouteValueDictionary rvd)
        {
            return rvd.ToDictionary(kvp => kvp.Key, kvp => Convert.ToString(kvp.Value, CultureInfo.InvariantCulture));
        }
    }

Then where you need to use it, do this:
Click me!

@Eilon Could the asp-all-route-data tag be enhanced so that the default behaviour, when used without a value, is to provide all of the route date? I.e.:
Click me!

would be equivalent to:
Click me!


Incorrect dicom reading

Hello! I found dicom that is not loading properly, though some viewers (Weasis) open it correctly.  I load dicom using:
image = sitk.ReadImage(file_path)
image = sitk.GetArrayFromImage(image)[0]

The values of the image seems to be overflown, they lie in range 0...4095. Max value in viewer ~48k.
Some tags:
(0018, 1164) Imager Pixel Spacing                DS: [0.139, 0.139]
(0028, 0002) Samples per Pixel                   US: 1
(0028, 0004) Photometric Interpretation          CS: 'MONOCHROME1'
(0028, 0100) Bits Allocated                      US: 16
(0028, 0101) Bits Stored                         US: 16
(0028, 0102) High Bit                            US: 15
(0028, 0103) Pixel Representation                US: 0
(0028, 1050) Window Center                       DS: '22429.0'
(0028, 1051) Window Width                        DS: '43319.0'
(0028, 1052) Rescale Intercept                   DS: '0.0'
(0028, 1053) Rescale Slope                       DS: '1.0'
Operating system: Ubuntu 18.04.6
Python 3.10
SimpleITK version: 2.2.0 installed through pip
dicom: invalid_reading.zip

@lrunaways as SimpleITK uses ITK for reading, you should probably upstream this report to https://github.com/InsightSoftwareConsortium/ITK

Hello @lrunaways,
Please follow @gdevenyi guidance. Feel free to refer back to this issue or share the information provided below.
From a cursory review, this looks to be an issue possibly in an even deeper layer, GDCM which is used by ITK. It is caused by the conversion from MONOCHROME1 to MONOCHROME2 which ITK does via GDCM which I suspect is causing an overflow (haven't confirmed).
If you read the image via pydicom, which doesn't do the conversion, the original intensity values are retained.
import pydicom
import SimpleITK as sitk

ds = pydicom.dcmread("invalid_reading.dcm")
image = sitk.GetImageFromArray(ds.pixel_array)
sitk.Show(image)


Update brew command to use --cask

This change updates the Homebrew installation command in the README, per the new command format in Homebrew.
Running brew cask install hiddenbar in Homebrew 2.7.1 shows the following error:
Error: Calling brew cask install is disabled! Use brew install [--cask] instead.

This PR changes the command to brew install --cask hiddenbar which wotks.

Dupe of https://github.com/dwarvesf/hidden/pull/105


build: add basic devcontainer.json (for Rust, JavaScript, Python)

Adresses #1808

I tried this — it works pretty well! Because it's a devcontainer, I lose shell customizations, but if I were first arriving at a project, then it would be nice.
It does take a long time to run task install-cargo-tools, and yes I got an error too (though on restarting VSCode it went away). Given that the image is partly pre-built, is there any way of building those in CI, maybe as a Docker image, or as one of the "features"? They already build into our standard Docker image in CI.

Yeah, the installation takes an incredibly long time, I think it took over 30 minutes when I tried it on 2 core Codespaces......

It does take a long time to run task install-cargo-tools, and yes I got an error too (though on restarting VSCode it went away). Given that the image is partly pre-built, is there any way of building those in CI, maybe as a Docker image, or as one of the "features"? They already build into our standard Docker image in CI.

It is possible, but wouldn't it still take a long time to build in the end?
In the end, most of the time is spent compiling the Rust packages, so it will take the same amount of time no matter which layer we install as long as you install these packages.
I am wondering if it is possible to define another task that can install only minimal dependencies in a short time.

It is possible, but wouldn't it still take a long time to build in the end?
In the end, most of the time is spent compiling the Rust packages, so it will take the same amount of time no matter which layer we install as long as you install these packages.

I was thinking it might be possible to compile them in CI and then publish them.
For example, with Docker, we could compile these tools in a docker build and docker push them to a repository. Then when someone docker pulls the image, they have the tools immediately, there's no need for them to re-compile them.
Is that possible with this model too? For example, a "ghcr.io/prql/devcontainer-features/cargo-tools:1"?

From a quick browse, it seems like that's not the focus of devcontainers: https://code.visualstudio.com/blogs/2022/09/15/dev-container-features and https://containers.dev/implementors/spec/. We could make our own base image, but maybe that's more maintenance.
So maybe this isn't viable.
Another alternative is cargo binstall, which would make it much faster — one compilation step, and then mostly just downloading binaries. We could move the main task onto this.
(or any other ideas are very welcome)

I am wondering if it is possible to define another task that can install only minimal dependencies in a short time.

If we really need to, we could do this. I would say that tools like this tend to decay, and there's a potentially bad cycle of them falling out of date -> people using them less -> them not being maintained -> falling further out of date.
So one list which is tested is much better than two lists. But if we need to, that's totally fine with me.

Thanks for your response.

From a quick browse, it seems like pre-building is not the focus of devcontainers: code.visualstudio.com/blogs/2022/09/15/dev-container-features and containers.dev/implementors/spec — instead the just package the .sh scripts. We could make our own base image, but maybe that's more maintenance.

It is possible to pre-build using the Dev Container CLI.
In fact, the image mcr.microsoft.com/devcontainers/rust:0-1-bullseye used here as a base image was built that way.
https://github.com/devcontainers/images/blob/b361b069291b78b97396eda62ed6d6f62266a36f/src/rust/README.md
However, the problem here is the packages installed by cargo anyway, so I think we can greatly reduce the build time for each user by simply pushing an image built from a simple Dockerfile like the following to GHCR and using that as the base image.
FROM mcr.microsoft.com/devcontainers/rust:0-1-bullseye
USER vscode
RUN cargo install cargo install cargo-audit cargo-insta cargo-release \
        default-target mdbook mdbook-admonish mdbook-toc wasm-bindgen-cli \
        wasm-pack
USER root

The cons of this method are that it takes probably 3 hours to do a Docker Build with a multi-platform build of amd64 and arm64, and we need to periodically push the built image to the Registry. (Perhaps it would make sense to use Dependabot to update the Dockerfile once a month or so when the base image is updated and trigger the build.)
Nevertheless, I would give this a try as it will definitely improve the development experience for users.

If we really need to, we could do this. I would say that tools like this tend to decay, and there's a potentially bad cycle of them falling out of date -> people using them less -> them not being maintained -> falling further out of date.
So one list which is tested is much better than two lists. But if we need to, that's totally fine with me.

My concern was installing all the packages, including the need to install mdbook even when doing work unrelated to the book.
I think sometimes it is better to install the necessary tools as needed for the subprojects that need them.
Of course, we can support the method of defining higher-level tasks and installing them all at once.

Thanks for the response @eitsupi

However, the problem here is the packages installed by cargo anyway, so I think we can greatly reduce the build time for each user by simply pushing an image built from a simple Dockerfile like the following to GHCR and using that as the base image.

Yes, that could be great, that's somewhat what I was thinking re pre-building

The cons of this method are that it takes probably 3 hours to do a Docker Build with a multi-platform build of amd64 and arm64, and we need to periodically push the built image to the Registry. (Perhaps it would make sense to use Dependabot to update the Dockerfile once a month or so when the base image is updated and trigger the build.)

I think this is fine — we can build in CI and push automatically. It's very fast when it builds from cache; e.g. https://github.com/PRQL/prql/actions/runs/4226792538/jobs/7340613823, even though I imagine it does take a long time to build from scratch. But worst case we can just build on every release. I think we can do this without any marginal manual work on each image update.
Seeing whether dependabot can update the base image would be great too.
(I'm not sure whether we can use the same image for this and our existing docker setup — maybe we can only use the mcr.microsoft.com/devcontainers/rust base and replace the current base?)

My concern was installing all the packages, including the need to install mdbook even when doing work unrelated to the book.
I think sometimes it is better to install the necessary tools as needed for the subprojects that need them.

I agree with the point, but it's also work to maintain the lists (and not easy to automatically test them), so it's a tradeoff. And assuming we can build first, then it's very low cost to have all the tools — something like 2MB per binary.
That said, if folks feel strongly and are OK maintaining them, we can split up the lists.

we can build in CI and push automatically. It's very fast when it builds from cache

I don't know how best to use cache here......perhaps this feature?
I have never used it and will have to give it a try.
https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache

I don't know how best to use cache here......perhaps this feature?
I have never used it and will have to give it a try.
https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache

We already do this! https://github.com/PRQL/prql/blob/68752b7806cee40af10b22d44609230ed2a78dc1/.github/workflows/nightly.yaml#L92-L93
So I think the only things we need to change to get the Docker part of this working:

Change the base of our existing image
Push to an image repository in CI

I see no particular reason why it cannot be replaced.

I agree

We already do this!
https://github.com/PRQL/prql/blob/68752b7806cee40af10b22d44609230ed2a78dc1/.github/workflows/nightly.yaml#L92-L93

My understanding is that this cache is disabled when the base image (base layer) is changed.
I don't think it's a big deal if the cache doesn't work, so anyway I'll prototype and push it to this branch.......

I don't know how best to use cache here......perhaps this feature?
I have never used it and will have to give it a try.
<EMAIL_ADDRESS>
Note: It does not seem possible to run this on CI and share the cache between images now. docker/build-push-action#716

If my understanding is correct here: https://github.com/PRQL/prql/pull/1893/files#r1117593795, then I think we have two options:
Build docker image to hold everything

We build a docker image based on the devcontainer using this devcontainer.json in CI: https://github.com/PRQL/prql/pull/1893/files#diff-c28c246b0792371d4cb69440d78cc47b4876ea34eb98618d7daafa41f96ce10bR3. We use the devcontainer GitHub Action
We push that to the image repository
Then we supply a different devcontainer.json which is similar to that file, but the image value is a reference to the image we've built & pushed

That way, folks get a devcontainer with everything already built onto the base image. But we need two devcontainer.json files.
Build base docker image only

Build a docker image which contains the cargo tools with a normal docker workflow. We put these in an image since they take a long time to build (and we could add anything else there that was expensive, but need to express them as Dockerfile commands, not as references to devcontainer features).
Reference that image in a single devcontainer.json

This would be simpler, and IIUC is what you're suggesting in https://github.com/PRQL/prql/pull/1893#discussion_r1117103494. It requires a small amount of building on the client side — the devcontainer features that aren't built.
We then wouldn't use the devcontainer GitHub Action. Is that correct?

Both seem reasonable, the second seems simpler and easier to get something working; and could grow into the second in time if needed.
I think we could replace our existing docker build with this either way: https://github.com/PRQL/prql/blob/main/Dockerfile.

Thanks for keeping pushing on this. Feel free to proceed as you think best!

All good points. I wrote this up as an on-boarding document for newcomers to PRQL and especially those who are also new to Dev Containers. The hints here are a collection of the stuff I had to learn while casting about to get it going.
Most importantly, I feel the need to describe "what a Dev Container is good for" (working on the Book, Playground, Website, not so much improving the compiler) so that newcomers can see the goal and decide whether the startup hassle is worth it.
That said, please edit away! Thanks.

@max-sixty I have changed to build images to be pushed to GHCR from pure Dockerfile.
I would like to improve the documentation and the Taskfile after this (split only the part copied in the Dockerfile into a separate Taskfile to minimize cache invalidation due to Taskfile updates), but can we merge first to see if it works in its current state?

The base image is now available by #2028
https://github.com/PRQL/prql/pkgs/container/prql-devcontainer-base
Try using this and revise the documentation.

We tried this with Codespace, which has the lowest specs, and had the following problems:

It took a few dozen minutes to build by rustanalyzer. It seemed to build all workspace members, so perhaps you may need to add a setting to limit them.
rusanalyzer failed to build duckdb. It may be a lack of machine specs.
I could not see the preview when I ran mdbook serve. I do not know what the problem is.

I think we can merge this for now and deploy to the web branch as well.
Any further documentation improvements would include a wider range of issues ( Taskfile improvements, workflow improvements using Docker, etc.)

FYI I tried testing it, but I can't seem force it to use the amd64 image. I tried adding   "runArgs": ["--platform=linux/amd64"] to the devcontainer.json but it doesn't supply this to docker pull. When I pull it manually prior, I then get this error — looks like it's still relying on the platform for part of the process:

[33 ms] Dev Containers 0.282.0 in VS Code 1.76.0 (92da9481c0904c6adfe372c12da3b7748d74bdcb).
[32 ms] Start: Resolving Remote
[42 ms] Setting up container for folder or workspace: /Users/maximilian/workspace/prql
[43 ms] Start: Check Docker is running
[43 ms] Start: Run: docker version --format {{.Server.APIVersion}}
[222 ms] Server API version: 1.41
[222 ms] Start: Run: docker volume ls -q
[321 ms] Start: Run: docker ps -q -a --filter label=vsch.local.folder=/Users/maximilian/workspace/prql --filter label=vsch.quality=stable
[460 ms] Start: Run: docker ps -q -a --filter label=devcontainer.local_folder=/Users/maximilian/workspace/prql --filter label=devcontainer.config_file=/Users/maximilian/workspace/prql/.devcontainer/devcontainer.json
[544 ms] Start: Run: docker ps -q -a --filter label=devcontainer.local_folder=/Users/maximilian/workspace/prql
[667 ms] Start: Run: docker ps -q -a --filter label=devcontainer.local_folder=/Users/maximilian/workspace/prql
[785 ms] Start: Run: /Applications/Visual Studio Code.app/Contents/Frameworks/Code Helper (Plugin).app/Contents/MacOS/Code Helper (Plugin) --ms-enable-electron-run-as-node /Users/maximilian/.vscode/extensions/ms-vscode-remote.remote-containers-0.282.0/dist/spec-node/devContainersSpecCLI.js up --user-data-folder /Users/maximilian/Library/Application Support/Code/User/globalStorage/ms-vscode-remote.remote-containers/data --container-session-data-folder /tmp/devcontainers-ed28eb90-9dda-4332-ac16-7f985020db651678145467696 --workspace-folder /Users/maximilian/workspace/prql --workspace-mount-consistency cached --id-label devcontainer.local_folder=/Users/maximilian/workspace/prql --id-label devcontainer.config_file=/Users/maximilian/workspace/prql/.devcontainer/devcontainer.json --log-level debug --log-format json --config /Users/maximilian/workspace/prql/.devcontainer/devcontainer.json --default-user-env-probe loginInteractiveShell --mount type=volume,source=vscode,target=/vscode,external=true --skip-post-create --update-remote-user-uid-default on --mount-workspace-git-root true
[965 ms] (node:57320) [DEP0005] DeprecationWarning: Buffer() is deprecated due to security and usability issues. Please use the Buffer.alloc(), Buffer.allocUnsafe(), or Buffer.from() methods instead.
[965 ms] (Use Code Helper (Plugin) --trace-deprecation ... to show where the warning was created)
[965 ms] @devcontainers/cli 0.31.0. Node.js v16.14.2. darwin 22.3.0 arm64.
[965 ms] Start: Run: docker buildx version
[1242 ms] github.com/docker/buildx v0.10.0<PHONE_NUMBER>12d36679153c3414f7689626251501
[1242 ms]
[1242 ms] Start: Resolving Remote
[1257 ms] Start: Run: git rev-parse --show-cdup
[1277 ms] Start: Run: docker ps -q -a --filter label=devcontainer.local_folder=/Users/maximilian/workspace/prql --filter label=devcontainer.config_file=/Users/maximilian/workspace/prql/.devcontainer/devcontainer.json
[1383 ms] Start: Run: docker inspect --type image ghcr.io/prql/prql-devcontainer-base:latest
[1584 ms] local container features stored at: /Users/maximilian/.vscode/extensions/ms-vscode-remote.remote-containers-0.282.0/dist/node_modules/vscode-dev-containers/container-features
[1587 ms] Start: Run: tar --no-same-owner -x -f -
[1607 ms] * Processing feature: ghcr.io/devcontainers/features/hugo:1
[2422 ms] * Processing feature: ghcr.io/devcontainers/features/python:1
[2806 ms] * Processing feature: ghcr.io/devcontainers/features/node:1
[3207 ms] * Processing feature: ghcr.io/eitsupi/devcontainer-features/go-task:1
[3638 ms] * Fetching feature: hugo_1_oci
[4005 ms] * Fetching feature: python_2_oci
[4400 ms] * Fetching feature: node_3_oci
[4770 ms] * Fetching feature: go-task_4_oci
[5105 ms] Start: Run: docker buildx build --load --build-context dev_containers_feature_content_source=/var/folders/vk/btr3xc6x6f1_2cbv4h0xzp500000gn/T/devcontainercli/container-features/0.31.0-1678145469826 --build-arg _DEV_CONTAINERS_BASE_IMAGE=ghcr.io/prql/prql-devcontainer-base:latest --build-arg _DEV_CONTAINERS_IMAGE_USER=root --build-arg _DEV_CONTAINERS_FEATURE_CONTENT_SOURCE=dev_container_feature_content_temp --target dev_containers_target_stage -t vsc-prql-793278263691d95561acba5161b934dc-features -f /var/folders/vk/btr3xc6x6f1_2cbv4h0xzp500000gn/T/devcontainercli/container-features/0.31.0-1678145469826/Dockerfile.extended /Users/maximilian/Library/Application Support/Code/User/globalStorage/ms-vscode-remote.remote-containers/data/empty-folder
[+] Building 15.5s (8/8) FINISHED
=> [internal] booting buildkit                                            9.2s
=> => pulling image moby/buildkit:buildx-stable-1                         6.5s
=> => creating container buildx_buildkit_sharp_raman0                     2.7s
=> [internal] load build definition from Dockerfile.extended              0.0s
=> => transferring dockerfile: 4.77kB                                     0.0s
=> [internal] load .dockerignore                                          0.0s
=> => transferring context: 2B                                            0.0s
=> resolve image config for docker.io/docker/dockerfile:1.4               2.7s
=> [auth] docker/dockerfile:pull token for registry-1.docker.io           0.0s
=> CACHED docker-image://docker.io/docker/dockerfile:1.4@sha256:9ba7531b  0.0s
=> => resolve docker.io/docker/dockerfile:1.4@sha256:9ba7531bd80fb0a8586  0.0s
=> ERROR [internal] load metadata for ghcr.io/prql/prql-devcontainer-bas  2.3s
=> [context dev_containers_feature_content_source] load .dockerignore     0.0s
=> => transferring dev_containers_feature_content_source: 2B              0.0s

[internal] load metadata for ghcr.io/prql/prql-devcontainer-base:latest:

Dockerfile.extended:13
11 |     RUN chmod -R 0700 /tmp/build-features
12 |
13 | >>> FROM $_DEV_CONTAINERS_BASE_IMAGE AS dev_containers_target_stage
14 |
15 |     USER root
ERROR: failed to solve: no match for platform in manifest sha256:57593976654df7c02f8e8bd06d08f621cbc0196a6a8d19d41f5f15c5fc61ab31: not found
[20850 ms] Error: Command failed: docker buildx build --load --build-context dev_containers_feature_content_source=/var/folders/vk/btr3xc6x6f1_2cbv4h0xzp500000gn/T/devcontainercli/container-features/0.31.0-1678145469826 --build-arg _DEV_CONTAINERS_BASE_IMAGE=ghcr.io/prql/prql-devcontainer-base:latest --build-arg _DEV_CONTAINERS_IMAGE_USER=root --build-arg _DEV_CONTAINERS_FEATURE_CONTENT_SOURCE=dev_container_feature_content_temp --target dev_containers_target_stage -t vsc-prql-793278263691d95561acba5161b934dc-features -f /var/folders/vk/btr3xc6x6f1_2cbv4h0xzp500000gn/T/devcontainercli/container-features/0.31.0-1678145469826/Dockerfile.extended /Users/maximilian/Library/Application Support/Code/User/globalStorage/ms-vscode-remote.remote-containers/data/empty-folder
[20850 ms]     at cie (/Users/maximilian/.vscode/extensions/ms-vscode-remote.remote-containers-0.282.0/dist/spec-node/devContainersSpecCLI.js:1930:3293)
[20851 ms]     at S7 (/Users/maximilian/.vscode/extensions/ms-vscode-remote.remote-containers-0.282.0/dist/spec-node/devContainersSpecCLI.js:1930:3229)
[20851 ms]     at process.processTicksAndRejections (node:internal/process/task_queues:96:5)
[20851 ms]     at async xie (/Users/maximilian/.vscode/extensions/ms-vscode-remote.remote-containers-0.282.0/dist/spec-node/devContainersSpecCLI.js:1945:2625)
[20851 ms]     at async Wf (/Users/maximilian/.vscode/extensions/ms-vscode-remote.remote-containers-0.282.0/dist/spec-node/devContainersSpecCLI.js:1945:3750)
[20851 ms]     at async noe (/Users/maximilian/.vscode/extensions/ms-vscode-remote.remote-containers-0.282.0/dist/spec-node/devContainersSpecCLI.js:2069:17325)
[20851 ms]     at async roe (/Users/maximilian/.vscode/extensions/ms-vscode-remote.remote-containers-0.282.0/dist/spec-node/devContainersSpecCLI.js:2069:17066)
[20853 ms] Exit code 1
[20855 ms] Command failed: /Applications/Visual Studio Code.app/Contents/Frameworks/Code Helper (Plugin).app/Contents/MacOS/Code Helper (Plugin) --ms-enable-electron-run-as-node /Users/maximilian/.vscode/extensions/ms-vscode-remote.remote-containers-0.282.0/dist/spec-node/devContainersSpecCLI.js up --user-data-folder /Users/maximilian/Library/Application Support/Code/User/globalStorage/ms-vscode-remote.remote-containers/data --container-session-data-folder /tmp/devcontainers-ed28eb90-9dda-4332-ac16-7f985020db651678145467696 --workspace-folder /Users/maximilian/workspace/prql --workspace-mount-consistency cached --id-label devcontainer.local_folder=/Users/maximilian/workspace/prql --id-label devcontainer.config_file=/Users/maximilian/workspace/prql/.devcontainer/devcontainer.json --log-level debug --log-format json --config /Users/maximilian/workspace/prql/.devcontainer/devcontainer.json --default-user-env-probe loginInteractiveShell --mount type=volume,source=vscode,target=/vscode,external=true --skip-post-create --update-remote-user-uid-default on --mount-workspace-git-root true
[20856 ms] Exit code 1

One concern is that it's a) VSCode encourages you to use a Devcontainer with a pop-up and b) it's a very confusing error message, which suggests that the image doesn't exist, which makes the library look poorly maintained.
Is there any way of making that a more friendly failure? Can we disable the pop-up? Or run something before attempting to launch that checks for the arch and gives a better message?
I'm surprised this isn't more of an issue with devcontainers — is it just us? Or maybe they're not that widely adopted yet?

It took a few dozen minutes to build by rustanalyzer. It seemed to build all workspace members, so perhaps you may need to add a setting to limit them.

rusanalyzer failed to build duckdb (libduckdb-sys). It may be a lack of machine specs.

Yes, I think this just it running cargo build which does take a lot of resources, and will break on insufficient RAM because of DuckDB. I'm not sure there's much we can do there — maybe checking RAM and warning? But that seems difficult to maintain...

@max-sixty Thanks for testing.
Unfortunately I don't use macOS so I don't know about that.
I have looked and could not even find a similar issue. Could you please report about it here?
https://github.com/microsoft/vscode-remote-release
The only solution I have found that may be the only solution is to set the environment variable DOCKER_DEFAULT_PLATFORM.
https://docs.docker.com/engine/reference/commandline/cli/#environment-variables

I can report it, though my guess is that the solution is "provide an image with the correct platform".
How close do you think providing both architectures is? Is this something that's around the corner or months away? If it's the former then maybe we prioritize that?

How close do you think providing both architectures is? Is this something that's around the corner or months away? If it's the former then maybe we prioritize that?

I do not see this as a problem that can be solved quickly.
DOCKER_DEFAULT_PLATFORM did not work?

https://medium.com/@tonistiigi/faster-multi-platform-builds-dockerfile-cross-compilation-guide-part-1-ec087c719eaf
I tried to cross-compile (e.g. cargo install --target aarch64-unknown-linux-gnu mdbook) in an attempt to perform the steps in this article without success.
Do you know how to cross-compile with cargo install?
If cross-compilation cannot be performed, building images for arm64 does not seem realistic until GHA's arm64 runner comes along.

DOCKER_DEFAULT_PLATFORM did not work?

It does work!
Though the selling point of this was that it was a super-easy setup "If you just want to contribute in 20 minutes, here's a setup that will get you productive in 2 minutes"; and that's less the case if it requires setting env vars before launching. It also is much slower because of the emulation.
But it's OK — there's still a case for folks who want everything packaged, and setting the env var is the one thing they need to do.

When I launch, I'm getting an error /usr/local/cargo/bin/cargo: No such file or directory — is that just me? This means that rust-analyzer returns [ERROR rust_analyzer::lsp_utils] rust-analyzer failed to load workspace: "cargo" "--version" failed, exit status: 127.
Can you see better output here?

It's not that fast to start, one of the features takes 207 seconds. It's unfortunate that it's not easy to cache this. (I don't think we should try and do this ourselves; possibly VSCode will allow better caching in future..., rather than us having two devcontainer.json files.)
 => [dev_containers_target_stage 2/7] COPY --from=dev_containers_feature_  0.0s
 => [dev_containers_target_stage 3/7] RUN echo "_CONTAINER_USER_HOME=$(ge  0.2s
 => [dev_containers_target_stage 4/7] RUN --mount=type=bind,from=dev_cont  4.2s
 => [dev_containers_target_stage 5/7] RUN --mount=type=bind,from=dev_con  47.0s
 => [dev_containers_target_stage 6/7] RUN --mount=type=bind,from=dev_co  207.9s
 => [dev_containers_target_stage 7/7] RUN --mount=type=bind,from=dev_cont  2.4s

Re the architectures — my understanding is that enabling QEMU would allow for emulation, which will be slow, but will work. (and that's fine, there's no urgency for the building...) — https://blog.thesparktree.com/docker-multi-arch-github-actions
...but presumably this relies on VSCode having arm64 images available. Am I correct in thinking they don't have them available?

Thanks for trying this!

Though the selling point of this was that it was a super-easy setup "If you just want to contribute in 20 minutes, here's a setup that will get you productive in 2 minutes"; and that's less the case if it requires setting env vars before launching. It also is much slower because of the emulation.

Unfortunately, Docker (Linux containers) run on Linux, so I think running them on macOS is generally regarded as not a very good experience (due to file system differences, etc.)
Also, since it only supports amd64, running it on an arm64 mac will further degrade performance.
In short, the main target users here are amd64 Linux (including WSL2) users.

my understanding is that enabling QEMU would allow for emulation, which will be slow, but will work.

docker buildx build already uses QEMU.
https://github.com/PRQL/prql/blob/f04baf22a69e02bfa858f23b77926ad702d56ee8/.github/workflows/devcontainer.yaml#L36-L37
The problem here is that cargo install fails under too high a load for emulation by QEMU.

When I launch, I'm getting an error /usr/local/cargo/bin/cargo: No such file or directory — is that just me? This means that rust-analyzer returns [ERROR rust_analyzer::lsp_utils] rust-analyzer failed to load workspace: "cargo" "--version" failed, exit status: 127.
Can you see better output here?

I haven't seen any such error.
Maybe it's an emulation problem...

OK great, thanks for the clear answers.
So maybe we just say it's only supported for amd64... That's a lot of WSL2 users.

Perhaps another devcontainer.json could be added to build from the Dockerfile locally to support arm64 machines.
(VS Code seems to have recently added support for multiple devcontainer.json)
I can do that in another PR in the next time.

Thank you for taking the time to review this.

Thank you very much @eitsupi !
Maybe we could also add to the Changelog so folks learn about it?

Maybe we could also add to the Changelog so folks learn about it?

Thanks, I forgot about that. I will update Changelog too with a follow-up PR.

Perhaps another devcontainer.json could be added to build from the Dockerfile locally to support arm64 machines.

I think possibly we could see how people find the amd64 version, and we could see whether MS make it easier to build arm64 images — my guess is that it'll become much easier...

I believe the GHA arm64 runner preview was set for the 2023Q3 milestone.
Conversely, we will not be able to provide pre-built images for arm64 for at least half a year.

@eitsupi FYI I swapped the labels

@eitsupi FYI I swapped the labels

Thank you for correcting the label. (But seems to have failed to deploy? No problem to carry over until next release)

(But seems to have failed to deploy? No problem to carry over until next release)

Ah, sorry, I got the label wrong too! We should have done backport.
Yeah it'll go out later anyway!


No toolbar at CropImageActivity

Toolbar is not showing when I open CropActivity. Am I missing anything?

It's because of your theme style, if you are using NoActionBar
Create a new Style in your Style.xml
<style name="NewTheme" parent="Theme.AppCompat.Light.DarkActionBar">         <!-- Customize your theme here. -->         <item name="colorPrimary">@color/colorPrimary</item>         <item name="colorPrimaryDark">@color/colorPrimaryDark</item>         <item name="colorAccent">@color/colorAccent</item>     </style>
And apply it to the activity in manifest.
<activity android:name="com.theartofdev.edmodo.cropper.CropImageActivity"                 android:theme="@style/NewTheme"/>

Yeah I am using NoActionBar. With this new theme is working fine. Thanks 👍


simplify scripts

Description
Moving logic for tests in protocols.py
Status

[x] Ready to go

Welcome to Codecov :tada:
Once merged to your default branch, Codecov will compare your coverage reports and display the results in this comment.
Thanks for integrating Codecov - We've got you covered :open_umbrella:


Code Inspector: Fix background processing

This commit enables the code inspector for background processing.
Before this commit is applied code inspection in background resulted in lock errors.

hmm, think I've used the old approach before, without lock errors
what kind of lock errors occured?

Also running in a background job? I get this error

created here

You can reproduce it with this code
PARAMETERS: package TYPE devclass.

TRY.
    DATA(li_syntax_check) = zcl_abapgit_factory=>get_syntax_check( package ).

    DATA(lt_list) = li_syntax_check->run( ).

    cl_demo_output=>display( lt_list ).

  CATCH zcx_abapgit_exception INTO DATA(lx_error).
    MESSAGE lx_error TYPE 'S' DISPLAY LIKE 'E'.
ENDTRY.


Rewrote LinearInterp from scratch

LinearInterp was previously mostly an extension of
scipy.interpolate.UnivariateSpline.  It has now been rewritten as a
fully custom class for HARK.  This was done because UnivariateSpline
uses 6-7x more memory than is necessary.  This is not a problem in
isolation, but can lead to memory constraints being binding in very
large models-- those with tens of thousands of LinearInterps.
All code runs identically as before these changes.  Old version remains
as LinearInterpOLD, for testing.  Should be deleted.  Some applications
run 5% faster, others are up to 14% slower than before.  We can fix
speed later by adding an option to the class that makes it use 50% more
memory.
Also fixed all classes where distance_criteria was not a static
attribute.

Please let me know what you think of my suggested changes, Matt.  Most obviously, there's now just one method _evalOrDer that does anything.  I've also dropped the _isscalar test for i, and now just have the line i      = np.maximum(np.searchsorted(self.x_list[:-1],x),1), because np.maximum returns a float if x is a float.  I really think this change works, but it's possible I broke something and missed it.  The code also no longer initializes y or dydx as numpy arrays of type float.  I believe by changing 1 to 1. in the evaluation of y, the y array will always be a float now anyway.  There is no such guarantee for dydx, I guess, so if we really want to make sure it is an array of floats, some code might need to be added there.
I'd especially be grateful if you could test this with other parts of HARK you understand better than I do, e.g. maybe cstwMPC, to make sure it doesn't break anything there.  This push is not necessarily supposed to be accepted as-is, but just as a suggestion and perhaps a starting point for something else.  [Though if it works well, great!]

The changes you just made are compatible with the rest of HARK.  Methods like _eval should never be accessed externally, only through HARKinterpolator1D.call (etc).  I support this code reduction, and we should implement it for the other classes.

Awesome!  Sorry again this took me so long


use key value in 2pc stage

In 2PC stage，TiSpark's write can only focus on writing not checking. So we can just use key value pair to create a RDD.

/run-all-tests

/run-all-tests


duplicate node in match?

from max_matching import Match
unique_edges = [(0, 1), (0, 14), (1, 2), (2, 15), (2, 3), (3, 4), (4, 27), (5, 6), (5, 26), (6, 7), (7, 25), (7, 8), (8, 9), (9, 24), (9, 10), (10, 11), (12, 13), (12, 16), (13, 14), (14, 15), (16, 17), (18, 19), (18, 21), (19, 20), (21, 22), (22, 23), (24, 25), (26, 27)]
edges = unique_edges + [(j,i) for i,j in unique_edges]
match = Match.from_edges(28, edges)
match.maximum_matching()
for n in match.nodes:
if n.mate is not None:
i = n.index
j = n.mate.index
if i < j:
print(i,j)
9 is duplicated
#0 14
#2 3
#4 27
#5 26
#6 7
#8 9
#9 24
#10 11
#12 13
#16 17
#18 21
#19 20
#22 23

Hi, the bug is fixed. Thank you for reporting this.
Outcome:
0 1
2 3
4 27
5 26
6 7
8 9
10 11
12 13
14 15
16 17
18 21
19 20
22 23
24 25


[bug]: Effect package version mismatch?

Provide environment information
System:
    OS: Windows 10 10.0.19045
    CPU: (12) x64 AMD Ryzen 5 3600 6-Core Processor
    Memory: 14.20 GB / 31.95 GB
  Binaries:
    Node: 20.11.1 - C:\Program Files\nodejs\node.EXE
    npm: 10.2.4 - C:\Program Files\nodejs\npm.CMD
  Browsers:
    Edge: Chromium (126.0.2592.68)
    Internet Explorer: 11.0.19041.4355
  npmPackages:
    @uploadthing/react: ^6.7.0 => 6.7.0
    typescript: ^5.4.2 => 5.5.2
    uploadthing: ^6.13.0 => 6.13.0

Describe the bug
On attempting to upload image via the uploadthing button:
IN BROWSER:
POST http://localhost:3000/api/uploadthing?actionType=upload&slug=imageUploader 500 (Internal Server Error)
VSCODE TERMINAL:
 ⨯ (FiberFailure) RuntimeException: Cannot execute an Effect versioned 3.4.4 with a Runtime of version 3.4.2     at C:\Users\XXX\Desktop\XXX\react\teethreegeegee\.next\server\chunks\633b9_effect_dist_esm_internal_core_cdae7d.js:616:241  POST /api/uploadthing?actionType=upload&slug=imageUploader 500 in 98ms
As far as I can tell, the issue seems to be: node_modules/effect being installed at v3.4.4 and @effect/schema following through by having peerDependencies effect 3.4.4 while uploadthing still wants 3.4.2? Looking at your minimal template it's the same thing there.
Link to reproduction
https://stackblitz.com/github/pingdotgg/uploadthing/tree/main/examples/minimal-appdir?file=package-lock.json
To reproduce
Create the usual t3 app, install uploadthing and set up normally, try to use file upload feature, get mismatching effect version error.
As for repro link, ctrl+f 3.4.4 & 3.4.2 to see what I think is the mismatch
Additional information
Haven't had to deal with such an issue yet, so I don't know if my idea is right or if it's just a skill issue on my part and it's just a coincidence.
👨👧👦 Contributing

[ ] 🙋♂ Yes, I'd be down to file a PR fixing this bug!

Code of Conduct

[X] I agree to follow this project's Code of Conduct

After 3 hours of solid sleep due to noisy neighbours I went into the project and manually downgraded @effect/schema to 0.68.9 which uses effect 3.4.2 - this fixed the problem. So as far as I can tell the issue is that the uploadthing package can install mismatching versions of effect due to @effect/schema?

Had this same error after installing the newest update a couple hours ago... Will try manually downgrading the library aswell

That fixed it for me aswell. Maybe someone upgraded the package version and didnt find this braking change -> should be reverted asap and seams like an easy fix
npm install<EMAIL_ADDRESS>npm install<EMAIL_ADDRESS>

That fixed it for me aswell. Maybe someone upgraded the package version and didnt find this braking change -> should be reverted asap and seams like an easy fix
npm install<EMAIL_ADDRESS>npm install<EMAIL_ADDRESS>

Worked for me as well 👍


Remove the "Impling" part of the minimap/in world impling overlay.

Currently, you see "Baby impling", "Essence impling".
I'd like it if it changed to just "Baby", and "Essence". This would make it have less text, and remove redundancy.

Is it just for the Baby and Essence, or for all of them?
Could you provide a screenshot as well for it?
I believe I understand the issue, but, would like to clarify.

It's for all of them.
https://user-images.githubusercontent.com/2388657/39968766-6f4544b2-56a0-11e8-8e87-9cdf793e9e95.png

We're highlighting implings the same way as the NPC Indicators plugin would, which includes writing their full NPC name on the minimap, like any other highlighted NPC. In places where this causes clutter (probably just Puro-Puro?), I recommend adjusting the plugin's settings to highlight only the tiers of imps you're focusing on catching.


[REVIEW] Update conda env files for cudf 0.16.0

cudf 0.16.0 no longer supports CUDA 10.0 and has moved pandas and pyarrow dependencies forward to post-1.0 versions.

@benfred Added a commit to bump versions there too.

rerun tests

rerun tests


[DOCFIX] fix cn doc Alluxio-Storage.md

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/Alluxio-Pull-Request-Builder/19610/
Test PASSed.


Create coefficient from file.

Hi everyone,
I'm currently working on solving the Poisson equation
nabla^2 u = ρ
I am interested in the case where the right-hand side is given by the density of some particles (non-analytical). I have the density ρ stored in a text file with n^3 rows, each row containing the coordinates (x, y, z) and the corresponding density value:
#x y z rho
0 0 0 rho(0,0,0)
.
.
.
L L L rho(L,L,L)

I think I could create a mesh using MakeCartesian3D from 0 to L of n^3 rows and read the array rho using ifstream.
My question is how should I construct the Coefficient from the rho density to assemble the linear form b?
Thank you in advance!

To create a coefficient from your density values, you will need some way of evaluating the density at arbitrary locations in the computational domain. For example, this could be done with polynomial interpolation.
Are you considering your density to be a piecewise linear function with vertex values given from the file?
Do the values in the file necessarily correspond with vertices of your mesh, or could they be different? Are the coordinates always arranged in a Cartesian grid? Do you know the ordering in which the  values appear in the file?

Thanks for your answer,
Yes, we can consider the density a piecewise linear function and interpolate it using linear interpolation.
In this case, the coordinates are initially Cartesian, but if we solve it using MFEM, it doesn't matter if we work in Cartesian.
The ordering is
row = (N*i+j)*N+k

So if your mesh is arbitrary, I would create a custom coefficient that identifies the cell of your coefficient grid, and then manually performs the trilinear interpolation, something like this:
struct TrilinearCoefficient : Coefficient
{
   DenseTensor data; // Store your grid of data points as a dense tensor

   TrilinearCoefficient()
   {
      // Load data from file here, place into data tensor...
   }

   double Eval(ElementTransformation &T, const IntegrationPoint &ip)
   {
      double data[3];
      Vector xvec(data, 3);
      T.Transform(ip, xvec);

      const double h = /* size of coefficient grid*/;

      const int i = xvec[0]/h;
      const int j = xvec[1]/h;
      const int k = xvec[2]/h;

      const double ax = xvec[0]/h - i;
      const double ay = xvec[1]/h - j;
      const double az = xvec[2]/h - k;

      // Perform trilinear interpolation here using
      // data(i [+ 1], j [+ 1], k [ + 1])
      // https://en.wikipedia.org/wiki/Trilinear_interpolation
   }
}

Let me know if it's clear or if more details would be helpful.

Hi again!
Thanks for your detailed answer. If I understood well, I have to create this new Coefficient as
const int mesh_size = 64;  // For a short test I can start with 64^4 meshgrid
const double h = 1.0 /mesh_size // Unitary box;

struct TrilinearCoefficient : Coefficient
{
   DenseTensor data; // Store your grid of data points as a dense tensor

   TrilinearCoefficient()
   {
       // Open the file
       std::ifstream file("density_mesh.dat");

       data.SetSize(mesh_size,mesh_size,mesh_size);

       // Read the mesh and the values from the file
       for (int i = 0; i < mesh_size; ++i)
           for (int j = 0; j < mesh_size; ++j)
               for (int k = 0; k < mesh_size; ++k)
			   {
				   double x, y, z, density;
				   file >> x >> y >> z >> density;
				   data(i, j, k) = density;
			   }

       file.close();
   }

   double Eval(ElementTransformation &T, const IntegrationPoint &ip)
   {
	   double data[3];
      Vector xvec(data,3);
      T.Transform(ip, xvec);

      const int i = xvec[0]/h;
      const int j = xvec[1]/h;
      const int k = xvec[2]/h;

      const double ax = xvec[0]/h - i;
      const double ay = xvec[1]/h - j;
      const double az = xvec[2]/h - k;

      // Perform trilinear interpolation here using
      // data(i [+ 1], j [+ 1], k [ + 1])
      // https://en.wikipedia.org/wiki/Trilinear_interpolation
      const double c000 = data(i, j, k);
      const double c100 = data(i + 1, j, k);
      const double c010 = data(i, j + 1, k);
      const double c001 = data(i, j, k + 1);
      const double c110 = data(i + 1, j + 1, k);
      const double c101 = data(i + 1, j, k + 1);
      const double c011 = data(i, j + 1, k + 1);
      const double c111 = data(i + 1, j + 1, k + 1);

      return (1 - ax) * (1 - ay) * (1 - az) * c000 + ax * (1 - ay) * (1 - az) * c100 + (1 - ax) * ay * (1 - az) * c010 +
             (1 - ax) * (1 - ay) * az * c001 + ax * ay * (1 - az) * c110 + ax * (1 - ay) * az * c101 +
             (1 - ax) * ay * az * c011 + ax * ay * az * c111;
   }
}

If everything is ok, then how should I use this new coefficient?
I modified ex0.cpp to solve Poisson using an RHS (right-hand side) function rho; now, Should I use this new Coefficient, and that's it?
   // 6. Set up the linear form b(.) corresponding to the right-hand side.
   //FunctionCoefficient rhs(rho);
   TrilinearCoefficient rhs;  // New coefficient
   LinearForm b(&fespace);
   b.AddDomainIntegrator(new DomainLFIntegrator(rhs));
   b.Assemble();

Ok, I can see it is working!
Thank you! Up to this point, my question has been answered.
Please let me ask you another question.
I have to work with periodic boundaries, and with this function, it is easy to create a box
Mesh mesh(Mesh::MakeCartesian3D(N,N,N,Element::Type::QUADRILATERAL,L-h,L-h,L-h,false));

I need a periodic box with size L-h, where h = L/N is the spatial step. This is important to define the boundaries; after that, I wanted to use the AMR method.
My question is simple: how can I make this mesh periodic?
In particular, the function
Mesh mfem::Mesh::MakePeriodic(const Mesh & orig_mesh, const std::vector< int > & v2v)

Can do this for me. However, I have to create the v2v vector using this function:
std::vector< int > mfem::Mesh::CreatePeriodicVertexMapping (const std::vector< Vector > & translations, real_t tol = 1e-8) const

Here, I don't know how to create the translation vector.
is it something like this?
std::vector<Vector> translations(3);
translations[0] = Vector(3);
translations[0][0] = L-h;
translations[0][1] = 0.0;
translations[0][2] = 0.0;

translations[1] = Vector(3);
translations[1][0] = 0.0;
translations[1][1] = L-h;
translations[1][2] = 0.0;

translations[2] = Vector(3);
translations[2][0] = 0.0;
translations[2][1] = 0.0;
translations[2][2] = L-h;

Thank you again!

Yes, that's right. The translation vectors represent the translations that line up the vertices that are identified in the periodic mapping.
You can do something like this:
Mesh orig_mesh = Mesh::MakeCartesian3D(N,N,N,Element::Type::QUADRILATERAL,L-h,L-h,L-h,false));
std::vector<Vector> translations =
{
   Vector({L-h, 0.0, 0.0}),
   Vector({0.0, L-h, 0.0}),
   Vector({0.0, 0.0, L-h})
};
Mesh mesh = Mesh::MakePeriodic(orig_mesh, orig_mesh.CreatePeriodicVertexMapping(translations));

Hello again,
I have a question regarding the periodic boundaries.
After some tests, I solved the poison equation using analytical rhs. However, for my periodical density I have some problems. My question is:
Should the mesh.bdr_attributes.Size() be zero after using the MakePeriodic function?
because I got
Boundary attributes in the mesh: 6

here
printf("Boundary attributes in the mesh: %d\n",mesh.bdr_attributes.Size());
 if (mesh.bdr_attributes.Size())
 {
    ess_bdr.SetSize(mesh.bdr_attributes.Max());
    ess_bdr = 0;
    fespace.GetEssentialTrueDofs(ess_bdr, boundary_dofs);
 }

I set the ess_bdr to zero, but the PCG method diverges, and the GMRES gets stuck around some level
  Pass : 50   Iteration : 500  ||B r|| = 3.66721e-07,

However, the solution is a good approximation, although imperfect, compared to the solution using FFTW.
On the other hand, if a set  ess_bdr = 1, the method converges quickly, but the boundary conditions are set to zero (or the initialized value given to x).

PCG should converge, but you need to make sure that your right-hand side is orthogonal to the nullspace (constants). You can do this by subtracting the mean of the right-hand side vector.
I think that once the mesh is made periodic, the boundary elements are retained as "interior boundaries". For your problem, ess_bdr should be zero everywhere.

Thanks, actually I have subtracted the mean of the RHS already.
Maybe the interpolation is failing in making my density periodic.
I tried adding a condition if i, j or k are greater than mesh_size then become zero. But still not converge.
Anyways, I will try more ideas.
Best regards and thanks for your help.

Hi again,
So, PCG is now converging, it was a problem with my discretization process... anyway
is it normal that this function
   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);

needed to go so far away?
   Iteration :  83  (B r, r) = 9.97053e-20
Average reduction factor = 0.844423

Thanks!

is it normal that this function
   PCG(A, M, B, X, 1, 200, 1e-12, 0.0);

needed to go so far away?
   Iteration :  83  (B r, r) = 9.97053e-20
Average reduction factor = 0.844423

Yes, note that (B r, r) is the square of the residual norm induced by the preconditioner, the actual (preconditioned) residual norm would be around $10^{-10}$ in this case.
(Are you using AMG preconditioners? The reduction factor seems a bit high to me, but that could also because of challenging coefficients, etc.).

Hi thanks,
I am using this
	   // 9. Solve the system using PCG with symmetric Gauss-Seidel preconditioner.
	   GSSmoother M(A);

That should be the case because my density ranges for 3 orders of magnitude.
After that, I wanted to try the mesh refinement,

Is it ok to have hanging nodes in the (periodical) boundaries?
Thanks again for the support.


Fix bug in DeleteNeighbor

stop iterating after changing the list reference

Related to https://github.com/docker/editions/issues/439
Signed-off-by: Alessandro Boch<EMAIL_ADDRESS>

ping @mrjana

LGTM


peer review

There are multiple README files in the repo and you could add details to the README file about how to run your test

do not push node modules to the github repo

are u sure the test server run on localhost:3000 instead of 5173? i cannot run tests


Create a source for CloudWatch

Example use cases: send AWS CloudWatch logs to Datadog or Oracle's storage Service.
originally created by @cab105

Will setup the source to extract both log and metric data.


Update to v1.6.2

Version bump, merging immediately.

docker-library PR: https://github.com/docker-library/official-images/pull/582


Styling individual elements example missing.

In the last version of recurly.js, we could create individual card elements and style them.  This doesn't seem possible in the latest version, and if it is there are no examples here on how to style a form with custom elements.  Please advise.

btw - here is the HTML code that I'm trying along with the js:
<form data-action="subscriptions#paymentFormSubmit" action="/user/subscriptions" accept-charset="UTF-8" method="post">
<input data-recurly="token" name="recurly-token" type="hidden">
<input data-recurly="month" id="cardMonth" name="cardMonth" type="hidden">
<input data-recurly="year" id="cardYear" name="cardYear" type="hidden">

First Name
<input class="form-control" data-recurly="first_name" type="text" value="Ebony" style="background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAfBJREFUWAntVk1OwkAUZkoDKza4Utm61iP0AqyIDXahN2BjwiHYGU+gizap4QDuegWN7lyCbMSlCQjU7yO0TOlAi6GwgJc0fT/fzPfmzet0crmD7HsFBAvQbrcrw+Gw5fu+AfOYvgylJ4TwCoVCs1ardYTruqfj8fgV5OUMSVVT93VdP9dAzpVvm5wJHZFbg2LQ2pEYOlZ/oiDvwNcsFoseY4PBwMCrhaeCJyKWZU37KOJcYdi27QdhcuuBIb073BvTNL8ln4NeeR6NRi/wxZKQcGurQs5oNhqLshzVTMBewW/LMU3TTNlO0ieTiStjYhUIyi6DAp0xbEdgTt+LE0aCKQw24U4llsCs4ZRJrYopB6RwqnpA1YQ5NGFZ1YQ41Z5S8IQQdP5laEBRJcD4Vj5DEsW2gE6s6g3d/YP/g+BDnT7GNi2qCjTwGd6riBzHaaCEd3Js01vwCPIbmWBRx1nwAN/1ov+/drgFWIlfKpVukyYihtgkXNp4mABK+1GtVr+SBhJDbBIubVw+Cd/TDgKO2DPiN3YUo6y/nDCNEIsqTKH1en2tcwA9FKEItyDi3aIh8Gl1sRrVnSDzNFDJT1bAy5xpOYGn5fP5JuL95ZjMIn1ya7j5dPGfv0A5eAnpZUY3n5jXcoec5J67D9q+VuAPM47D3XaSeL4AAAAASUVORK5CYII="); background-repeat: no-repeat; background-attachment: scroll; background-size: 16px 18px; background-position: 98% 50%; cursor: auto;">

Last Name
<input class="form-control" data-recurly="last_name" type="text" value="Scott">

Card Number
<input class="form-control" data-recurly="number" id="cardNumber" type="text">

Expiration
<input class="form-control" data-subscriptions-target="cardExpiry" id="cardExpiry" placeholder="MM/YY" type="text">

CVC
<input class="form-control" id="cardCVV">

Zipcode
<input class="form-control" data-recurly="postal_code" type="text">

<button class="btn btn-primary mt-3 w-100">Complete Your Membership</button>

</form>

const elements = recurly.Elements();
const cardNumberElement = elements.CardNumberElement();
const cardMonthElement = elements.CardMonthElement();
const cardYearElement = elements.CardYearElement();
const cardCvvElement = elements.CardCvvElement();

// form submit
        let expParts = this.cardExpiryTarget.value.split('/')
        console.log('expParts', expParts, this.cardExpiryTarget.value);
        $('#cardMonth').val(expParts[0]);
        $('#cardYear').val(expParts[1]);

        cardNumberElement.attach('#cardNumber');
        cardMonthElement.attach('#cardMonth');
        cardYearElement.attach('#cardYear');
        cardCvvElement.attach('#cardCVV');

Hello!
Looking at your html code I'm making a quick assumption that you're upgrading from using the v3 version of our library. First I'd like to point you to https://developers.recurly.com/reference/recurly-js/index.html#upgrading-from-v3 which will go over how you can rewrite your integration and get you properly setup to work with our latest version.
Once that is setup you can refer to: https://developers.recurly.com/reference/recurly-js/index.html#styling-elements for documentation on stylizing the elements as you instantiate them.

@gilv93 not at all.  Everything you see there was build in the last week directly from your existing docs - the exact ones you point to in your response - which DO NOT provide ANY guidance on how to attach to individual form elements like the above.

@gilv93 as a point of reference, if you look at the docs for 4.10.3 (https://developers.recurly.com/reference/recurly-js/v4.10.3/index.html) there is a section called "Individual card fields" that goes into detail about how to set them up, bind them and style them.  This is MISSING from the latest docs and needs to be added to provide us with some guidance, or at least include an example in this repo that demonstrates this.

Sorry about my assumption from before!
You're right that the docs aren't clear enough and we'll be updating them to add more clarity where we can.
Starting with v4.11.0 we introduced Elements and steered away from Hosted Fields.  In previous versions such as 4.10.3 we could configure these Hosted Fields using syntax such as recurly.configure, but since we're using elements we can now configure the style when instantiating them:
const cardMonthElement = elements.cardMonthElement({
    inputType: 'mobileSelect',
    style: {
      fontSize: '1em',
      placeholder: {
        color: 'gray !important',
        fontWeight: 'bold',
        content: {
          number: 'Card number',
          cvv: 'CVC'
        }
      },
      invalid: {
        fontColor: 'red'
      }
   }
})

The attach method itself should be called on a container HTMLElement such as an HTMLDivElement, calling it on an  tag may prevent the element from rendering. We have just updated the docs to be more clear on this.
If you're still having trouble, if you could show us a demo or put your code on https://codesandbox.io/ (or another similar tool), then we can inspect to see what’s going on.

@gilv93 I really think you are way to close to this to understand the issue.
This isn't about STYLING.  This is about ATTACHING and BINDING to the fields.  This was clearly spelled out in the previous docs.
I've already had to use your standard recurlyElement, which is in the docs, since nothing else worked, so I don't have code.  I do have that form above, which clearly shows individual card elements.  I am STRONGLY recommending you create a new example in this repo called individual_card_elements and create a form that has a credit card number, card expiration (with mm and yy TOGETHER), CVV and zipcode form fields and then the JS code showing how to bind and style them.

That's a great idea, thank you. We appreciate your suggestion and are working on creating a multi-element card form example now to help clear things up.


Add method to delete datasource

Needed to be able to delete the datasource for hot reloading via webpack to work with nestjs. Thanks for the fork!
example:
  TypeOrmModule.forRootAsync({
      imports: [ConfigModule],
      inject: [AppConfig],
      useFactory: async (app_config: AppConfig) => app_config.getConnectionOptions(),
      async dataSourceFactory(options) {
        if (!options) {
          throw new Error('Invalid options passed');
        }
        deleteDataSourceByName('default'); // Hot reloading requires we remove the previous patched data source
        return addTransactionalDataSource(new DataSource(options));
      },
    }),

Thanks for this PR.
You can also use something like that to prevent adding data source several times:
import { addTransactionalDataSource, getDataSourceByName } from 'typeorm-transactional';

// ...

TypeOrmModule.forRootAsync({
    dataSourceFactory: async (options) => {
      return getDataSourceByName('default') || addTransactionalDataSource(new DataSource(options));
    },
  });

Thanks for this PR.
You can also use something like that to prevent adding the data source several times:
import { addTransactionalDataSource, getDataSourceByName } from 'typeorm-transactional';

// ...

TypeOrmModule.forRootAsync({
    dataSourceFactory: async (options) => {
      return getDataSourceByName('default') || addTransactionalDataSource(new DataSource(options));
    },
  });

I tried that first, but the nestjs typeorm didn't like reusing the same datasource. I think there was a race condition during hot reloading where it would get closed by the previous server.

Thanks for merging!


Update MongoDB Version

Updating MongoDB Java Driver to 2.13.0

last error was fixed by #210 . restarted CI build.

Could you rebase to current master so you'll pick up the fix for mapkeeper?