Flaky test_coin_selection

#873 introduced some assertions on the size of ancestors. These might not hold, however it only happens with a low probability (around 2% in my tests).
How to reproduce
First make sure you've got pytest-repeat in your virtual env to be able to use --count. Then just run the test a few hundreds time in parallel until it fails:
cargo build && pytest -vvv -n 40 --count 100 -k coin_selection tests/

Here is an example of failures:
==================================================================================================== FAILURES =====================================================================================================                                                                                [225/12954]
___________________________________________________________________________________________ test_coin_selection[33-100] ___________________________________________________________________________________________
[gw32] linux -- Python 3.11.8 /home/darosior/projects/wizardsardine/liana/venv/bin/python3

lianad = <test_framework.lianad.Lianad object at 0x7f1048213c10>, bitcoind = <test_framework.bitcoind.Bitcoind object at 0x7f1048446b10>

    def test_coin_selection(lianad, bitcoind):
        """We can create a spend using coin selection."""
        # Send to an (external) address.
        dest_addr_1 = bitcoind.rpc.getnewaddress()
        # Coin selection is not possible if we have no coins.
        assert len(lianad.rpc.listcoins()["coins"]) == 0
        assert "missing" in lianad.rpc.createspend({dest_addr_1: 100_000}, [], 2)

        # Receive a coin in an unconfirmed deposit transaction.
        recv_addr_1 = lianad.rpc.getnewaddress()["address"]
        deposit_1 = bitcoind.rpc.sendtoaddress(recv_addr_1, 0.0012)  # 120_000 sats
        wait_for(lambda: len(lianad.rpc.listcoins()["coins"]) == 1)
        # There are still no confirmed coins or unconfirmed change
        # to use as candidates for selection.
        assert len(lianad.rpc.listcoins(["confirmed"])["coins"]) == 0
        assert len(lianad.rpc.listcoins(["unconfirmed"])["coins"]) == 1
        assert lianad.rpc.listcoins(["unconfirmed"])["coins"][0]["is_change"] is False
        assert "missing" in lianad.rpc.createspend({dest_addr_1: 100_000}, [], 2)

        # Confirm coin.
        bitcoind.generate_block(1, wait_for_mempool=deposit_1)
        wait_for(lambda: len(lianad.rpc.listcoins(["confirmed"])["coins"]) == 1)
        # Coin selection now succeeds.
        spend_res_1 = lianad.rpc.createspend({dest_addr_1: 100_000}, [], 2)
        assert "psbt" in spend_res_1
        assert len(spend_res_1["warnings"]) == 0
        # Increase spend amount and we have insufficient funds again even though we
        # now have confirmed coins.
        assert "missing" in lianad.rpc.createspend({dest_addr_1: 200_000}, [], 2)

        # The transaction contains a change output.
        spend_psbt_1 = PSBT.from_base64(spend_res_1["psbt"])
        assert len(spend_psbt_1.o) == 2
        assert len(spend_psbt_1.tx.vout) == 2

        # Sign and broadcast this Spend transaction.
        spend_txid_1 = sign_and_broadcast_psbt(lianad, spend_psbt_1)
        wait_for(lambda: len(lianad.rpc.listcoins()["coins"]) == 2)
        # Check that change output is unconfirmed.
        assert len(lianad.rpc.listcoins(["unconfirmed"])["coins"]) == 1
        assert lianad.rpc.listcoins(["unconfirmed"])["coins"][0]["is_change"] is True
        assert len(lianad.rpc.listcoins(["spending"])["coins"]) == 1
        # We can use unconfirmed change as candidate.
        # Depending on the feerate, we'll get a warning about paying extra for the ancestor.
        dest_addr_2 = bitcoind.rpc.getnewaddress()
        # If feerate is higher than ancestor, we'll need to pay extra.

        # Try 10 sat/vb:
        spend_res_2 = lianad.rpc.createspend({dest_addr_2: 10_000}, [], 10)
        assert "psbt" in spend_res_2
        spend_psbt_2 = PSBT.from_base64(spend_res_2["psbt"])
        # The spend is using the unconfirmed change.
        assert spend_psbt_2.tx.vin[0].prevout.hash == uint256_from_str(
            bytes.fromhex(spend_txid_1)[::-1]
        )
        assert bitcoind.rpc.getmempoolentry(spend_txid_1)["ancestorsize"] == 161
        assert bitcoind.rpc.getmempoolentry(spend_txid_1)["fees"]["ancestor"] * COIN == 339
        # ancestor vsize at feerate 10 sat/vb = ancestor_fee / 10 = 339 / 10 = 33
        # extra_weight <= (extra vsize * witness factor) = (161 - 33) * 4 = 512
        # additional fee at 10 sat/vb (2.5 sat/wu) = 512 * 2.5 = 1280
        assert len(spend_res_2["warnings"]) == 1
        assert (
            spend_res_2["warnings"][0]
            == "An additional fee of 1280 sats has been added to pay for ancestors at the target feerate."
        )

        # Try 3 sat/vb:
        spend_res_2 = lianad.rpc.createspend({dest_addr_2: 10_000}, [], 3)
        assert "psbt" in spend_res_2
        spend_psbt_2 = PSBT.from_base64(spend_res_2["psbt"])
        # The spend is using the unconfirmed change.
        assert spend_psbt_2.tx.vin[0].prevout.hash == uint256_from_str(
            bytes.fromhex(spend_txid_1)[::-1]
        )
        assert bitcoind.rpc.getmempoolentry(spend_txid_1)["ancestorsize"] == 161
        assert bitcoind.rpc.getmempoolentry(spend_txid_1)["fees"]["ancestor"] * COIN == 339
        # ancestor vsize at feerate 3 sat/vb = ancestor_fee / 3 = 339 / 3 = 113
        # extra_weight <= (extra vsize * witness factor) = (161 - 113) * 4 = 192
        # additional fee at 3 sat/vb (0.75 sat/wu) = 192 * 0.75 = 144
        assert len(spend_res_2["warnings"]) == 1
        assert (
            spend_res_2["warnings"][0]
            == "An additional fee of 144 sats has been added to pay for ancestors at the target feerate."
        )

        # 2 sat/vb is same feerate as ancestor and we have no warnings:
        spend_res_2 = lianad.rpc.createspend({dest_addr_2: 10_000}, [], 2)
        assert "psbt" in spend_res_2
        assert len(spend_res_2["warnings"]) == 0
        spend_psbt_2 = PSBT.from_base64(spend_res_2["psbt"])
        # The spend is using the unconfirmed change.
        assert spend_psbt_2.tx.vin[0].prevout.hash == uint256_from_str(
            bytes.fromhex(spend_txid_1)[::-1]
        )

        # Get another coin to check coin selection with more than one candidate.
        recv_addr_2 = lianad.rpc.getnewaddress()["address"]
        deposit_2 = bitcoind.rpc.sendtoaddress(recv_addr_2, 30_000 / COIN)
        wait_for(lambda: len(lianad.rpc.listcoins(["unconfirmed"])["coins"]) == 2)
        assert (
            len(
                [
                    c
                    for c in lianad.rpc.listcoins(["unconfirmed"])["coins"]
                    if c["is_change"]
                ]
            )
            == 1
        )
        dest_addr_3 = bitcoind.rpc.getnewaddress()
        # As only one unconfirmed coin is change, we have insufficient funds.
        assert "missing" in lianad.rpc.createspend({dest_addr_3: 20_000}, [], 10)

        # If we include both unconfirmed coins manually, it will succeed.
        # We'll need to pay extra for each unconfirmed coin's ancestors.
        outpoints = [c["outpoint"] for c in lianad.rpc.listcoins(["unconfirmed"])["coins"]]

        spend_res_3 = lianad.rpc.createspend({dest_addr_3: 20_000}, outpoints, 10)
        assert "psbt" in spend_res_3
>       assert bitcoind.rpc.getmempoolentry(deposit_2)["ancestorsize"] == 165
E       assert 164 == 165

tests/test_spend.py:383: AssertionError
---------------------------------------------------------------------------------------------- Captured stdout setup ----------------------------------------------------------------------------------------------
Running tests in /tmp/lianad-tests-y1mp2i8l
-------------------------------------------------------------------------------------------- Captured stdout teardown ---------------------------------------------------------------------------------------------
Test failed, leaving directory '/tmp/lianad-tests-y1mp2i8l/test_coin_selection_1' intact
___________________________________________________________________________________________ test_coin_selection[71-100] ___________________________________________________________________________________________
[gw30] linux -- Python 3.11.8 /home/darosior/projects/wizardsardine/liana/venv/bin/python3

lianad = <test_framework.lianad.Lianad object at 0x7fe8e4e9d890>, bitcoind = <test_framework.bitcoind.Bitcoind object at 0x7fe8e5ac4590>

    def test_coin_selection(lianad, bitcoind):
        """We can create a spend using coin selection."""
        # Send to an (external) address.
        dest_addr_1 = bitcoind.rpc.getnewaddress()
        # Coin selection is not possible if we have no coins.
        assert len(lianad.rpc.listcoins()["coins"]) == 0
        assert "missing" in lianad.rpc.createspend({dest_addr_1: 100_000}, [], 2)

        # Receive a coin in an unconfirmed deposit transaction.
        recv_addr_1 = lianad.rpc.getnewaddress()["address"]
        deposit_1 = bitcoind.rpc.sendtoaddress(recv_addr_1, 0.0012)  # 120_000 sats
        wait_for(lambda: len(lianad.rpc.listcoins()["coins"]) == 1)
        # There are still no confirmed coins or unconfirmed change
        # to use as candidates for selection.
        assert len(lianad.rpc.listcoins(["confirmed"])["coins"]) == 0
        assert len(lianad.rpc.listcoins(["unconfirmed"])["coins"]) == 1
        assert lianad.rpc.listcoins(["unconfirmed"])["coins"][0]["is_change"] is False
        assert "missing" in lianad.rpc.createspend({dest_addr_1: 100_000}, [], 2)

        # Confirm coin.
        bitcoind.generate_block(1, wait_for_mempool=deposit_1)
        wait_for(lambda: len(lianad.rpc.listcoins(["confirmed"])["coins"]) == 1)
        # Coin selection now succeeds.
        spend_res_1 = lianad.rpc.createspend({dest_addr_1: 100_000}, [], 2)
        assert "psbt" in spend_res_1
        assert len(spend_res_1["warnings"]) == 0
        # Increase spend amount and we have insufficient funds again even though we
        # now have confirmed coins.
        assert "missing" in lianad.rpc.createspend({dest_addr_1: 200_000}, [], 2)

        # The transaction contains a change output.
        spend_psbt_1 = PSBT.from_base64(spend_res_1["psbt"])
        assert len(spend_psbt_1.o) == 2
        assert len(spend_psbt_1.tx.vout) == 2

        # Sign and broadcast this Spend transaction.
        spend_txid_1 = sign_and_broadcast_psbt(lianad, spend_psbt_1)
        wait_for(lambda: len(lianad.rpc.listcoins()["coins"]) == 2)
        # Check that change output is unconfirmed.
        assert len(lianad.rpc.listcoins(["unconfirmed"])["coins"]) == 1
        assert lianad.rpc.listcoins(["unconfirmed"])["coins"][0]["is_change"] is True
        assert len(lianad.rpc.listcoins(["spending"])["coins"]) == 1
        # We can use unconfirmed change as candidate.
        # Depending on the feerate, we'll get a warning about paying extra for the ancestor.
        dest_addr_2 = bitcoind.rpc.getnewaddress()
        # If feerate is higher than ancestor, we'll need to pay extra.

        # Try 10 sat/vb:
        spend_res_2 = lianad.rpc.createspend({dest_addr_2: 10_000}, [], 10)
        assert "psbt" in spend_res_2
        spend_psbt_2 = PSBT.from_base64(spend_res_2["psbt"])
        # The spend is using the unconfirmed change.
        assert spend_psbt_2.tx.vin[0].prevout.hash == uint256_from_str(
            bytes.fromhex(spend_txid_1)[::-1]
        )
        assert bitcoind.rpc.getmempoolentry(spend_txid_1)["ancestorsize"] == 161
        assert bitcoind.rpc.getmempoolentry(spend_txid_1)["fees"]["ancestor"] * COIN == 339
        # ancestor vsize at feerate 10 sat/vb = ancestor_fee / 10 = 339 / 10 = 33
        # extra_weight <= (extra vsize * witness factor) = (161 - 33) * 4 = 512
        # additional fee at 10 sat/vb (2.5 sat/wu) = 512 * 2.5 = 1280
        assert len(spend_res_2["warnings"]) == 1
        assert (
            spend_res_2["warnings"][0]
            == "An additional fee of 1280 sats has been added to pay for ancestors at the target feerate."
        )

        # Try 3 sat/vb:
        spend_res_2 = lianad.rpc.createspend({dest_addr_2: 10_000}, [], 3)
        assert "psbt" in spend_res_2
        spend_psbt_2 = PSBT.from_base64(spend_res_2["psbt"])
        # The spend is using the unconfirmed change.
        assert spend_psbt_2.tx.vin[0].prevout.hash == uint256_from_str(
            bytes.fromhex(spend_txid_1)[::-1]
        )
        assert bitcoind.rpc.getmempoolentry(spend_txid_1)["ancestorsize"] == 161
        assert bitcoind.rpc.getmempoolentry(spend_txid_1)["fees"]["ancestor"] * COIN == 339
        # ancestor vsize at feerate 3 sat/vb = ancestor_fee / 3 = 339 / 3 = 113
        # extra_weight <= (extra vsize * witness factor) = (161 - 113) * 4 = 192
        # additional fee at 3 sat/vb (0.75 sat/wu) = 192 * 0.75 = 144
        assert len(spend_res_2["warnings"]) == 1
        assert (
            spend_res_2["warnings"][0]
            == "An additional fee of 144 sats has been added to pay for ancestors at the target feerate."
        )

        # 2 sat/vb is same feerate as ancestor and we have no warnings:
        spend_res_2 = lianad.rpc.createspend({dest_addr_2: 10_000}, [], 2)
        assert "psbt" in spend_res_2
        assert len(spend_res_2["warnings"]) == 0
        spend_psbt_2 = PSBT.from_base64(spend_res_2["psbt"])
        # The spend is using the unconfirmed change.
        assert spend_psbt_2.tx.vin[0].prevout.hash == uint256_from_str(
            bytes.fromhex(spend_txid_1)[::-1]
        )

        # Get another coin to check coin selection with more than one candidate.
        recv_addr_2 = lianad.rpc.getnewaddress()["address"]
        deposit_2 = bitcoind.rpc.sendtoaddress(recv_addr_2, 30_000 / COIN)
        wait_for(lambda: len(lianad.rpc.listcoins(["unconfirmed"])["coins"]) == 2)
        assert (
            len(
                [
                    c
                    for c in lianad.rpc.listcoins(["unconfirmed"])["coins"]
                    if c["is_change"]
                ]
            )
            == 1
        )
        dest_addr_3 = bitcoind.rpc.getnewaddress()
        # As only one unconfirmed coin is change, we have insufficient funds.
        assert "missing" in lianad.rpc.createspend({dest_addr_3: 20_000}, [], 10)

        # If we include both unconfirmed coins manually, it will succeed.
        # We'll need to pay extra for each unconfirmed coin's ancestors.
        outpoints = [c["outpoint"] for c in lianad.rpc.listcoins(["unconfirmed"])["coins"]]

        spend_res_3 = lianad.rpc.createspend({dest_addr_3: 20_000}, outpoints, 10)
        assert "psbt" in spend_res_3
>       assert bitcoind.rpc.getmempoolentry(deposit_2)["ancestorsize"] == 165
E       assert 164 == 165

tests/test_spend.py:383: AssertionError
-------------------------------------------------------------------------------------------- Captured stdout teardown ---------------------------------------------------------------------------------------------
Test failed, leaving directory '/tmp/lianad-tests-ygbb0b2w/test_coin_selection_2' intact
___________________________________________________________________________________________ test_coin_selection[90-100] ___________________________________________________________________________________________
[gw19] linux -- Python 3.11.8 /home/darosior/projects/wizardsardine/liana/venv/bin/python3

lianad = <test_framework.lianad.Lianad object at 0x7fc416fd79d0>, bitcoind = <test_framework.bitcoind.Bitcoind object at 0x7fc416df2d90>

    def test_coin_selection(lianad, bitcoind):
        """We can create a spend using coin selection."""
        # Send to an (external) address.
        dest_addr_1 = bitcoind.rpc.getnewaddress()
        # Coin selection is not possible if we have no coins.
        assert len(lianad.rpc.listcoins()["coins"]) == 0
        assert "missing" in lianad.rpc.createspend({dest_addr_1: 100_000}, [], 2)

        # Receive a coin in an unconfirmed deposit transaction.
        recv_addr_1 = lianad.rpc.getnewaddress()["address"]
        deposit_1 = bitcoind.rpc.sendtoaddress(recv_addr_1, 0.0012)  # 120_000 sats
        wait_for(lambda: len(lianad.rpc.listcoins()["coins"]) == 1)
        # There are still no confirmed coins or unconfirmed change
        # to use as candidates for selection.
        assert len(lianad.rpc.listcoins(["confirmed"])["coins"]) == 0
        assert len(lianad.rpc.listcoins(["unconfirmed"])["coins"]) == 1
        assert lianad.rpc.listcoins(["unconfirmed"])["coins"][0]["is_change"] is False
        assert "missing" in lianad.rpc.createspend({dest_addr_1: 100_000}, [], 2)

        # Confirm coin.
        bitcoind.generate_block(1, wait_for_mempool=deposit_1)
        wait_for(lambda: len(lianad.rpc.listcoins(["confirmed"])["coins"]) == 1)
        # Coin selection now succeeds.
        spend_res_1 = lianad.rpc.createspend({dest_addr_1: 100_000}, [], 2)
        assert "psbt" in spend_res_1
        assert len(spend_res_1["warnings"]) == 0
        # Increase spend amount and we have insufficient funds again even though we
        # now have confirmed coins.
        assert "missing" in lianad.rpc.createspend({dest_addr_1: 200_000}, [], 2)

        # The transaction contains a change output.
        spend_psbt_1 = PSBT.from_base64(spend_res_1["psbt"])
        assert len(spend_psbt_1.o) == 2
        assert len(spend_psbt_1.tx.vout) == 2

        # Sign and broadcast this Spend transaction.
        spend_txid_1 = sign_and_broadcast_psbt(lianad, spend_psbt_1)
        wait_for(lambda: len(lianad.rpc.listcoins()["coins"]) == 2)
        # Check that change output is unconfirmed.
        assert len(lianad.rpc.listcoins(["unconfirmed"])["coins"]) == 1
        assert lianad.rpc.listcoins(["unconfirmed"])["coins"][0]["is_change"] is True
        assert len(lianad.rpc.listcoins(["spending"])["coins"]) == 1
        # We can use unconfirmed change as candidate.
        # Depending on the feerate, we'll get a warning about paying extra for the ancestor.
        dest_addr_2 = bitcoind.rpc.getnewaddress()
        # If feerate is higher than ancestor, we'll need to pay extra.

        # Try 10 sat/vb:
        spend_res_2 = lianad.rpc.createspend({dest_addr_2: 10_000}, [], 10)
        assert "psbt" in spend_res_2
        spend_psbt_2 = PSBT.from_base64(spend_res_2["psbt"])
        # The spend is using the unconfirmed change.
        assert spend_psbt_2.tx.vin[0].prevout.hash == uint256_from_str(
            bytes.fromhex(spend_txid_1)[::-1]
        )
>       assert bitcoind.rpc.getmempoolentry(spend_txid_1)["ancestorsize"] == 161
E       assert 160 == 161

tests/test_spend.py:319: AssertionError
-------------------------------------------------------------------------------------------- Captured stdout teardown ---------------------------------------------------------------------------------------------
Test failed, leaving directory '/tmp/lianad-tests-z13nmef0/test_coin_selection_3' intact
============================================================================================= short test summary info =============================================================================================
FAILED tests/test_spend.py::test_coin_selection[33-100] - assert 164 == 165
FAILED tests/test_spend.py::test_coin_selection[71-100] - assert 164 == 165
FAILED tests/test_spend.py::test_coin_selection[90-100] - assert 160 == 161
==================================================================================== 3 failed, 197 passed in 84.23s (0:01:24) =====================================================================================

The value of a given vsize (from getmempoolentry) can vary from one test to another. I've checked and this also happens with the weight (e.g. from getrawtransaction).
The failures above and those that I've encountered all involve the RPC command returning a value 1 less than expected. Perhaps it's to do with a rounding error somewhere.
It's worth noting that the failing assertions are not essential to the test and what's important are the calculations given those values. Such a change has been applied to the test in #985 and the assertions could in principle be removed.

Yes, however i think they do end up making the calculation incorrect. Running the test a number of times on my Taproot branch would end up showcasing the flakiness iirc.

I think it's low-R signature grinding but don't take my word for it. Let me see if we can disable it.

Actually what about just adding a send to self before? So we use this tx as parent. And we don't grind R so its size should be fixed.


Getting rid of the static ioping binary

The static binary was originally added for three reasons:

Less effort and system clutter:
Among nench's software requirements, ioping is arguably the most uncommon
and least likely to be present by default.
Providing the static binary avoids additional installations, at least on
binary-compatible systems (Linux amd64 in this case).

Consistent output format:
ioping's output format changed between version 0.9 and 1.0, but 1.0 still
reports the old version:
% apt-cache policy ioping
ioping:
  Installed: 1.0-2
  Candidate: 1.0-2
  Version table:
 *** 1.0-2 500
        500 http://httpredir.debian.org/debian stretch/main amd64 Packages
        100 /var/lib/dpkg/status
% ioping -v
ioping 0.9

The same happens with the static binary, but there I can be certain which
version it is (and parse the output appropriately).

Consistent test scope:
ioping 1.0 added a second summary line (marked with > below) that reports
iops and throughput for the full deadline period (-w) requested on the
commandline, rather than the old summary that excludes data from some of the
probes (such as the "warmup" at the beginning):
% ioping -DRL -w 5 . # ioping 0.9

--- . ( ) ioping statistics ---
301 requests completed in 5.11 s, 58 iops, 14.7 MiB/s
min/avg/max/mdev = 778 us / 17.0 ms / 236.0 ms / 31.4 ms

% ioping -DRL -w 5 . # ioping 1.0

--- . ( ) ioping statistics ---
1.65 k requests completed in 4.99 s, 413.5 MiB read, 331 iops, 82.9 MiB/s
> generated 1.66 k requests in 5.00 s, 413.8 MiB, 330 iops, 82.7 MiB/s
min/avg/max/mdev = 2.75 ms / 3.01 ms / 66.0 ms / 2.10 ms

Long story short, the static binary could be made superfluous by switching to
the "opportunistic" summary line (which is present in both old and new
versions) and selecting it by its content rather than its position in the
output.
It would also be possible to use the -B option (which I wasn't aware of when
I created nench, but which is present even in older versions) and parse the
"raw" output it produces. It may be possible to re-use the Bps_to_MiBps
formatter that's already used for the dd and curl output.
The primary disadvantages would be potentially breaking comparability between
old and new nench results, and requiring the installation of ioping just
for the test.
Perhaps offer the static binary for manual installation (then documented in
the README) and use only the system-installed version if the user doesn't
request otherwise?

+1, to enable this to work on ARM platforms (like ampere altra)


Wrong command for Mobility Service Windows installation

On step 2 of the Windows installation procedure for the Mobility Service, the command is missing a parameter. It should include the /Platform "VmWare" flag, as documented here:
UnifiedAgent.exe /Role "MS" /InstallLocation "C:\Program Files (x86)\Microsoft Azure Site Recovery" /Platform "VmWare" /Silent

Document Details
⚠ Do not edit this section. It is required for docs.microsoft.com ➟ GitHub issue linking.

ID: 7f3d7cc0-ecf8-a48f-3c6e-8e18c7787c41
Version Independent ID: 31b358bd-1ee6-a058-8590-802941350e90
Content: Migrate machines as physical server to Azure with Azure Migrate. - Azure Migrate
Content Source: articles/migrate/tutorial-migrate-physical-virtual-machines.md
Service: azure-migrate
GitHub Login: @rayne-wiselman
Microsoft Alias: raynew

@helderpinto Thank you for bringing this to our notice, we will review the feedback and get back to you at the earliest.

@helderpinto Thanks a lot for bringing this to our attention. I have raised a PR to reflect the feedback, once its reviewed and approved it will be merged with the live document in next few days.
At this point it appears that there's no further edits or updates needed to the documentation, other than what is already listed above. We will go ahead and close this issue, however if you have any further queries please tag us in your reply and will be happy to reopen the thread and continue the conversation. Thank you


[listener] Logger should write using async pattern

Changed Logger writing to use fh -> end pattern when logging data

Coverage remained the same at 73.234% when pulling 90f7d779529c5f45b657627abf86bb761cbfa529 on svrdlans:logger_should_write_using_async_pattern into 8b4697dcccd28a821df39aa4b88e399ccfb37f8a on exponentially:master.


docs(samples): add auth user flow and query

Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly:

[X] Make sure to open an issue as a bug/issue before writing your code!  That way we can discuss the change, evaluate designs, and agree on the general idea
[X] Ensure the tests and linter pass
[X] Code coverage does not decrease (if any source code was changed)
[X] Appropriate docs were updated (if necessary)

since we cannot manually approve approval prompt, we should try to do mock testing instead here.


New API - Post review deployment

API review was completed in #99. However, that review was structured for visibility and cannot be merged directly.
This PR contains the approved API along with a buildable skeleton of the up-cpp library + tests. This aims to merge with main.
(Note: in draft state while verifying CI works still and finalizing a couple small items.)

The API has already been reviewed and approved in PR #99. I'm going to go ahead and merge this so that implementation work can start.


Search results item preview refactoring

Needs to refactor search results item preview, in particular - markdown editor.

Already refactored


Cannot find @angular modules from node_modules folder

TypeScript Version:  2.5.3 / insiders (2.5.3-insiders.20170919)
VS Code Version:  1.17.0-insider
Code
import { Component } from '@angular/core';

Expected behavior:
TypeScript should find the module as long as the node_modules folder is on the same level as the tsconfig.json file.
Actual behavior:
The module can't be found. When however you reference the location relative to the file the module is found, so this works:
import { Component } from '../node_modules/@angular/core';

Further Information
The project is a basic Angular project created by the angular-cli. It worked until the latest VS Code Insiders update.

This is not a support forum.
Questions should be asked at StackOverflow or on Gitter.im.

This is not a support forum.

Neither is this a question. It's a bug report. Other people have already reported similar issues which ended up being fixed, and this one started to appear with the latest insider version of TypeScript included in VS Code.

Make sure you're using one or both of the following:

moduleResolution: node
module: es2015

@DanielRosenwasser moduleResolution: node is set.

Can you share a self-contained repro.
also can you verify that it is not https://github.com/Microsoft/vscode/issues/34681

Automatically closing this issue for housekeeping purposes. The issue labels indicate that it is unactionable at the moment or has already been addressed.


Refactors worker waiting function

Adds a test to the cleanup case.

Closes #372
Signed-off-by: Chuck Ha<EMAIL_ADDRESS>What this PR does / why we need it:
I think this simplifies the logic for the worker.
Which issue(s) this PR fixes

Fixes #372

Special notes for your reviewer:
It's ok if you don't think this actually makes the code easier to read/follow. I'm happy to close it without merging.
Release note:
release-note

need a re-base then feel free to self-merge.


test sensor connections

DHT22
Water Level sensor

https://gpiozero.readthedocs.io/

https://www.raspberrypi.org/documentation/usage/gpio/

https://thejeswarareddyr.blogspot.com/2016/03/how-to-connect-mp3002mcp3008-adc-chip.html

https://www.rototron.info/raspberry-pi-analog-water-sensor-tutorial/

https://www.linuxnorth.org/raspi-sump/

https://pimylifeup.com/raspberry-pi-humidity-sensor-dht22/

DHT22 logging works. https://gist.github.com/rajasiman/cef147470c689e3fd4a97e73edcb7949
Water Level Sensor pending.

Water Level Sensor:
Wiring:

https://github.com/adafruit/Adafruit_CircuitPython_MCP3xxx/

All sensors with wiring


 This PR fixes it.

#9856  👈
master

This stack of pull requests is managed by Graphite. Learn more about stacking.

Join @benesjan and the rest of your teammates on  Graphite


Reliably uploading local files as inputs

Hi everyone,
I'm struggling a bit with uploading/running local files to a hosted model. I am following the python instructions on this page: Input Files.
So far I have the input argument in my model's predict signature set to type of cog.Path. Next, I'll process my local file as so: contents = open("my_file.pdf", "rb").
But when I try sending that _io.BufferedReader as part of replicate.run("model_name", input={"file": contents}), it seems that only an empty file goes through. It's definitely not a Path that's arriving to the function.
The main question: what exactly happens to this buffered reader when it's sent to the model from a local file? How should I be processing the object in the Predictor's predict to appropriately handle this opened reader? Do I need to save it back out to a temp file?
Thanks in advance for the help!

So far I have the input argument in my model's predict signature set to type of cog.Path. Next, I'll process my local file as so: contents = open("my_file.pdf", "rb").

When you say the input argument, does it look like this:
def predict(self, file: Path = Input("a file")):

The main question: what exactly happens to this buffered reader when it's sent to the model from a local file?

The Replicate client library will upload that file to a temporary location on the Replicate service and use the URL in the input to the model. So your local file will be transformed into something like: https://api.replicate.com/v1/files/123/download?expiry=1731497450&owner=xyz&signature=abc. The cog model will then download this file and write it to disk, providing the predict with a path on disk to that file.

How should I be processing the object in the Predictor's predict to appropriately handle this opened reader? Do I need to save it back out to a temp file?

Path is a subclass of pathlib.Path, and can be accessed using methods like file.absolute() or file.open().
Could you explain a little more about how you are using the replicate client to work with a local cog server? I have a feeling the issue you are seeing is that the files API will not exist as part of cog.
One thing to try immediately would be to use data URLs instead of the default files API:
output = replicate.run("my/model:xyz", input={}, file_encoding_strategy="base64")

Though, I've just found a bug in that path so let me look into this some more.


What happens when multiple autoscale rules are specified? Is it AND or OR?

If I have two scale-in rules specified, will scale-in (e.g. instance count reduced) occur if EITHER scale-in rule applies, or only if BOTH apply?
For example: let's say I have an autoscale rule that says <50% CPU avg for 5 minutes, and an autoscale rule that says <100 items in queue (which triggers the function).
Will autoscale occur when I am at 10% CPU for 5 minutes and 1 second, even if there are consistently 200 messages on the queue?
Please provide additional documentation around more nuanced contexts such as a Function app hosting two functions, needing multiple autoscale rules, and how the autoscale rules combine (AND or OR).

Document Details
⚠ Do not edit this section. It is required for docs.microsoft.com ➟ GitHub issue linking.

ID: 339821ab-e64d-f317-953b-d04952d69359
Version Independent ID: 26728114-60a2-5a0d-d037-7947c99a3bad
Content: Get started with autoscale in Azure
Content Source: articles/azure-monitor/platform/autoscale-get-started.md
Service: azure-monitor
Sub-service: autoscale
GitHub Login: @rajram
Microsoft Alias: rajram

Found the Azure portal editor has "When... AND" and "When... OR" - would still be good to highlight in docs.

@plzm Thanks for the feedback. Glad that you have self resolved the issue.
@rajram Can you please consider document enhancement with a note explaining about "When AND" and "When OR" ?


oneTBB fails to link properly when the project is hosted in a directory with space characters

Environment: Ubuntu 2022.4.1 LTS over WSL.
Compiler: Clang 14
my project is hosted in /mnt/d/Github Repos/, when building the project it will fail to build and report:
[build] clang: error: no such file or directory: 'Repos/NovelRT/build/internal/oneTBB/src/src/tbb/def/lin64-tbb.def'

It appears that the path starts after the space. After replacing the space character it builds as expected.

after looking at it, i think this might be a duplicate of #387 as the symptoms seem the same. so i am closing this for now.


Move to yarn?

Would love to know your thoughts on this Dustan.

Dooooeeeetttt!

LOL, awesome! I will add it to this one https://github.com/iamdustan/smoothscroll/pull/104
You can review it without the yarn, I'll just update the docs and generate the lock file.

Just my personal view: after switch to npm 5 there is no need for yarn.

Regardless of npm 5 or yarn - play with the lock file. It's a nice idea, but well, it's very cumbersome to have it in git. I don't found best strategy to handle this file.

It's true that there aren't huge differences with npm5. I do like better how it handle scripts, flags and on screen messages.
Either way, people will only need it if they collaborate with code which doesn't happen often, and having to install yarn is not a curse or a big blocker.


keydb 5.3.3 OS X Build Break

Describe the bug
relates to https://github.com/Homebrew/homebrew-core/pull/52164
** Log Files **
build log, https://github.com/Homebrew/homebrew-core/pull/52164/checks?check_run_id=536351908
In file included from fastlock.cpp:65:
/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk/usr/include/ucontext.h:43:2: error: The deprecated ucontext routines require _XOPEN_SOURCE to be defined
#error The deprecated ucontext routines require _XOPEN_SOURCE to be defined
 ^
fastlock.cpp:180:5: error: use of undeclared identifier 'getcontext'
    getcontext(&ctxt);
    ^
2 errors generated.

Please copy everything after this line.
To Reproduce

unzip v5.3.3.tar.gz
make install

Simple fix, I updated the release tag to 98da479d

@JohnSully Thanks, it works for me.


[feature] express test

Codecov Report

:exclamation: No coverage uploaded for pull request base (feature/onetap_v2@35ce6dc). Click here to learn what that means.
The diff coverage is 0%.

@@                 Coverage Diff                 @@
##             feature/onetap_v2   #1369   +/-   ##
===================================================
  Coverage                     ?   2.31%
  Complexity                   ?     124
===================================================
  Files                        ?     434
  Lines                        ?   16702
  Branches                     ?    1414
===================================================
  Hits                         ?     386
  Misses                       ?   16291
  Partials                     ?      25

Impacted Files
Coverage Δ
Complexity Δ

.../android/px/internal/datasource/GroupsService.java
0% <0%> (ø)
0 <0> (?)

...tures/paymentresult/components/HeaderRenderer.java
0% <0%> (ø)
0 <0> (?)

...android/px/internal/view/AmountDescriptorView.java
0% <0%> (ø)
0 <0> (?)

...ess_result/components/BusinessPaymentRenderer.java
0% <0%> (ø)
0 <0> (?)

...ernal/features/express/ExpressPaymentFragment.java
0% <0%> (ø)
0 <0> (?)

...ndroid/px/configuration/AdvancedConfiguration.java
0% <0%> (ø)
0 <0> (?)

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update 35ce6dc...2d2ad92. Read the comment docs.


Remove Debug Configuration

@guidow PTAL, I've pulled this out of #420 since this is something we can discuss/merge outside of that larger change.

I'm actually using the debug configuration on my development machine. As far as I can tell, the second commit there should make sure it still works for this, but I'm going to have to actually test that when I get back to the office – which won't be before wednesday.

Thanks, take your time.  No real rush here since #420 is still a work in progress for another day or so and this PR is just a related subset to chose changes.

I've had a chance to test this now. It still works, and I have no objections left to merging this.


M3-409 - Global Volume Create Drawer Tests

Added tests that test creating a volume via the global create drawer (creating with a region, no linode, creating with a region + lionde, removing all volumes after the test).
Added additional data-qa attributes to volume drawer for tests

To Test
yarn && yarn start
yarn selenium # in a separate shell
yarn e2e --file e2e/specs/create/create-volume.spec.js # in a separate shell

╰─ yarn e2e --file e2e/specs/create/create-volume.spec.js             ✘
yarn run v1.3.2
$ ./node_modules/.bin/wdio ./e2e/config/wdio.conf.js --file e2e/specs/create/create-volume.spec.js

․․․․․․․․․

9 passing (89.40s)

✨  Done in 90.83s.


BuildOutputCleanupRegistry forces OOTB clean task to be eagerly configured even when not required

BuildOutputCleanupRegistry resolveOutputs() forces clean task to be eagerly configured even when not required, invalidating task avoidance. Easily reproduced by gw help on any project that applies the java plugin (thus creating and registering outputs of a clean task)
Expected Behavior
The clean task should not be eagerly configured when not required
Current Behavior
The clean task is always configured, no matter what task is fired.
Context
The general guidance of Gradle is to aim for all tasks to support avoidance, such that a very simple build like gw help should only have 1 task configured, but that isn't possible with the OOTB plugins and build output registry forcing eager configuration of tasks. I want to be able to setup a test of our Gradle project to assert that gw help only has 1 task configured, such that future changes to our Gradle scripts don't accidentally create eager configured tasks, but I can't do that until I can actually get it to only configure one task, for a simple gw help in our project we get 84 tasks configured (83 clean tasks, 1 help task..
Steps to Reproduce
https://github.com/nhoughto/gradle-clean-eager-repro
gw help --debug | grep Realize
2022-01-06T11:00:14.969+1100 [DEBUG] [org.gradle.internal.operations.DefaultBuildOperationRunner] Build operation 'Realize task :help' started
2022-01-06T11:00:14.969+1100 [DEBUG] [org.gradle.internal.operations.DefaultBuildOperationRunner] Completing Build operation 'Realize task :help'
2022-01-06T11:00:14.969+1100 [DEBUG] [org.gradle.internal.operations.DefaultBuildOperationRunner] Build operation 'Realize task :help' completed
2022-01-06T11:00:14.980+1100 [DEBUG] [org.gradle.internal.operations.DefaultBuildOperationRunner] Build operation 'Realize task :clean' started
2022-01-06T11:00:14.980+1100 [DEBUG] [org.gradle.internal.operations.DefaultBuildOperationRunner] Completing Build operation 'Realize task :clean'
2022-01-06T11:00:14.980+1100 [DEBUG] [org.gradle.internal.operations.DefaultBuildOperationRunner] Build operation 'Realize task :clean' completed

Your Environment
Build scan URL: https://gradle.com/s/cogh5z3kduqmi

We're seeing this too and needed to manually narrow down where this was coming from as build scans didn't indicate any of our plugins and build scripts for that. It would be nice to indicate when this is caused by gradle itself.


Regression test for #1489

Current coverage is 92.91%

Merging #1511 into master will decrease coverage by -0.04% as of 97d0d5c

@@            master   #1511   diff @@
======================================
  Files           87      87
  Stmts         5505    5505
  Branches      1022    1022
  Methods          0       0
======================================
- Hit           5117    5115     -2
  Partial          9       9
- Missed         379     381     +2

Review entire Coverage Diff as of 97d0d5c

Powered by Codecov. Updated on successful CI builds.


KAFKA-2790: doc improvements

Some general comments:

We could also change all the hrefs to the 0.9.0 java docs in api.html (e.g. new producer section).
We could remove the migration section as a whole since it is from xx -> 0.8, or change the section as "if your current version is older than 0.8.0, before upgrade to 0.9.0 you need to do a one-time migration following instructions here {href}" in the upgrade.html.

@junrao and @guozhangwang - addressed your suggestions.

Also the auto generated data table seems ill-formated in browser (or is it only my browser)? http://kafka.apache.org/090/documentation.html#producerapi

Thanks for the patch. LGTM


add extra feilds that will be needed in the course schema and dtos for client side integration

Add extra fields that will be needed in the course schema and dtos for client side integration:-
1). tags with its enums
2). courseDetails
3). courseLevel
4). courseThumbnail
5). courseTrailerUrl

[x] My code follows the code style of this project.
[x] All new and existing tests passed.
[x] My changes generate no new warnings
[x] The title of my pull request is a short description of the requested changes.

@Abhishek-kumar09 the extra fields that will be required

Please add swagger response body for this PR.

ok

Please add swagger response body for this PR.

There is no option to get all the reviews on a course.
Add a get request that fetches all the review for a course.

The reviews are populated with the getall courses object

So with this, all reviews will be available for that course.

@Abhishek-kumar09 the changes have been made

@Abhishek-kumar09     courseLevel converted to enum type and @Isenum prop addd to both and tags converted to array

@Abhishek-kumar09 added swagger for the new routes and merged conflcts


Use an interface for the replica type.

This change introduces an interface to represent replicas, instead of a raw struct.
The underlying implementation is now provided by a protocol buffers struct, via an adaptor (see replicapb.Adaptor.
This allows us to have a protocol-buffers representation for replicas, which is fast to obtain (since it's the only implementation, at least currently), but without coupling all of the other packages to the protocol-buffers representation.

@koden-km I know you're not familiar with this code, but it was quite a big change, so I'd appreciate your usual veracity in telling me what I messed up, if you find the time :)

Fixed on master, thanks man :)


[Question] Will there ever be an update to this that includes support for spatial shaders?

I was really looking forward to utilizing this in order to convert Godot shaders into GLSL shaders that I could use for another project, but I noticed the lack of support for Spatial shaders as I was getting ready to compile this, and I was wondering if there would ever be an update to add support for them.

dfranx doesn't have as much time to work on SHADERed lately, so I doubt this plugin will be updated anytime soon.

Understandable


Add additional scrape configs for getting minio relevant metrics

Summary

MinIO scrapes metrics from four different endpoints depending on what the metrics are (cluster, node, bucket, resource)
We want to scrape all of them to ensure we can effectively monitor the instances as well as the buckets

Checklist

[x] Categorize the PR by setting a good title and adding one of the labels:
bug, enhancement, documentation, change, breaking, dependency
as they show up in the changelog
[x] Update tests.
[ ] Link this PR to related issues.

LGTM Are you sure you added every endpoint?

Yes. According to https://min.io/docs/minio/linux/operations/monitoring/collect-minio-metrics-using-prometheus.html there are currently 4 endpoints available.


Get rid of the "Babel + browserslist defaults config hack"

See aa249895dd73568380df26eeb0b8e86e8a85f2ea.
We also might want to consider to revert .babelrc.js back to a JSON file if there isn't any other reason for it being a script, since it should decrease build times.
We could also move the browserslist config to its own file then, although the docs actually recommend having it in the package.json file. What I don't like about that is that you can't add comments, there.

Things might change here with Babel 8: See https://github.com/babel/babel/issues/9962 and https://github.com/babel/babel/issues/10746.
Until then, we can't really do anything here, I guess.


Telegram does not send photo if URL requires HttpDigestAuth

Home Assistant release (hass --version):
0.43.2
Python release (python3 --version):
3.5
Component/platform:
notify/telegram
Description of problem:
When creating an automation/script using notify/Telegram with URL photo support, it will not work if the webserver requires an HTTPDigestAuth.
For example, the new Amcrest firmware requires a HtttpDigestAuth to download the camera snapshot.
Expected:
Notification to work and send photo/picture as expected.
Problem-relevant configuration.yaml entries and steps to reproduce:
#scripts.yaml
send_snapshot_telegram_driveway:
  alias: 'Send snapshot driveway camera'
  sequence:
    - service: notify.telebot
      data:
        title: "Driveway Motion"
        message: "There is a motion detected on the driveway"
        data:
          photo:
            - url: !secret amcrest_driveway_snapshot_url
              username: !secret amcrest_driveway_username
              password: !secret amcrest_driveway_password

Traceback (if applicable):
-04-30 21:49:12 INFO (MainThread) [homeassistant.components.automation] Executing Send snapshot while backyard motion sensor is True
17-04-30 21:49:12 ERROR (MainThread) [homeassistant.core] Error doing job: Task exception was never retrieved
Traceback (most recent call last):
  File "/opt/srv/homeassistant/.homeassistant/deps/telegram/inputfile.py", line 83, in __init__
    self.mimetype = self.is_image(self.input_file_content)
  File "/opt/srv/homeassistant/.homeassistant/deps/telegram/inputfile.py", line 164, in is_image
    raise TelegramError('Could not parse file content')
telegram.error.TelegramError: Could not parse file content

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib64/python3.5/asyncio/tasks.py", line 241, in _step
    result = coro.throw(exc)
  File "/opt/srv/homeassistant/.virtualenvs/hass_venv/lib/python3.5/site-packages/homeassistant/core.py", line 1015, in _event_to_service_call
    yield from service_handler.func(service_call)
  File "/opt/srv/homeassistant/.virtualenvs/hass_venv/lib/python3.5/site-packages/homeassistant/components/notify/__init__.py", line 136, in async_notify_message
    yield from notify_service.async_send_message(**kwargs)
  File "/usr/lib64/python3.5/asyncio/futures.py", line 380, in __iter__
    yield self  # This tells Task to wait for completion.
  File "/usr/lib64/python3.5/asyncio/tasks.py", line 304, in _wakeup
    future.result()
  File "/usr/lib64/python3.5/asyncio/futures.py", line 293, in result
    raise self._exception
  File "/usr/lib64/python3.5/concurrent/futures/thread.py", line 55, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/srv/homeassistant/.virtualenvs/hass_venv/lib/python3.5/site-packages/homeassistant/components/notify/telegram.py", line 105, in send_message
    self.send_photo(photo_data)
  File "/opt/srv/homeassistant/.virtualenvs/hass_venv/lib/python3.5/site-packages/homeassistant/components/notify/telegram.py", line 156, in send_photo
    chat_id=self._chat_id, photo=photo, caption=caption)
  File "/opt/srv/homeassistant/.homeassistant/deps/telegram/bot.py", line 125, in decorator
    result = func(self, *args, **kwargs)
  File "/opt/srv/homeassistant/.homeassistant/deps/telegram/bot.py", line 151, in decorator
    result = self._request.post(url, data, timeout=kwargs.get('timeout'))
  File "/opt/srv/homeassistant/.homeassistant/deps/telegram/utils/request.py", line 198, in post
    data = InputFile(data)
  File "/opt/srv/homeassistant/.homeassistant/deps/telegram/inputfile.py", line 87, in __init__
    self.mimetype = mimetypes.guess_type(self.filename)[0] or DEFAULT_MIME_TYPE
  File "/usr/lib64/python3.5/mimetypes.py", line 289, in guess_type
    return _db.guess_type(url, strict)
  File "/usr/lib64/python3.5/mimetypes.py", line 114, in guess_type
    scheme, url = urllib.parse.splittype(url)
  File "/usr/lib64/python3.5/urllib/parse.py", line 862, in splittype
    match = _typeprog.match(url)
TypeError: expected string or bytes-like object

Adding on my plate to test if it works on 0.45.1

Works great on 0.46
        data:
          photo:
            - url: !secret amcrest_living_room_snapshot_url
              authention: 'digest'
              username: !secret amcrest_living_room_username
              password: !secret amcrest_living_room_password


Failing to decode bytes when cicp index is specified by user for some mhas conformace streams

Hello,
I have found a following behavior of the decoder:

set str_dec_api.input_config.ui_cicp_layout_idx to a value other than the default(0), for example 1 or 2
try to decode 'C104_3_FD.mhas' from the conformance streams
ia_mpegh_dec_execute will not return error, but:

pstr_out_cfg->i_bytes_consumed > 0
pstr_out_cfg->num_out_bytes == 0
Expected:
User can select cicp index to use for decoding and decoder will provide output bytes > 0, or if not possible ia_mpegh_dec_init + ia_mpegh_dec_execute should fail if selected cicp index is not valid.
Selecting cicp index manually works for other conformance streams and MHA mp4 files, for example: when decoding 'H8_3_FD.mhas' I can select any cicp value and decoder will set proper num_out_bytes. Or I can set the default cicp value and decoder will detect the cicp layout from the file.
I guess I have also follow up question:

is user allowed to set any cicp index?
if the user provided index does not match the file or stream - is the decoder expected to mux the channels to fit into selected cicp index?

Could you please clarify?
Best Regards,
Adam

Hi @miartad,
We will try this and get back to you at the earliest!

Hi @miartad ,
The stream "C104_3_FD.mhas" is a special stream with speaker layout index value set to "0" in the bit stream. The value 0 for this field refers to "any speaker set up". Since the source speaker set up is not clear, its not possible to initialize a rendering algorithm for any CICP option. We have corrected the error propogation path in  #41.
Also we have tested CICP option for other streams where speaker layout set index value in the bit-stream is not "0" and the option works fine for those cases.
Thanks!

Hello @miartad,
I was wondering if you got a chance to validate the fix pushed for this issue.
Thanks!

hi @SakethSathuvalli
I tried to decode the file.
when ui_cicp_layout_idx in input config is set to 0 the file will decode.
if I try to set a ui_cicp_layout_idx to value other than 0 for this kind of file the error during decoder init is propagated correctly.
thanks!


ddev start output showing docker ad "What's next"

Expected Behavior
I deleted all images and then did a ddev start
I expected normal loading behavior
Actual Behavior
ddev start shows many of these:
What's Next?
  View summary of image vulnerabilities and recommendations → docker scout quickview ddev/ddev-ssh-agent:v1.22.0-beta1

Steps To Reproduce
docker rmi -f $(docker images -q)
ddev start
Anything else?
No response

https://github.com/docker/for-mac/issues/6904


goreleaser: added termux build

Pull Request Test Coverage Report for Build<PHONE_NUMBER>5
Details

0 of 0   changed or added relevant lines in 0 files are covered.
No unchanged relevant lines lost coverage.
Overall coverage remained the same at 52.32%

Totals

Change from base Build<PHONE_NUMBER>5:
0.0%

Covered Lines:
203

Relevant Lines:
388

💛  - Coveralls


Raw Therapee label

An Open Source RAW Converter

I am having issues with this label. It looks like the .zip contains a folder which then contains a .dmg.
Installomator is attempting to mount: /Users/bigmacadmin/Documents/GitHub/installomator/build/RawTherapee_macOS_13.3_Universal_5.10.dmg
however, the full path is actually: /Users/bigmacadmin/Documents/GitHub/installomator/build/RawTherapee_macOS_13.3_Universal_5.10_folder/RawTherapee_macOS_13.3_Universal_5.10.dmg
Is it possible to get Installomator to find this dmg by adjusting the label? Or would we require an improvement to how Installomator locates the DMG path?
./utils/test-pr.sh 1380 rawtherapee
From https://github.com/Installomator/Installomator
 * branch            refs/pull/1380/head -> FETCH_HEAD
Already up to date.
2024-03-11 23:10:32 : REQ   : rawtherapee : ################## Start Installomator v. 10.6beta, date 2024-03-11
2024-03-11 23:10:32 : INFO  : rawtherapee : ################## Version: 10.6beta
2024-03-11 23:10:32 : INFO  : rawtherapee : ################## Date: 2024-03-11
2024-03-11 23:10:32 : INFO  : rawtherapee : ################## rawtherapee
2024-03-11 23:10:32 : DEBUG : rawtherapee : DEBUG mode 1 enabled.
2024-03-11 23:10:33 : DEBUG : rawtherapee : name=RawTherapee
2024-03-11 23:10:33 : DEBUG : rawtherapee : appName=
2024-03-11 23:10:33 : DEBUG : rawtherapee : type=appInDmgInZip
2024-03-11 23:10:33 : DEBUG : rawtherapee : archiveName=
2024-03-11 23:10:33 : DEBUG : rawtherapee : downloadURL=https://rawtherapee.com/shared/builds/mac/RawTherapee_macOS_13.3_Universal_5.10.zip
2024-03-11 23:10:33 : DEBUG : rawtherapee : curlOptions=
2024-03-11 23:10:33 : DEBUG : rawtherapee : appNewVersion=5.10
2024-03-11 23:10:33 : DEBUG : rawtherapee : appCustomVersion function: Not defined
2024-03-11 23:10:33 : DEBUG : rawtherapee : versionKey=CFBundleShortVersionString
2024-03-11 23:10:33 : DEBUG : rawtherapee : packageID=
2024-03-11 23:10:33 : DEBUG : rawtherapee : pkgName=
2024-03-11 23:10:33 : DEBUG : rawtherapee : choiceChangesXML=
2024-03-11 23:10:33 : DEBUG : rawtherapee : expectedTeamID=5SJ86G6Q2R
2024-03-11 23:10:33 : DEBUG : rawtherapee : blockingProcesses=
2024-03-11 23:10:33 : DEBUG : rawtherapee : installerTool=
2024-03-11 23:10:33 : DEBUG : rawtherapee : CLIInstaller=
2024-03-11 23:10:33 : DEBUG : rawtherapee : CLIArguments=
2024-03-11 23:10:33 : DEBUG : rawtherapee : updateTool=
2024-03-11 23:10:33 : DEBUG : rawtherapee : updateToolArguments=
2024-03-11 23:10:33 : DEBUG : rawtherapee : updateToolRunAsCurrentUser=
2024-03-11 23:10:33 : INFO  : rawtherapee : BLOCKING_PROCESS_ACTION=tell_user
2024-03-11 23:10:33 : INFO  : rawtherapee : NOTIFY=success
2024-03-11 23:10:33 : INFO  : rawtherapee : LOGGING=DEBUG
2024-03-11 23:10:33 : INFO  : rawtherapee : LOGO=/System/Applications/App Store.app/Contents/Resources/AppIcon.icns
2024-03-11 23:10:33 : INFO  : rawtherapee : Label type: appInDmgInZip
2024-03-11 23:10:33 : INFO  : rawtherapee : archiveName: RawTherapee.zip
2024-03-11 23:10:33 : INFO  : rawtherapee : no blocking processes defined, using RawTherapee as default
2024-03-11 23:10:33 : DEBUG : rawtherapee : Changing directory to /Users/bigmacadmin/Documents/GitHub/installomator/build
2024-03-11 23:10:33 : INFO  : rawtherapee : name: RawTherapee, appName: RawTherapee.app
2024-03-11 23:10:33.308 mdfind[8200:5446723] [UserQueryParser] Loading keywords and predicates for locale "en_US"
2024-03-11 23:10:33.308 mdfind[8200:5446723] [UserQueryParser] Loading keywords and predicates for locale "en"
2024-03-11 23:10:33.373 mdfind[8200:5446723] Couldn't determine the mapping between prefab keywords and predicates.
2024-03-11 23:10:33 : WARN  : rawtherapee : No previous app found
2024-03-11 23:10:33 : WARN  : rawtherapee : could not find RawTherapee.app
2024-03-11 23:10:33 : INFO  : rawtherapee : appversion:
2024-03-11 23:10:33 : INFO  : rawtherapee : Latest version of RawTherapee is 5.10
2024-03-11 23:10:33 : REQ   : rawtherapee : Downloading https://rawtherapee.com/shared/builds/mac/RawTherapee_macOS_13.3_Universal_5.10.zip to RawTherapee.zip
2024-03-11 23:10:33 : DEBUG : rawtherapee : No Dialog connection, just download
2024-03-11 23:10:48 : DEBUG : rawtherapee : File list: -rw-r--r--  1 bigmacadmin  staff   141M Mar 11 23:10 RawTherapee.zip
2024-03-11 23:10:48 : DEBUG : rawtherapee : File type: RawTherapee.zip: Zip archive data, at least v1.0 to extract, compression method=store
2024-03-11 23:10:48 : DEBUG : rawtherapee : curl output was:
*   Trying <IP_ADDRESS>:443...
* Connected to rawtherapee.com (<IP_ADDRESS>) port 443
* ALPN: curl offers h2,http/1.1
* (304) (OUT), TLS handshake, Client hello (1):
} [320 bytes data]
*  CAfile: /etc/ssl/cert.pem
*  CApath: none
* (304) (IN), TLS handshake, Server hello (2):
{ [122 bytes data]
* (304) (IN), TLS handshake, Unknown (8):
{ [19 bytes data]
* (304) (IN), TLS handshake, Certificate (11):
{ [4250 bytes data]
* (304) (IN), TLS handshake, CERT verify (15):
{ [520 bytes data]
* (304) (IN), TLS handshake, Finished (20):
{ [36 bytes data]
* (304) (OUT), TLS handshake, Finished (20):
} [36 bytes data]
* SSL connection using TLSv1.3 / AEAD-CHACHA20-POLY1305-SHA256
* ALPN: server accepted h2
* Server certificate:
*  subject: CN=rawtherapee.com
*  start date: Feb 24 11:42:15 2024 GMT
*  expire date: May 24 11:42:14 2024 GMT
*  subjectAltName: host "rawtherapee.com" matched cert's "rawtherapee.com"
*  issuer: C=US; O=Let's Encrypt; CN=R3
*  SSL certificate verify ok.
* using HTTP/2
* [HTTP/2] [1] OPENED stream for https://rawtherapee.com/shared/builds/mac/RawTherapee_macOS_13.3_Universal_5.10.zip
* [HTTP/2] [1] [:method: GET]
* [HTTP/2] [1] [:scheme: https]
* [HTTP/2] [1] [:authority: rawtherapee.com]
* [HTTP/2] [1] [:path: /shared/builds/mac/RawTherapee_macOS_13.3_Universal_5.10.zip]
* [HTTP/2] [1] [user-agent: curl/8.4.0]
* [HTTP/2] [1] [accept: */*]
> GET /shared/builds/mac/RawTherapee_macOS_13.3_Universal_5.10.zip HTTP/2
> Host: rawtherapee.com
> User-Agent: curl/8.4.0
> Accept: */*
>
< HTTP/2 200
< server: openresty
< date: Tue, 12 Mar 2024 06:10:33 GMT
< content-type: application/zip
< content-length: 147545603
< last-modified: Sat, 17 Feb 2024 04:58:14 GMT
< etag: "8cb5e03-6118cb53ee9e5"
< accept-ranges: bytes
<
{ [3946 bytes data]
* Connection #0 to host rawtherapee.com left intact

2024-03-11 23:10:48 : DEBUG : rawtherapee : DEBUG mode 1, not checking for blocking processes
2024-03-11 23:10:48 : REQ   : rawtherapee : Installing RawTherapee
2024-03-11 23:10:48 : INFO  : rawtherapee : Unzipping RawTherapee.zip
2024-03-11 23:10:48 : INFO  : rawtherapee : found dmg: /Users/bigmacadmin/Documents/GitHub/installomator/build/RawTherapee_macOS_13.3_Universal_5.10.dmg
2024-03-11 23:10:48 : INFO  : rawtherapee : Mounting /Users/bigmacadmin/Documents/GitHub/installomator/build/RawTherapee_macOS_13.3_Universal_5.10.dmg
hdiutil: attach failed - No such file or directory
2024-03-11 23:10:48 : DEBUG : rawtherapee : DEBUG mode 1, not reopening anything
2024-03-11 23:10:48 : ERROR : rawtherapee : ERROR: Error mounting /Users/bigmacadmin/Documents/GitHub/installomator/build/RawTherapee_macOS_13.3_Universal_5.10.dmg error:
Log

2024-03-11 23:10:49 : REQ   : rawtherapee : ################## End Installomator, exit code 3

something went wrong, stopping here

It cannot mount the DMG, as it has a long license to read. Not really sure how to fix this.

@Theile We pipe yes to the mount command that should account for accepting license/TOS. https://github.com/Installomator/Installomator/blob/main/fragments/functions.sh#L597 Should this label be closed out or can it be updated?


[improvement] add per session view

Hi
In real time view, you can click on "visitor" which means database "per session id" related traffic.
Can we imagine a new ui-view for a given session id ?

It would focus on page view and events for a given session
the goal is to analyse/understand one enduser-session UX-experience.

And in addition to go deep we could imagine an optional "get session info" url extension to configure per site:

on umami sessionid clic: does website have "get session info" url ?
(if yes) send session Id, and get target website username : enrich ui with real target website username

This would indeed be very useful to track how a user behaves in an SPA
Where is it documented what constitues a session? Hash of browser, os, mobile/desktop and browser window resolution?


Iterating and Typing JSONArray

Thank you for open sourcing this library 🙏
I'm having some issues using the results of a values-query (.query(values: "…")). Take for example the "all authors"-query from the examples: https://www.swift-linux.com/sextant/ (first one). It returns an optional JSONArray. However if I try to cast it to [String] or map/compactMap it, the compiler strikes.
How do I cast JSONArray to continue working with it?
How do I iterate over JSONArray and run JSONPath-queries on its member-json-objects?

I'm trying to do something like this:
        let likes: [LikedPost]? = body.parsed { json in
            guard let elems = json?.query(elements: "$.likes_media_likes[*]") else {
                req.logger.warning("No elems.")
                return []
            }
            let likes = elems.compactMap { elem in
                guard
                    let title = elem.query(string: "$.title"),
                    let obj = elem.query(element: "$.string_list_data[0]"),
                    let href = obj.query(string: "$.href"),
                    let timestamp = obj.query(double: "$.timestamp")
                else {
                    req.logger.warning("Invalid item: \(elem)")
                    return
                }

                return LikedPost(profile: title, href: href, timestamp: .init(timeIntervalSince1970: timestamp))
            }
        }

However I cannot get it to work. Currently the compiler fails with "Cannot convert value of type 'LikedPost' to closure result type '()'" on the "return LikedPost".

There are basically three ways you can interface with sextant.  Using query(values) will give you an [Any?] kind of like the JSONSerialization APIs; its more there for legacy. Second way is to use Swift to coerce the values by using let results: [String] = query().  Last method is to use JsonElements, which is Sextant's internal structures.
        // ===== Method 0: compatible with JSONSerialization =====
        // First method is like using JSONSerialization; returns a "JSONArray" which
        // is just a typealias for [Any?]. Least efficient method, annoying to use.
        let result0 = json.query(values: "$[*]")!

        // prints: Optional([Optional(["title": Optional("Post 1"), "timestamp": Optional(1)]), Optional(["name": Optional("Post 2"), "timestamp": Optional(2)])])
        print(result0)

        if let result0JsonData = try? JSONSerialization.data(withJSONObject: result0, options: [.sortedKeys]),
           // prints: [{"timestamp":1,"title":"Post 1"},{"title":"Post 2","timestamp":2}]
           let result0JsonString = String(data: result0JsonData, encoding: .utf8) {
            print(result0JsonString)
        }

        // ===== Method 1: coerce Codable structs =====
        // Middle efficiency, more convenient coding.
        struct LikedPost: Codable {
            let title: String
            let timestamp: Int
        }

        let result1: [LikedPost] = json.query("$[*]")!
        // prints: "Post 1"
        print(result1[0].title)

        // ===== Method 2a: JsonElement =====
        // JsonElements are the internal structures used by Sextant; they have references
        // which point to the original content so they can be used without any unnecessary
        // copying of data. Since they reference the original data, you need to ensure that
        // they get converted to a more real format for use elsewhere.

        var result2a: [LikedPost] = []
        json.query(forEach: "$[*]") { item in
            if let title = item[string: "title"],
               let timestamp = item[int: "timestamp"] {
                result2a.append(LikedPost(title: title, timestamp: timestamp))
            }
        }
        // prints: "Post 1"
        print(result2a[0].title)

        // ===== Method 2b: JsonElement =====
        // When working with JsonElements, its often advisable to use the parsed() method. This ensures
        // that the data the JsonElements reference is alive while using them. It also allows you to
        // perform multiple queries against the same json very efficiently, as you have the root
        // JsonElement directly for all queries.
        var result2b: [LikedPost] = []
        json.parsed { root in
            guard let root = root else { return }

            // Contrived: pulling the two values out using different queries just to prove the
            // point. The json is already parsed so queries inside here are very fast
            if let titles: [String] = root.query("$..title"),
               let timestamps: [Int] = root.query("$..timestamp"),
               titles.count == timestamps.count {
                for idx in 0..<titles.count {
                    result2b.append(LikedPost(title: titles[idx], timestamp: timestamps[idx]))
                }
            }
        }
        // prints: "Post 1"
        print(result2b[0].title)

Awesome, thank you @KittyMac - this is great! I used Method 0 until now and that was super fast already imo. I'll try the other methods 2a/b now as then seem the nicest.
I think the main stumble point for me was that I cannot use .map() between Sextant types and Swift types but should use forEach and appending instead. .map() seems to always return [()].


Adding macports usage.

Adding a section to the readme for those who use macports as their OSX package manager. Also added info about LIBRARY_PATH so that users are less likely to have to hunt through the issues.

Great stuff :)


Web validation: Fix tests, add support for slices

Working on a feature request where I thought I'd need slices, but, as that no longer appears to be the case, submitting a pull request with no intent to use. It has tests so isn't all bad. 😄
I fixed the other unit tests along the way as they were simple enough.

I should probably change to a proper validation library at some point, but thanks


User Experience Design

:clipboard: Description
I popped into shields.io recently to make a badge and at first glance, I had zero idea what I was looking at. It was my first time using this service. While the site is technical and straight to the waffles, I felt there could be an improvement on the frontend.
🏄🏽♀ I scanned through past issues (research, I believe) to see if previous concerns have been earlier raised. I was able to identify a few discussions that hinge on enhancing the current interface.
Let me list a tiny number of them here:

https://github.com/badges/shields/pull/1163#issue-265025516
https://github.com/badges/shields/issues/843#issue-196938870
https://github.com/badges/shields/issues/2496#issue-389114635
https://github.com/badges/shields/issues/777#issue-172880880
https://github.com/badges/shields/issues/7176#issue-1030683469
https://github.com/badges/shields/discussions/7129#discussion-3616401
https://github.com/badges/shields/issues/1899#issue-349718225
https://github.com/badges/shields/pull/1070#issue-255100790

User Research
I would love to contribute to shields by improving the user experience. This will involve:

Usability Audit
User research
Ultimately, a redesign of the interface

Glad to hear you're interested in helping, that's fantastic news! As you've probably gathered from discussions and issues you've read, it's a widely shared opinion and goal amongst the maintainer team and all signs point to the community feeling the same. The biggest roadblock to making significant progress really has been the dearth of extra bandwidth amongst ourselves, and a lack of volunteers (so far) from the community to really push on the bigger pieces, so any assistance is greatly appreciated.
While I do absolutely agree that a refresh of the site is needed, I also want to stress the importance of keeping things framed in terms of our goals and audience.
At the end of the day, our service is functionally a static content generator/provider/host that's targeted explicitly at developers/technologists. The Shields.io site has largely, in my opinion, simply provided a lightweight catalog to our service designed to help our users figure out the url path for their desired static content so that they can then embed it somewhere (e.g. a markdown file).
I mention this because IMO it's very important for our site/catalog to remain focused on that user persona with a clean, simple experience. I wouldn't want to see a Shields.io site try to cater to other personas and get too noisy and busy; I don't think we want to be in the markdown-101 business, so I don't think our site needs to explain basics like adding images to markdown files (even though we do get those types of questions from time to time).
I'd also really like to avoid an unnecessarily complex or feature rich frontend that increases complexity in terms of things like source code management and dependency tree complexities (the latter is already problem enough)

Hey @calebcartwright! Massive thanks for hitting this stuff right on the head. I really do not know what to say right now.
But hear this, from your response around personas I can say you've got a Blazin knowledge about UX design and I'm glad this conversation would be fruitful.
I dig your concerns about keeping the project lean and tight. No unnecessary complexity. Well noted.
Markdown-101 business 🤣
I noticed in #1899 someone started on the redesign, are these the reasons why it wasn't implemented?

I think we can all acknowledge the front page of shields.io is in need of improvement. One of the reasons nothing is happening on this is time, but also the skillset of the core team is orientated towards back end dev and devops expertise. I don't think its controversial to say there is not anyone in the group of active maintainers who is a great frontend dev or designer.
In general I think you've done a pretty good job of picking out a selection of relevant issues but it is probably worthwhile a bit more of a trawl through https://github.com/badges/shields/issues?q=is%3Aissue+is%3Aopen+label%3Afrontend there are a variety of smaller issues with that tag which highlight paper cuts and usability problems with the current design and information architecture. From a user research perspective there is already something of a body of evidence there.

I noticed in #1899 someone started on the redesign, are these the reasons why it wasn't implemented?

As you've noted, there have been a number of times where someone has popped up and offered to do some design work and it hasn't really come to fruition. Its difficult to say exactly why this is. I think there are several possible things that might have contributed:

It looks simple because it is one page, but there is more stuff going on in the frontend than it probably first appears. Its a deceptively large job.
The current frontend code is not super-easy to work on. We've put a lot of work into the docs/code structure/dev experience around contributing service badges as this is by far the most common code contribution and pretty much always the entry point for a new contributor, but it is not really there for the frontend. Its harder to get started.
Although everyone on the core team agrees the frontend needs improvement, none of us really have a clear vision of what an improved version looks like so maybe we aren't able to offer great guidance/feedback.

Happy to try and alleviate some of those as far as possible.

I'd also really like to avoid an unnecessarily complex or feature rich frontend that increases complexity in terms of things like source code management and dependency tree complexities (the latter is already problem enough)

Yes. The current frontend is trapped in a gatsby/typescript hellscape. In general, I think it is probably sensible to approach this design-first rather than tech-first but when(if) we get to the stage of implementing a design, there is an opportunity to review and simplify the tech at the same time, approaching it from the angle of trying to keep the maintenance overhead low.

I don't think its controversial to say there is not anyone in the group of active maintainers who is a great frontend dev or designer.

I won't speak for anyone else, but readily confirm I'm a quite terrible frontend dev :rofl:

@chris48s
I gotta say this, I love the way you've articulated the throbbing issues and things-to-take-note-of.

I don't think it's controversial to say there is not anyone in the group of active maintainers who is a great frontend dev or designer.

🤣 About the designer part, that's not new to open source (I think). It's a common itch I've begun to see most open source projects are having a tough day scratching. That's why I'm advocating for designers into open source and stuff. I won't mind filling those shoes if they are ready to be filled. I started my product design journey recently and would love an opportunity to work on something that literally saves lives...or could

The current frontend is trapped in a gatsby/typescript hellscape.

Ouuuf! That's hot.

In general, I think it is probably sensible to approach this design-first rather than tech-first but when(if) we get to the stage of implementing a design, there is an opportunity to review and simplify the tech at the same time, approaching it from the angle of trying to keep the maintenance overhead low.

Keywords: ** design-first rather than tech-first**
I'm with you.
Actionable steps from here; I think I'd just do that trawling you spoke of and attempt to condense all that information into good stuff that we can actually use (now, or later)

📯 And It begins. Artemis.
Here is a decision I've taken after a brief moment of indecision - tracking the whole process with Miro. As this progressively unfolds, I'd greatly appreciate inputs and feedback. Please request for edit permissions on Miro.

Hey, just felt like I should chime in for my part here. I’ve had very minimal time to contribute to Shields over the last couple years. (Basically I try to show up for our ops meeting!) However I am interested in resuming work on the Shields frontend.
FWIW I work on frontends in my professional life, and have good skills in UX and wireframing, though not visual design.  I’ve pitched it to a few friends who were looking to work with on their design portfolios. I’ve tried to engage with outside contributors on it too, though agree with Chris that it’s generally been hard to connect.

It looks simple because it is one page, but there is more stuff going on in the frontend than it probably first appears. Its a deceptively large job.

Agree with this.

The current frontend code is not super-easy to work on. We've put a lot of work into the docs/code structure/dev experience around contributing service badges as this is by far the most common code contribution and pretty much always the entry point for a new contributor, but it is not really there for the frontend. Its harder to get started.

For a mid- to senior-level frontend developer, I don’t think the stack would be a major barrier to entry. React is the most popular frontend framework and TypeScript is very popular too. (I do think we should replace Gatsby… though probably all these questions about the frontend stack are for a separate discussion.)

Although everyone on the core team agrees the frontend needs improvement, none of us really have a clear vision of what an improved version looks like so maybe we aren't able to offer great guidance/feedback.

I’d frame this a little differently by saying we don’t have a workable design process, where people can propose changes and get approval on them prior to code-review time. IMO this lack of process, even more than any technical barrier, make it hard for an experienced frontend developer to plug in without a lot of existing context on Shields. In addition, being effective at making UX changes requires a great deal of collaborative bandwidth with the core team. (And I have had almost no time.)
In that light, I think there are three main methods of getting into this work:

Focus on incremental usability improvements.

A developer could start on one of the open frontend issues.
A designer could, as @kohasummons suggested, conduct a UX audit. This could involve identifying new issues and ranking them alongside the existing issues which have been discussed.

Create a process for proposing and reviewing designs for the Shields team and other community members to implement. Initially these probably should be small and focused, and then we can take on larger proposals over time.
Independently from the Shields team + site, create new, independent frontends which consume the Shields badge data. Through these experiments, play around with new ideas and get feedback from the community about what they like.

@kohasummons You mentioned starting with a UX audit and I think that's a great idea. What do you think about starting your work there? You could identify what you think are the biggest UX issues, combine your list with the open frontend issues/bugs/suggestions, and then try to reason about the priorities.
Honestly what you wrote in your top post:

… at first glance, I had zero idea what I was looking at. It was my first time using this service.

… might reasonably be described as the biggest problem, at least for a certain user persona.
(Have requested edit access on Miro)

I also meant to add that while I'm :100: open to using any additional tools for the truly visual aspects of this, I've got some reservations about anything that bifurcates the enumeration and status tracking of work to a different tool.
My preference would be to keep work in GitHub as much as possible

What if the landing page was just this :wink:

So I hear you want to make a badge?

For a mid- to senior-level frontend developer, I don’t think the stack would be a major barrier to entry. React is the most popular frontend framework and TypeScript is very popular too.

I think you're right that contributing isn't difficult because we are using some exotic or unusual tech. I was thinking more about the stuff around that like docs and code structure. The onboarding experience if you want to contribute a service badge is pretty good IMO. For the frontend there is a lot less to go on.

https://www.figma.com/file/6w9sI3u2LPm4XlNhJ2axtW/Shields.io?node-id=501%3A8601
@chris48s @paulmelnikow @calebcartwright @znarf, You could poke around in explorations to see where I'm taking this. 🤾🏽

Could we have a design channel on the discord?

Could we have a design channel on the discord?

done

Hi. Thanks for continuing to work on this. I have seen it. I did have a look at the mockups and have scribbled some quick notes.
I haven't managed to find the time to write that up into something intelligible. In general I'm struggling to find loads of time for this project/open source as a whole right now. I'm going to try and slot in some time for it in but it will probably be next week.

Hello. That would be awesome. Can't wait to hear what you think already. There are a few things i noticed are common among other open source projects like sponsor tiers for example. I don't know if we even have a structure for stuffs like that. So yes, what ever feedback you can offer now would greatly make my decision making process less painful.

There are a few things i noticed are common among other open source projects like sponsor tiers for example. I don't know if we even have a structure for stuffs like that.

Let me know if I've misunderstood you, but we are actually already set up on OpenCollective (https://opencollective.com/shields) for people to make donations.
The donations we receive through OpenCollective mostly cover the operational/infrastructure costs of running the service (more or less), but that's really separate from maintainer bandwidth. Even if any of us wanted Shields to be our day job that paid the bills we need to live, the level of donations is nowhere close to what it would need to be for even one person to get by :smile: Other programs, like GitHub Sponsors, don't fundamentally change that calculus, and can actually add a lot of overhead and work for the person due to tax liabilities in many jurisdictions.
As such, like most projects, we're all operating on a strictly volunteer basis when and as we can, and the amount of time just ebbs and flows depending on work, other things going in life, etc. That's part of why we haven't made much progress on this front, and why it's great when folks like yourself are eager to pitch in and assist!

@calebcartwright No. No. You haven't.  They have this sort of "sponsor tiers". So there is Bronze sponsor, gold sponsor. I feel this are incentives to motivate donation.
Check it out: https://opencollective.com/vuejs
Just saying. 🛸

No. No. You haven't.  They have this sort of "sponsor tiers". So there is Bronze sponsor, gold sponsor. I feel this are incentives to motivate donation.

Thanks for clarifying and sharing. While funding and sponsorship just about always make for interesting dialog, I feel like it's also fair to say that it's orthogonal to the thrust of the issue, and as such I'd like to suggest that we keep discussion focused on the topic of this issue around the frontend design.
(I realize you're exploring incorporating sponsorship information into the UI, but I still think the specifics are off topic)

https://www.figma.com/file/6w9sI3u2LPm4XlNhJ2axtW/Shields.io?node-id=501%3A8601
@chris48s @paulmelnikow @calebcartwright @znarf, You could poke around in explorations to see where I'm taking this. 🤾🏽

It took me a while to realize there were other wireframes scattered around :sweat_smile:
Aesthetically it looks nice, and very well done! It definitely evokes more of a "modern" feel for me.
Feel free to correct me, but I interpret the relative placement of pages/screens to connote a progressive call/path a user would take. One concern I'd have, if that's indeed a correct understanding, is that I feel like we're increasing the mandatory click-depth required by at least 1 in order for users to get to the area that drives 99% of our users to the site.
Fundamentally, our site is about enumerating our catalog of badges and providing aid to users that need to construct their badge url. The landing page you've proposed strikes me as more of a stereotypical product overview/branding/marketing page that I don't personally think is necessary, or at least which shouldn't be front and center. I.e. we're not trying to convince people to buy our product or service, so I don't think it's important that essentially the first thing people see is big-name projects/companies that user our service, and certainly not at the expense of adding a click (ostensibly the Create badge button) in order to utilize the core functionality of the site.
As for the badge factory page, it similarly looks really nice, but I'm curious what the experience would be like on smaller screens given the page layout? Would users end up having to do a lot of vertical scrolling (e.g. we have just about 1,000 services, or "projects" as you labeled them which would be enumerated)

At this stage, I don't think bikeshedding exact design details (colour schemes, logos, etc) is the most important thing to focus on so I'm mostly going to steer clear of that. One of the things that reviewing these wireframes is forcing us to do is really think about and express clearly what use-case we are trying to serve and the information architecture. That may be one of the most valuable things you are helping us to do.
I'll also note that I've been working on this in parallel to @calebcartwright 's comments so there's a certain amount of duplication here (good sign that these points are important). I'm just going to post it all anyway.
Shields - Home
I think the first big thing I'd challenge is dominating the home page with featured sponsors and donation asks. We do currently have a page for acknowledging and thanking sponsors. https://shields.io/community At the moment it is tucked away in the footer. We should make more of this than we currently do but it shouldn't occupy most of the real-estate on the front page. The most critical user need we want to service here is "I want to put badges in my README". Fundraising is a secondary goal. Any re-design should keep the core "product" (for want of a better word) up front and centre. I'd also assume that we will continue to as far as possible outsource keeping track of who is/isn't sponsoring us to the OpenCollective auto-generated SVGs for this in preference to taking on the maintenance of this ourselves.
Maybe the other thing we could consider devoting some real-estate to on the landing page of shields.io are:

More clearly communicating to a new user who is not already familiar with shields "what is this thing?" / "how does it work?" / "why do you care about it?".
Talking about (or at least linking to) the other things we do ( badge-maker NPM library and shields specification )

In general, I don't think using the VueJS site as a template or inspiration point is a great match. Vue JS is a library. Shields.io is (primarily) a web service. The role of the website is a bit different.
I think we can also ditch the graphics with badges overlapping text and at an angle, but as I say, I don't think this is the most important place to focus at this stage.
Shields - Badge Factory
My interpretation of your "badge factory" screenshot is that what we're looking at there is a "modal" with the inputs on it, but the badge preview on the "page" behind it? Is that right? We don't collect any stats on visitors to the site (intentionally, and we don't plan to). I suspect we skew more towards desktop than a lot of sites but I think one of the (many) problems with our existing design is the UX on mobile/tablet and if my interpretation of your design is right we'd be looking at replicating some of those problems. I think if we're looking at a redesign we should probably be moving towards something that looks like a "page" (may not actually be implemented as a page if we end up with another SPA - lets abstract that for now) for each builder rather than a modal popup. I think we'd also want to think about what any design would look like in a mobile view.

One of the leaps we've made here is assuming we will associate an icon or logo with each service family. I can't find a link to the issue, but this is something that has come up before and we've decided against it. Assume for the purposes of front-end design we won't be doing this. Probably the two most important considerations here are:

Badges should be informational rather than promotional
There is maintenance overhead associated with this which we don't want to take on.

but there are others. In general I think we should be aiming towards something that is cleaner and more visually appealing than what we currently have but which is fundamentally text based (rather than icon-heavy).
The hierarchical left-hand menu you've proposed looks like it would work ok for services (you've labelled them "Project" but we call them "services") that only include a few badges but we have over 40 different github badges and for the categories view our largest category is version badges. We have over 100 version badges. We need to think about a design that will scale well enough to allow a user to easily browse long lists here. Another common pattern is subtly different variations of the same thing with long descriptions e.g:

Docker Image Version (latest by date)
Docker Image Version (latest semver)
Docker Image Version (tag latest semver)

or

Libraries.io dependency status for latest release
Libraries.io dependency status for specific release
Libraries.io dependency status for latest release, scoped npm package
Libraries.io dependency status for specific release, scoped npm package
Libraries.io dependency status for GitHub repo

A design that allows the user to read the whole text (rather than truncating) is also going to be quite important. Maybe we could reduce the need for some of this with a better builder UI, but some of this will be unavoidable.

"show me around" is an interesting idea, but for now it is just a link title. I'd be interested in what you'd envisage that being (text description is fine - doesn't need to be a wireframe). I'd have a couple of competing ideas, but also this feels like an ambitious stretch goal. Focussing on improving what we've got in a simple and maintainable way seems like the first step rather than imaging ambitious new features.

How do you see "Docs" or "API" as being different to the badge builder? (for shields.io the URL schema is the API)

In general, the badge builder is there to assist the user in constructing a URL not hide it (remember: the target audience for this is developers, and the URL schema is the API), so we really want to keep the terminology consistent (e.g: the param is &logo= - lets not present it as "Icon" in the UI) and expose it explicitly everywhere. I'm not saying this is exactly what we should aim for, but one of the things badgen.net 's builder ( https://badgen.net/builder ) conceptually gets right IMO is that the task is very explicitly framed as helping the user to construct a URL , not abstracting it.
When it comes to logos, SimpleIcons has over 2,000 icons and is growing. A drop-down with >2,000 things in it is too many things for a drop-down. Auto-complete would be nice. MVP is leave it as text but improve the documentation here on how to find the slug.
Have you had any thoughts on how to present:

static badge
available named colours (esp semantic aliases)
endpoint badge
dynamic badges (json/xml/yaml)
globally available params?

?

Hi @calebcartwright @chris48s Thanks for weighing in on this. I read through the entire feedback and they have helped  me understand the scope of the shields service to a much greater extent. In the same way, also exposed areas where I still need clarification.
I think having a conversation about the project generally, something like a call, would help. So, If anyone would want to have that call, I would greatly appreciate it.

Speaking strictly for myself and not the others. While I'm not necessarily opposed to having a more synchronous connection down the road, I'd prefer and recommend that we try to accomplish as much as possible asynchronously, and only fall back to something like a call if absolutely necessary.
I realize and readily acknowledge that there benefits to voice and synchronous discussion, but it's also one of the most time consuming, expensive, and logistically challenging communication mediums, particularly in the distributed+volunteer context of open source. We already have a lot of difficulty just trying to get a handful of the team together due to other obligations, time zones, etc. and I'm just not convinced that doing so for this topic will be practical nor would it be the best approach at this time.
We've got this issue and a dedicated chat thread, and I'd encourage leveraging those to ask clarifying questions and bouncing ideas.

Before jumping in on rereading the comments, I see two issue here.
1. There is no separate user stories
This one is pretty good, actually.

popped into shields.io recently to make a badge and at first glance, I had zero idea what I was looking at. It was my first time using this service.

Not sure it has a solution, but for establishing metrics of success it can be useful. Like recording bounce rate, first time on page. I know in the past shields.io claimed that it is no-tracking service. Not sure if anything changed since then.
2. It's not the first redesign
I remember the page was way more simple than it is now. Evaluating at least how it looked before can also help to see establish targets for A/B experiments.

When it comes to logos, SimpleIcons has over 2,000 icons and is growing. A drop-down with >2,000 things in it is too many things for a drop-down. Auto-complete would be nice. MVP is leave it as text but improve the documentation here on how to find the slug. (@chris48s  https://github.com/badges/shields/issues/7815#issuecomment-1153272882)

We could use loading="lazy" for icons in the selector, with a search input. Like https://simpleicons.org does.
For React, we also can use https://github.com/bvaughn/react-virtualized for rendering a virtualized list.

I've been working on a proof-of-concept for a new shields frontend. There's some more links and notes on this over at https://github.com/badges/shields/issues/7982#issuecomment-1346871332 . While what I've done doesn't directly implement @kohasummons design, the conversation we've had around this has definitely fed into and influenced what I've done there.

@adsingh14 - we are tantalisingly close to deploying a new frontend (PR: https://github.com/badges/shields/pull/9014 )
Watch this space..

We have just launched a completely redesigned frontend https://shields.io/
While we did not directly implement the designs here, the work in this thread definitely helped and influenced the process. Thanks :heart:


rejectUnauthorized, requestCert, agent

Can anyone explain me what exactly these three options does?
rejectUnauthorized: false,
requestCert: true,
agent: false
Many Thanks

cert, agent, rejectUnauthorized and some others derived from https module.
requestCert documented in tls module.


Redesign prep

This PR includes several changes in anticipation of the upcoming redesign of the Drought homepage (as built and iterated in PR #270). Most of the code is new, so the merge should be fairly clean. Here are the highlights.

There are five new components in the src/components folder.

data-viz-base (This is a common module reused by several other components below.)
precipitation-levels
snowpack-levels
precipitation-levels
spei-map

There's a new GitHub Action to fetch data for the above visualizations, .github/workflows/fetch-drought-data.yml.
There are several CSS changes to support the new homepage layout in src/css/sass/data-visualizations.scss.
There's a new npm script, npm run process:spei-map:img, which does some image processing in support of the spei-map data visualization.
The Javascript bundling pipeline has been migrated from Rollup to esbuild. This change helped facilitate development of the new components within the site itself. It's also nice and fast.

There are no actual content updates here. We shouldn't expect any outwardly visible changes to the site. This PR is behind-the-scenes code prep.

One more note. I will likely follow this PR with a couple more after it's merged.

Update Design System components to latest versions.
Do a lighthouse audit.

I didn't want to mix those things up with this update.


Cocoapods

Summary
SwiftPM is nice and cool, but not everyone uses, knows or is able to use it. Knowing that the most known dependency manager for iOS projects is Cocoapods, this PR is here to adds the support (and a bit of boilerplate) of Cocoapods.
There are some requirements for this to work:

Please do NOT open the xcworkspace and do not test the project there, it's full of errors. Use only the SwiftPM part of the project to develop and test it. Remember, it's just a wrapper with the minimum configuration needed to have this release at Cocoapods.
That's the first time I add support to Cocoapods to a SwiftPM project, I am not 100% what each file does and what is necessary. If possible we will keep cutting these files down if we see that they are not needed to publish the project on Cocoapods.
Name the tag respecting the structure x.x.x instead of vx.x.x. This is a requirement from Cocoapods linter and it fails if it doesn't respect it.
After the merge of this PR, we will need to test it again locally and create a new release.

You can find the full tutorial explaining how to create a Cocoapods library here: https://guides.cocoapods.org/making/making-a-cocoapod.html

bors try


We need those system to work without importing 3d party libraries

$ pip install -r requirements.txt
Collecting napalm-base (from -r requirements.txt (line 1))
  Downloading napalm-base-0.16.0.tar.gz (232kB)
    100% |################################| 233kB 1.8MB/s
    Traceback (most recent call last):
      File "<string>", line 20, in <module>
      File "/tmp/pip-build-3bUemy/napalm-base/setup.py", line 6, in <module>
        from napalm_base.helpers import find_version
      File "napalm_base/__init__.py", line 24, in <module>
        from napalm_base.base import NetworkDriver
      File "napalm_base/base.py", line 20, in <module>
        import napalm_base.helpers
      File "napalm_base/helpers.py", line 9, in <module>
        import jinja2
    ImportError: No module named jinja2
    Complete output from command python setup.py egg_info:
    Traceback (most recent call last):

      File "<string>", line 20, in <module>

      File "/tmp/pip-build-3bUemy/napalm-base/setup.py", line 6, in <module>

        from napalm_base.helpers import find_version

      File "napalm_base/__init__.py", line 24, in <module>

        from napalm_base.base import NetworkDriver

      File "napalm_base/base.py", line 20, in <module>

        import napalm_base.helpers

      File "napalm_base/helpers.py", line 9, in <module>

        import jinja2

    ImportError: No module named jinja2

    ----------------------------------------
    Command "python setup.py egg_info" failed with error code 1 in /tmp/pip-build-3bUemy/napalm-base
The command "pip install -r requirements.txt" failed and exited with 1 during .

👍