prediction from dask 100-10000x slower than non-dask

Related to codes in: https://github.com/dmlc/xgboost/issues/7004
This is on a 1 GPU system, so cuda worker is only 1, but runs CPU in xgboost.  So only 1 process and n_jobs=1
from dask.distributed import Client
from dask_cuda import LocalCUDACluster
from dask import dataframe as dd
import xgboost as xgb
import numpy as np
import random
import time

def main(client):

    # data
    import pickle
    params, X, y = pickle.load(open("bad_dask.pkl", "rb"))
    for k in ["accuracy", "debug_verbose", "disable_gpus", "early_stopping_rounds", "early_stopping_threshold",
              "ensemble_level", "interpretability", "lossguide", "model_class_name",
              "monotonicity_constraints", "score_f_name", "silent", "time_tolerance",
              "train_shape", "valid_shape"]:
        del params[k]
    print(params)
    print(X.shape)
    target = "__TARGET__"
    X[target] = y
    num_preds = 5

    # dask
    np.random.seed(1234)
    random.seed(1234)
    dask_df = dd.from_pandas(X, chunksize=5000).persist()
    X = X.drop(target, axis=1)
    yd = dask_df[target]
    Xd = dask_df.drop(target, axis=1)

    dtrain = xgb.dask.DaskDMatrix(client, Xd, yd)
    output = xgb.dask.train(client,
                            params,
                            dtrain,
                            num_boost_round=params['n_estimators'])
    ntree_limit = output['booster'].best_ntree_limit
    preds = xgb.dask.predict(client, output, dtrain, iteration_range=(0, ntree_limit))

    print(preds.compute()[0:num_preds])

    ntree_limit = output['booster'].best_ntree_limit
    print("dask ntree_limit: %s" % ntree_limit)
    t0 = time.time()
    for upper_range in range(0, ntree_limit + 1):
        print("dask iteration_range: (0, %d)" % upper_range)
        preds = xgb.dask.predict(client, output, dtrain, iteration_range=(0, upper_range))
        print(preds[0:num_preds].compute())
    print("dask duration: %g" % (time.time() - t0))

    # non-dask
    np.random.seed(1234)
    random.seed(1234)
    dtrain = xgb.DMatrix(X, y)
    output = xgb.train(params,
                            dtrain,
                            num_boost_round=params['n_estimators'])
    preds = output.predict(dtrain)
    print(preds[0:num_preds])

    ntree_limit = output.best_ntree_limit
    print("non-dask ntree_limit: %s" % ntree_limit)
    t0 = time.time()
    for upper_range in range(0, ntree_limit + 1):
        print("non-dask iteration_range: (0, %d)" % upper_range)
        preds = output.predict(dtrain, iteration_range=(0, upper_range))
        print(preds[0:num_preds])
    print("non-dask duration: %g" % (time.time() - t0))

    model = xgb.XGBRegressor(**params)
    model.fit(X, y)
    preds = model.predict(X)
    print(preds[0:num_preds])

    ntree_limit = model.best_ntree_limit
    print("ntree_limit: %s" % ntree_limit)
    t0 = time.time()
    for upper_range in range(0, ntree_limit + 1):
        print("iteration_range: (0, %d)" % upper_range)
        preds = model.predict(X, iteration_range=(0, upper_range))
        print(preds[0:num_preds])
    print("non-dask skl duration: %g" % (time.time() - t0))

if __name__ == '__main__':
    with LocalCUDACluster() as cluster:
        with Client(cluster) as client:
            main(client)

gives:
dask duration: 1.64682
non-dask duration: 0.00677085
non-dask skl duration: 0.0289192

Seems bad that dask is about 100x slower.  @trivialfis

Here's your version of code:
import time

from dask.distributed import Client
from dask_cuda import LocalCUDACluster
from dask import dataframe as dd
import xgboost as xgb
import numpy as np
import random

def main(client):
    # data
    import pickle

    params, X, y = pickle.load(open("bad_dask.pkl", "rb"))
    for k in [
        "accuracy",
        "debug_verbose",
        "disable_gpus",
        "early_stopping_rounds",
        "early_stopping_threshold",
        "ensemble_level",
        "interpretability",
        "lossguide",
        "model_class_name",
        "monotonicity_constraints",
        "score_f_name",
        "silent",
        "time_tolerance",
        "train_shape",
        "valid_shape",
    ]:
        del params[k]

    print(X.shape)
    target = "__TARGET__"
    X[target] = y

    # dask
    np.random.seed(1234)
    random.seed(1234)

    dask_df = dd.from_pandas(X, chunksize=5000).persist()
    X = X.drop(target, axis=1)
    Xd = dask_df.drop(target, axis=1)

    dtrain = xgb.DMatrix(X, y)
    output = xgb.train(params, dtrain, num_boost_round=params["n_estimators"])

    model = xgb.XGBRegressor(**params)
    model.fit(X, y)

    times = [0] * 3
    for i in range(0, params["n_estimators"]):
        iter_range = (0, i)

        t0 = time.time()
        dmatrix_preds = output.predict(dtrain, iteration_range=iter_range)
        times[0] += time.time() - t0
        t0 = time.time()
        skl_preds = model.predict(X, iteration_range=iter_range)
        times[1] += time.time() - t0
        t0 = time.time()
        dask_preds = xgb.dask.predict(client, output, Xd, iteration_range=iter_range).compute().values
        times[2] += time.time() - t0

        print(dmatrix_preds[0:5])
        print(skl_preds[0:5])
        print(dask_preds[0:5])

        np.testing.assert_allclose(dmatrix_preds, skl_preds)
        np.testing.assert_allclose(dmatrix_preds, dask_preds)

    print(times)

if __name__ == "__main__":
    with LocalCUDACluster() as cluster:
        with Client(cluster) as client:
            main(client)

[0.008350849151611328, 0.04452371597290039, 1.1518757343292236]

Could you please take a look at: https://xgboost.readthedocs.io/en/latest/tutorials/dask.html#running-prediction and see if it helps?
2 things worth noticing in the referenced document:

Consider using inplace_predict or using predict without DaskDMatrix.
Consider scattering the model first.

Longer explanasion is in the doc.

Ok, just seems a bit extreme.
E.g. this version:
import time

from dask.distributed import Client, LocalCluster
from dask_cuda import LocalCUDACluster
from dask import dataframe as dd
import xgboost as xgb
import numpy as np
import random

def main(client):
    # data
    import pickle

    params, X, y = pickle.load(open("bad_dask.pkl", "rb"))
    for k in [
        "accuracy",
        "debug_verbose",
        "disable_gpus",
        "early_stopping_rounds",
        "early_stopping_threshold",
        "ensemble_level",
        "interpretability",
        "lossguide",
        "model_class_name",
        "monotonicity_constraints",
        "score_f_name",
        "silent",
        "time_tolerance",
        "train_shape",
        "valid_shape",
    ]:
        del params[k]

    # ensure n_jobs=1
    params['n_jobs'] = 1

    print(X.shape)
    target = "__TARGET__"
    X[target] = y

    # dask
    np.random.seed(1234)
    random.seed(1234)

    do_dask = True
    do_sklearn = False
    do_native = False

    if do_dask:
        dask_df = dd.from_pandas(X, chunksize=1).persist()
        yd = dask_df[target]
        Xd = dask_df.drop(target, axis=1)
        dtraind = xgb.dask.DaskDMatrix(client, Xd, yd)
        outputd = xgb.dask.train(client, params, dtraind, num_boost_round=params["n_estimators"])

    if do_sklearn or do_native:
        X = X.drop(target, axis=1)

    if do_native:
        dtrain = xgb.DMatrix(X, y)
        output = xgb.train(params, dtrain, num_boost_round=params["n_estimators"])

    if do_sklearn:
        model = xgb.XGBRegressor(**params)
        model.fit(X, y)

    times = [0] * 3
    for i in range(0, params["n_estimators"]):
        iter_range = (0, i)

        if do_native:
            t0 = time.time()
            dmatrix_preds = output.predict(dtrain, iteration_range=iter_range)
            times[0] += time.time() - t0
            print(dmatrix_preds[0:5])

        if do_sklearn:
            t0 = time.time()
            skl_preds = model.predict(X, iteration_range=iter_range)
            times[1] += time.time() - t0
            print(skl_preds[0:5])

        if do_dask:
            t0 = time.time()
            dask_preds = xgb.dask.predict(client, outputd, Xd, iteration_range=iter_range).compute().values
            times[2] += time.time() - t0
            print(dask_preds[0:5])

        if do_dask and do_sklearn and do_native:
            np.testing.assert_allclose(dmatrix_preds, skl_preds)
            np.testing.assert_allclose(dmatrix_preds, dask_preds)

    print(times)

if __name__ == "__main__":
    with LocalCluster(n_workers=1, processes=True) as cluster:
        with Client(cluster) as client:
            main(client)

is just ridiculously slow.

Could you please take a look at: https://xgboost.readthedocs.io/en/latest/tutorials/dask.html#running-prediction and see if it helps?
2 things worth noticing in the referenced document:

Consider using inplace_predict or using predict without DaskDMatrix.
Consider scattering the model first.

Longer explanation is in the doc.

And note that in this code I'm not using dmatrix for prediction.  I'm just using dask dataframe.

Setting chunksize to 1 means you have n_samples partitions, than it means the predict function in xgboost is called n_samples times by dask.  It's gonna be slow.

If anything, the docs suggest that higher chunksize could be slower.

The doc suggests it's a balance between throughput and latency.

There is nothing to scatter. There is just one worker and 1 core in 1 process doing n_jobs=1.

The model is still pickled from the client process to the worker process.  You have specified the number of the worker process, there are still seperated client and schedule process who are communicating.

Setting chunksize to 1 means you have n_samples partitions, than it means the predict function in xgboost is called n_samples times by dask. It's gonna be slow.

If anything, the docs suggest that higher chunksize could be slower.

The doc suggests it's a balance between throughput and latency by changing chunk size.

There is nothing to scatter. There is just one worker and 1 core in 1 process doing n_jobs=1.

The model is still pickled from the client process to the worker process. You have specified the number of the worker process, there are still separated client and schedule process who are communicating. So, pre-scatter the model.
After pre-scattering the model, the time drops from 2.33 to 1.40. That's still much slower. But if you amortize it to 30 iterations, there are 0.078 seconds of overhead for each call. It seems reasonable to me since at the end of each prediction you are also gathering the result back to client process.

I don't see that change from pre-scatter as significant, it's within error of re-running over and over probably.
I'm basically trying to understand why a single threaded case is so much slower, regardless of chunk size.  It can be 10000X slower for chunksize=1 and 100X for chunk sizes approaching the maximum of 1 partition.
The model is super small, so pickle times should be small.  This can be seen here:
import time
import xgboost as xgb
import numpy as np
import random

def main():
    # data
    import pickle

    params, X, y = pickle.load(open("bad_dask.pkl", "rb"))
    for k in [
        "accuracy",
        "debug_verbose",
        "disable_gpus",
        "early_stopping_rounds",
        "early_stopping_threshold",
        "ensemble_level",
        "interpretability",
        "lossguide",
        "model_class_name",
        "monotonicity_constraints",
        "score_f_name",
        "silent",
        "time_tolerance",
        "train_shape",
        "valid_shape",
    ]:
        del params[k]

    # ensure n_jobs=1
    params['n_jobs'] = 1

    # ensure no sampling
    params['colsample_bytree'] = 1
    params['subsample'] = 1

    print(params)

    print(X.shape)
    target = "__TARGET__"
    X[target] = y

    # dask
    np.random.seed(1234)
    random.seed(1234)

    X = X.drop(target, axis=1)

    dtrain = xgb.DMatrix(X, y)
    output = xgb.train(params, dtrain, num_boost_round=params["n_estimators"])

    t0 = time.time()
    num = 1000
    for i in range(0, num):
        pickle.dumps(output)
    print("time per pickle: %s" % ( (time.time() - t0)/num))

if __name__ == "__main__":
    main()

gives:
time per pickle: 0.0025900232791900633

And the data result for each prediction row is super tiny and will take much much less time than that.
My expectation is the is an inefficiency in the use of dask, which is why I persist in questions/examples.

In the scheduler console, every predict looks identical no matter the iteration_range.  So I don't think the model state is a problem, unless for every row prediction the model is being pickled, which is what I'm worried about.

There seems to be always 1000 from_pandas and 2000 get_items.  I understand with chunksize=1 why from_pandas is 1000 for the X with 1000 rows.  I'm not sure why 2 gets for every row, but ok.
The above is with scatter of model  and gather of the predictions.
import time

from dask.distributed import Client, LocalCluster
from dask_cuda import LocalCUDACluster
from dask import dataframe as dd
import xgboost as xgb
import numpy as np
import random

def main(client):
    # data
    import pickle

    params, X, y = pickle.load(open("bad_dask.pkl", "rb"))
    for k in [
        "accuracy",
        "debug_verbose",
        "disable_gpus",
        "early_stopping_rounds",
        "early_stopping_threshold",
        "ensemble_level",
        "interpretability",
        "lossguide",
        "model_class_name",
        "monotonicity_constraints",
        "score_f_name",
        "silent",
        "time_tolerance",
        "train_shape",
        "valid_shape",
    ]:
        del params[k]

    # ensure n_jobs=1
    params['n_jobs'] = 1

    # ensure no sampling
    params['colsample_bytree'] = 1
    params['subsample'] = 1

    print(params)

    print(X.shape)
    target = "__TARGET__"
    X[target] = y

    # dask
    np.random.seed(1234)
    random.seed(1234)

    do_dask = True
    do_sklearn = False
    do_native = False

    if do_dask:
        chunksize = 1
        dask_df = dd.from_pandas(X, chunksize=chunksize).persist()
        yd = dask_df[target]
        Xd = dask_df.drop(target, axis=1)
        dtraind = xgb.dask.DaskDMatrix(client, Xd, yd)
        outputd = xgb.dask.train(client, params, dtraind, num_boost_round=params["n_estimators"])
        outputd = client.scatter(outputd['booster'], broadcast=True)

    if do_sklearn or do_native:
        X = X.drop(target, axis=1)

    if do_native:
        dtrain = xgb.DMatrix(X, y)
        output = xgb.train(params, dtrain, num_boost_round=params["n_estimators"])

    if do_sklearn:
        model = xgb.XGBRegressor(**params)
        model.fit(X, y)

    times = [0] * 3
    preds = []
    for i in range(0, params["n_estimators"]):
        iter_range = (0, i)

        if do_native:
            t0 = time.time()
            dmatrix_preds = output.predict(dtrain, iteration_range=iter_range)
            times[0] += time.time() - t0
            print(dmatrix_preds[0:5])

        if do_sklearn:
            t0 = time.time()
            skl_preds = model.predict(X, iteration_range=iter_range)
            times[1] += time.time() - t0
            print(skl_preds[0:5])

        if do_dask:
            t0 = time.time()
            dask_preds = xgb.dask.predict(client, outputd, Xd, iteration_range=iter_range) #.compute().values
            preds.append(dask_preds)
            times[2] += time.time() - t0
            #print(dask_preds[0:5])

        if do_dask and do_sklearn and do_native:
            np.testing.assert_allclose(dmatrix_preds, skl_preds)
            np.testing.assert_allclose(dmatrix_preds, dask_preds)

    print(times)

    preds = client.gather(preds)
    preds = [x.compute().values for x in preds]
    print(preds)

if __name__ == "__main__":
    with LocalCluster(n_workers=1, processes=True) as cluster:
    #with LocalCluster() as cluster:
        with Client(cluster) as client:
            main(client)

If you don't pre-scatter the model, XGBoost will do it for you inside the predict function: https://github.com/dmlc/xgboost/blob/816b789bf06864b37d3b2eacd4ce5d75fa3865cf/python-package/xgboost/dask.py#L1131  But it has to be done for every call to predict, so it's best if users can do it themselves.
You are partly right that it's 1 model pickle per chunk, but since we have scattered the model and uses the future object, so it's actually 1 pickled model per worker.  You can try removing that function, set chunk size to 1 then see the runtime. ;-)  I added a print to booster.__getstate__, with 100 partitions and 2 workers, the function is called 2 times during the prediction run.  You can verify it by yourself.

time per pickle of prediction row: 5.296945571899414e-06

You haven't included the time for loading the pickle.
I modified your script (attached at the end), it uses 1 worker, 1 partition (with 1000 samples in that partition, which is actually much smaller than in practice).  Also, the result is waited and remained in worker instead of being pulled back to the client, the runtime is listed below with 3 runs to obtain the average:
Dask prediction 1.1045029163360596
Native: 0.03764605522155762

Again, amortizing it, the overhead is 0.0368 seconds per call.  Which is reasonable to me.  Sure you can still argue that it's 30x slower, but it's just not fair to use such a small dataset.  For a reasonable dataset like HIGGS that is worth distributing the computation, here is the runtime with 16 partitions and 2 workers (I have 2 GPUs) running with 16 threads (I also printed out the call to pickle function __getstate__ and link to dashboard):
http://<IP_ADDRESS>:8787/status
__getstate__
__setstate__
__getstate__
__setstate__
Dask prediction 34.777162075042725
Native: 33.9640634059906

So, it's less than 1 second overhead, which is actually within the variance of multiple runs.

The script I used with your dataset
reproduce.py.txt

The script for predicting on HIGGS
higgs.py.txt

Thanks for the report.  My expectation is that in the limit of 1 prediction per row, 1 worker, 1 thread, n_jobs=1, and 1 partition, we should be able to identify the source of all timing leading to the total time of a predict call on a frame.  That should be closest to just native prediction.
Of course, if each worker has alot of work to do, like HIGGS with alot of data per worker's predict, then any other overheads or extra pickle times won't be noticeable.  But this doesn't mean that we understand the reason why things are slow in other limits.
I shouldn't have to include the pickle load time in each predict call since my version of the code already scattered the model, so the worker has the unpickled model and should just go through the rows to do the prediction.  So I can't quite follow your reasoning there.  There would be a single model unpickle, but that is small compared to the total time of 30ms over all partitions.
The fact that even with 1 partition things are 30x slower means something serious is going on with either dask or xgboost.  In practice you  might argue noone cares, but often times one is not just doing a massive predict on a giant frame.  One might be doing feature selection or other things where rapids sequence of smaller predictions are required.
So my hope is we can understand the actual reason why each one of those dask predict sequences takes about 30ms in this case, leading to 30-100x slower than native.  And, similarly, why is chunksize=1 10000x slower.  If we scattered the model, it should have already unpickled on the worker and just run through the predictions just like in native case at native speed.  The pickling of the predictions itself and sending that is negligible like I showed.

The fact that even with 1 partition things are 30x slower means something serious is going on with either dask or xgboost.

I prefer 0.0368 second overhead than 30x slower in this case.

And, similarly, why is chunksize=1 10000x slower.

I can come up with a few reasons.

From dask interface we need to transfer the column names and data shape for each partition, so here are the communication overhead. n_samples.  ;-)
For each partition, since you are not using inplace_predict, we need to create DMatrix, which means we have created n_samples DMatrix instances.
We are comparing the speed to Python iteration with omp dispatch, that's easily 100x diffierence.

Dask is really not built for low latency.

Dask is really not built for low latency.

Latency is not an issue for the 1 partition case, which is still 30-100x slower.

Is there a reason that xgboost does not scatter the model object for the user? I just opened a PR to do so in dask-ml's ParallelPostFit class and was looking around here to see if it made sense to do something similar with xgboost.dask.
Dask is also very inefficient if you don't pre-scatter, as it will send one copy of the model object for each partition in the dataset, and it all hangs out in scheduler memory, which can occupy a lot of memory even for small models and small datasets (npartitions * model size)

Xgboost does scatter the model.  You can pre scatter it so consecutive calls can use one scattered model


tfsec : Update version 1.0.2 and add autorefresh

What type of Apps related PR is this? (check all applicable)

[ ] ✨ Add Manifest
[x] ♻ Refactor
[x] 🐛 Bug Fix
[x] 👷 Optimization
[ ] 🚩 Other

Description

tfsec has resolve its issue with the windows availability, so we added the autorefresh functionnality
Related Tickets & Documents

None
Did you verify the followings

Add/Modify a description in the manifest?

[ ] 👍 yes
[ ] 🙋 yes, but I need help
[x] 🙅 no, because they aren't needed

Check URLs?

[ ] 👍 yes
[ ] 🙋 yes, but I need help
[x] 🙅 no, because they aren't needed

Update to the latest version?

[x] 👍 yes
[ ] 🙋 yes, but I need help
[ ] 🙅 no, because they aren't needed

Update Hashes?

[x] 👍 yes
[ ] 🙋 yes, but I need help
[ ] 🙅 no, because they aren't needed

Format the json manifest?

[ ] 👍 yes
[ ] 🙋 yes, but I need help
[x] 🙅 no, because they aren't needed

Did you identify extra steps
Notes in the manifest?

[ ] 👍 yes
[ ] 🙋 yes, but I need help
[x] 🙅 no, because they aren't needed

Application Dependencies?

[ ] 👍 yes
[ ] 🙋 yes, but I need help
[x] 🙅 no, because they aren't needed

Application Suggestions?

[ ] 👍 yes
[ ] 🙋 yes, but I need help
[x] 🙅 no, because they aren't needed

Pre-installation steps to perform?

[ ] 👍 yes
[ ] 🙋 yes, but I need help
[x] 🙅 no, because they aren't needed

Post-installation steps to perform?

[ ] 👍 yes
[ ] 🙋 yes, but I need help
[x] 🙅 no, because they aren't needed

File persistence to perform?

[ ] 👍 yes
[ ] 🙋 yes, but I need help
[x] 🙅 no, because they aren't needed

[optional] What gif/emoji best describes this PR or how it makes you feel?

https://gitmoji.dev/ => :alien: :zap: :sparkles: :recycle: :adhesive_bandage:

All changes look good.
Wait for review from human collaborators.
tfsec

[x] Description
[x] License
[x] Hashes
[x] Checkver
[x] Autoupdate
[x] Autoupdate Hash Extraction


Play/Pause Media shortcut remap not working

Microsoft PowerToys version
0.69.1
Installation method
PowerToys auto-update
Running as admin
None
Area(s) with issue?
Keyboard Manager
Steps to reproduce
Configure any shortcut an assign it to "Play/Pause Media".
✔ Expected Behavior
It should pause and resume the current media.
❌ Actual Behavior
It works sometimes, usually after restarting. But after using the PC for a while it stops working.
Other Software
No response

Not able to repro this, going to close. Let us know if you see this again!


Replace confusing links from CONTRIBUTING.md with short instruction

Objective
CONTRIBUTING.md contains links to pages that are restricted to Bevy Engine Org members.
First time contributors who read CONTRIBUTING.md will end up on a confusing 404 page if they try to follow the link.
Relevant discussion: #4365
Solution
Replace links with directions to the Triage Team page.
Note
I'm not sure if assign themselves as a member is accurate. I think that is what automatically request membership was referring to, but i can't check for myself :wink:

CI failure seems unrelated to my PR.
Surprisingly though it is failing on this file.

Thanks!

bors r+

Sorry one sec. Gonna make a quick change.


Adding other validations for Register Controller

The next commit will be finishing the Register Controller. IT needs to be made better error handler, also the username suggestion would be better, and finally making stronger validations on password, not to match with the first and last name with the user

Password in Register Controllers is not being hashed.
Look into sending email to the user so to authenticate himself before importing the data into the database (How to send emails in node JS )


let multiplayer name be set in browser storage rather than fully anonymous

[ ] Bugfix (non-breaking change which fixes an issue)
[x] New feature (non-breaking change which adds functionality)

Proposed changes
When using multiplayer option when not logged in you get assigned a random Anon-XYZ id - which then changes every time you refresh the browser connection. It would be nice to be able to set this to a fixed / useful value in someway so that you can make use of this multiplayer capability in situations where you may not have or need authenticated users.
This PR lets the name be set by using a value stored in the browser localstorage area.
So far this PR does not provide a UI to set this value but it can easily be tested by using the developer javascript console and using the command localStorage.setItem("multiplayer-name", "Fred") etc
Even without a UI I think the plumbing for this is useful to explore the concept.
Checklist

[x] I have read the contribution guidelines
[ ] For non-bugfix PRs, I have discussed this change on the forum/slack team.
[x] I have run npm run test to verify the unit tests pass
[ ] I have added suitable unit tests to cover the new/changed functionality

My concern is this allows an anonymous user to impersonate another user by allowing them to set their display name.

OK - Maybe need to add some flag to indicate the user isn't logged in..
maybe start with a ?    ?DCJ  etc ...


[SPARK-27629][PySpark] Prevent Unpickler from intervening each unpickling

What changes were proposed in this pull request?
In SPARK-27612, one correctness issue was reported. When protocol 4 is used to pickle Python objects, we found that unpickled objects were wrong. A temporary fix was proposed by not using highest protocol.
It was found that Opcodes.MEMOIZE was appeared in the opcodes in protocol 4. It is suspect to this issue.
A deeper dive found that Opcodes.MEMOIZE stores objects into internal map of Unpickler object. We use single Unpickler object to unpickle serialized Python bytes. Stored objects intervenes next round of unpickling, if the map is not cleared.
We has two options:

Continues to reuse Unpickler, but calls its close after each unpickling.
Not to reuse Unpickler and create new Unpickler object in each unpickling.

This patch takes option 1.
How was this patch tested?
Passing the test added in SPARK-27612 (#24519).

cc @HyukjinKwon @BryanCutler

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/testing-k8s-prb-make-spark-distribution-unified/10396/
Test PASSed.

Test build #105107 has started for PR 24521 at commit d7312fb.

Test build #105107 has finished for PR 24521 at commit d7312fb.

This patch fails Spark unit tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test FAILed.

Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/105107/
Test FAILed.

Retest this please.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/testing-k8s-prb-make-spark-distribution-unified/10400/
Test PASSed.

Test build #105112 has started for PR 24521 at commit d7312fb.

Test build #105112 has finished for PR 24521 at commit d7312fb.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/105112/
Test PASSed.

Nice try. My impression was that it somehow doesn't reuse that memo so was thinking we need a manual control after upgrading to 4.17 but looking into this again the fix looks correct.

Merged to master.

Ultimately, this does look like a bug in Pyrolite and this workaround might be fine. Would you mind adding a reference to the JIRA in your comment and in the test added in #24519?

I'd like to file an issue to Pyrolite. From my view, memo map should be cleared when op code STOP is hit, because each loading should be independent, so the memo map shouldn't be reuse without clearing.
I added few comments to the test and the JIRA.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/testing-k8s-prb-make-spark-distribution-unified/10405/
Test PASSed.

Test build #105117 has started for PR 24521 at commit 04a2e04.

BTW, I believe there are some more places to fix, for instance, BatchEvalPythonExec. Can you double check those places too while we're here?

Checked other places and fixed that. Thanks for reminding that.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/testing-k8s-prb-make-spark-distribution-unified/10406/
Test PASSed.

Test build #105118 has started for PR 24521 at commit 053e6a5.

Test build #105117 has finished for PR 24521 at commit 04a2e04.

This patch fails Spark unit tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test FAILed.

Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/105117/
Test FAILed.

Test build #105118 has finished for PR 24521 at commit 053e6a5.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/105118/
Test PASSed.

Merged to master.

@viirya Could you please help measure the perf?

Shouldn't be a big diff and I don't think there's other ways to fix this issue even if there's perf diff.
But yes, let's quickly run a simple job and check before/after to clarify the doubt. This was pointed out by @srowen too.

I'm ok. I also think it shouldn't be a big diff. I will post the perf diff.

I ran a pretty simple job and measure time before and after this PR in 5 runs:
>>> import time
>>> start = time.time()
>>> df = spark.createDataFrame([[1, 2, 3, 4]] * 100000, "array<integer>")
>>> collected = df.collect()
>>> end = time.time()
>>> print(end - start)

Before:
0.6398153305053711
0.6232590675354004
0.603309154510498
0.5923750400543213
0.5802857875823975
After:
0.6308741569519043
0.6202919483184814
0.5926861763000488
0.603518009185791
0.5818917751312256


🐛 fix resource version conversion.

Summary
This PR tries to fix the resourceversion convert issue in cloudevent decoder, the original issue is as follow:
failed to decode cloudevent: failed to get resourceversion extension: cannot convert 1 to string

Related issue(s)
Fixes #

LGTM
/assign @qiujian16

/approve
/lgtm


Inaccessible host on push (API)

Before opening, please confirm:

[X] I have installed the latest version of the Amplify CLI (see above), and confirmed that the issue still persists.
[X] I have searched for duplicate or closed issues.
[X] I have read the guide for submitting bug reports.
[X] I have done my best to include a minimal, self-contained set of instructions for consistently reproducing the issue.
[X] I have removed any sensitive information from my code snippets and submission.

How did you install the Amplify CLI?
No response
If applicable, what version of Node.js are you using?
No response
Amplify CLI Version
9.2.0
What operating system are you using?
Mac
Did you make any manual changes to the cloud resources managed by Amplify? Please describe the changes made.
no
Amplify Categories
api
Amplify Commands
push
Describe the bug
I wanna thank you for all the shit!
🛑 An error occurred during the push operation: /
Your socket connection to the server was not read from or written to within the timeout period. Idle connections will be closed.
⠏ Uploading files...⚠ Review the Amplify CLI troubleshooting guide for potential next steps:
An error occurred during the push operation: /
Inaccessible host: amplify-.....s3.eu-central-1.amazonaws.com' at port undefined'. This service may not be available in the `eu-central-1' region.
🛑 ENFILE: file table overflow, open '/Users/...
When this happens, there are no solutions, only reset all again with amplify init.
I've been working with Amplify for about a year now, and I have so many regrets.
Expected behavior
I expect 0 zero nada none issues
Reproduction steps
amplify push
GraphQL schema(s)

# Put schemas below this line

Project Identifier
No response
Log output

# Put your logs below this line

Additional information
No response

Of course, I do amplify init and everything is fine.
I don't know how many hours I lost behind all your bugs...
How can it be possible that until yesterday push worked perfectly and today, after I changed a couple of references, no?
Why is it asking me so much connection limit from one day to the next? Why especially this limit is not asked when I do a new "amplify init"? (Your socket connection to the server was not read from or written to within the timeout period)
It's just an abomination to have to do it all over again, recreate connections with auth storage and api.
I also tried paying for your customer service, and ended up with the same solution...amplify init.

We are closing this issue as it contains inappropriate conduct and language. This repository follows the AWS Open Source Code of Conduct found here: https://aws.github.io/code-of-conduct.


Skip / Include optimization on Apollo Android Client

Question
We have lots of GraphQL queries in our product that use conditional include and skips.
From what I understand, even if your query contains skip and include the server still has to create a query plan, that includes elements that will be skipped based on parameters, which can be time consuming.
Is there a way to optimize the query that is being sent to the server, on the client side, that evaluates the skip/include, thus sending already optimized query to the server?

Hi 👋

From what I understand, even if your query contains skip and include the server still has to create a query plan, that includes elements that will be skipped based on parameters, which can be time consuming.

It depends your server and I'm not the expert there but mostly yes. The server has to run validation and create som form of query plan. The query plan can be as simple as running the execution part of the spec where the central part is collecting all fields. In that case, the server would have to lookup the @skip/@include directive for each field which could take a bit of time but I would expect this to be negligeable in the grand scheme of things.

Is there a way to optimize the query that is being sent to the server, on the client side, that evaluates the skip/include, thus sending already optimized query to the server?

You could use Apollo AST to modify your queries at runtime and prune the skipped parts of your query but that's some work to get right and it's taking the risk of a disconnect with the normalized cache because the normalized cache will not see your runtime modification (should work but it's fragile)
All it all, I'd recommend to:

Measure what's gained from removing the @skip parts by sending both versions to your server and see which one is faster. You might have to ask your backend team for insights about resource usage if the difference doesn't show in round trip times.
Investigate persisted queries. If your app is performance sensitive, persisted queries will bring you potentially a lot of improvement by reducing the network transfer size as well as allowing the server to skip validation completely.

Anything else we can help with here?

Closing for now, please leave a comment if you want us to reopen.


Quick Field with Array

Doesn't seem to play well with arrays.  I have objects called Parameters that are in array in my schema.  The buttons to add or delete are scrunched up.

Also I already had semantic installed before adding your package (official atmosphere package).  Before and after adding your package it looked about the same (i think the form expanded to window size, but I think that was it).
Thanks,

I think something is wrong somewhere else @JeremyBYU. It seems like part (or all) of semantic-ui is missing from your project.
For example semantic-ui makes form's labels bold, yours aren't. Plus arrays are working properly (except for the issue #7).
Here is an example of an array field:

See? "Payments" is a block header attached to a segment for the array items. It looks like your output is missing CSS.
Did you remove semantic:ui-css package after adding fabienb4:autoform-semantic-ui? If so, you have to add it back. This package doesn't provide semantic-ui.

Added warning in README that semantic-ui isn't included in the package.
Feel free to re-open the issue if you still encounter the problem.

I apologize.  It seems that the call to set the default semantic template was not executed correctly. After executing correctly I am getting everything as expected.  Thank you so much.

Glad you found out what the problem was!
If you have more questions, don't hesitate to send me an email (in my profile)!


Feat: Model selection

It is technically possible to run evo using gpt-3.5-turbo-0613, we could modify the settings modal to include a dropdown where the user can select the model

This has been completed.


Add div layout migration support

This is to make sure the plugin is compatible with the migration from tables to divs that happened in Jenkins 2.264.

Thank you for the release @donmccasland 👏🏽 👏🏽 👏🏽 👏🏽 👏🏽


return of 1 on sourcing

Installing the plugin with tpm returns 1 after sourcing. I had previously had the catppuccin theme installed without an issue. These are the only two lines that I added to my really small config file, located at ~/.config/tmux/tmux.conf:
set -g @plugin 'rose-pine/tmux'
set -g @rose_pine_variant 'main'

Have you cleaned up the ~/.tmux/plugins/tmux directory?
Both catppuccin and this theme use the "tmux" namespace for the tmux plugin files

Thank you. I ended up re-cloning tpm and that cleaned up the issue.


Issue installing RCall

I am trying to use RCall on the Linux server. I get the following error however and I am not sure what it is.
ERROR: InitError: cglobal: could not find symbol R_NilValue in library
in init at /data/rszoh/.julia/v0.4/RCall/src/setup.jl:113
in _require_from_serialized at loading.jl:84
in _require_from_serialized at ./loading.jl:109
in require at ./loading.jl:235
during initialization of module RCall

What version of R are you using?

I am using : R version 3.2.2 (2015-08-14) -- "Fire Safety"

Hmm, that's odd.
Can you give more details on:

system & OS
Julia version (the output of versioninfo())
the R location by running,
R RHOME at the shell,
R.home() in R, and
Base.compilecache("RCall") in Julia

Here there are:
Julia Version 0.4.6-pre+24
Commit d939abb (2016-04-17 03:57 UTC)
Platform Info:
System: Linux (x86_64-redhat-linux)
CPU: Intel(R) Xeon(R) CPU E5-2665 0 @ 2.40GHz
WORD_SIZE: 64
BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Sandybridge)
LAPACK: libopenblas64_
LIBM: libopenlibm
LLVM: libLLVM-3.3

R.home()
[1] "/share/apps/R-latest/lib64/R"

[rszoh@compute-1-14 ~]$ R RHOME
/share/apps/R-latest/lib64/R
I should mentioned that to start R on the server, I have to do:
[rszoh@compute-1-14 ~]$ module load R-latest
julia> Base.compilecache("RCall")
R installation found at "/share/apps/R-latest/lib64/R"
"/data/rszoh/.julia/v0.4/lib/v0.4/RCall.ji"

I should mentioned that to start R on the server, I have to do:
[rszoh@compute-1-14 ~]$ module load R-latest

Ah, this could be the issue. Do you know what's in that modulefile?

I have no clue. How do I find that out?

As I keep trying I get this (maybe that helps):
julia> Pkg.build("RCall")
julia> using RCall
ERROR: InitError: cglobal: could not find symbol R_NilValue in library
in init at /data/rszoh/.julia/v0.4/RCall/src/setup.jl:113
in _require_from_serialized at loading.jl:84
in _require_from_serialized at ./loading.jl:109
in require at ./loading.jl:235
during initialization of module RCall
julia> using RCall
julia> reval("1+2")
ERROR: ccall: could not find function Rf_mkCharLen in library
in reval at /data/rszoh/.julia/v0.4/RCall/src/iface.jl:40

I have no clue. How do I find that out?

I've never used modules, so can't be much help, but you could try:
module display R-latest

this article might also be useful:
http://www.admin-magazine.com/HPC/Articles/Environment-Modules

ok one more question:
I am not sure what is this line doing in the setup.jl (line 38)
const libR  = Libdl.find_library(["libR"],[joinpath(Rhome,"lib")])

Ultimately everything in RCall package ends up calling a C function or obtaining a C global value in the shared object (DLL, dynamic library - whatever the preferred term is) called libR.  The RCall package needs to know the location of that library on your system,  Your error messages like not finding the function Rf_mkCharLen in the library means that the dynamic library that was found didn't have the expected symbols in it.
The call Libdl.find_library(["libR"]) by itself looks on the directories in the ld configuration.  (This is the Linux term, I don't know about Windows and OS X).
$ ldconfig -p | egrep libR.so
        libR.so (libc6,x86-64) => /usr/lib/libR.so

In my case, that's enough information to find it.
$ julia -e 'println(Libdl.find_library(["libR"]))'
libR

Sometimes the way R is installed doesn't put it on the default paths.  Instead it is in the lib subdirectory of Rhome which is determined in the previous line.   Usually running R RHOME in the shell will return the desired value.

Issue resolved! but I got more
@dmbates  Thank you so much for pointing us to the right direction! So after creating a libR.so file in $R_HOME/lib and recompiling things work - but not quiet perfectly. How do I make julia use my personal R packages file?

This is handled by R: see https://stat.ethz.ch/R-manual/R-devel/library/base/html/libPaths.html

Please reopen if there is anything else to be done here.


Peptides: UI/UX improvements

[x] Add Sequence Space viewer.
[x] Add tooltip for Mutation Cliffs pairs panel.
[x] Fix Monomer-Position not reacting to selection.
[ ] Align MacromoleculeDifference renderer.
[x] Improve Actions panel style to match other similar panels.
[x] Add icon to show any wide table in full screen (e.g. Mutation Cliffs pairs).
[x] Reduce histogram height in tooltips and panels.
[x] Fix Most Potent Residues interaction.
[x] Fix Logo Summary Table interaction.
[x] Fix WebLogo in header interaction.
[x] Use absolute value of mean difference for size coding.
[x] Improve tooltips (remove “Statistics:” label and merge count and ratio).
[x] Highlight rows on mouse-over in all viewers: monomer-position, most potent residues, logo summary table and logo in header.
[x] Adaptive Monomer column width.
[x] Remove steps from tutorial. Open analysis instead.
[x] Align viewer interactions. ref
[x] Add tooltips for all inputs.
[x] Improve Identity and Similarity panel layout.
[x] Remove the ? in Monomer-Position viewer; show info in tooltip.
[x] Show context panel even for selection from grids and other viewers.
[x] Scale coloring in viewer from blue (low values) to red (high values), same as in all other viewers.
[ ] Add buttons to show and hide viewers in analysis.
[x] Navigate through grids with keyboard.

This issue has been mirrored in Jira: https://reddata.atlassian.net/browse/GROK-14879

This issue has been mirrored in Jira: https://reddata.atlassian.net/browse/GROK-16135


Feat: Add --log-level option to wws

Fixes #253.
Adds a --log-level option to configure the actix_web logs that can be one of the valid log::LevelFilter options (off, error, warn, info, debug, trace). It will default to info if none is specified.
This is my first Rust-only PR. I'm sure some things can be more elegant, so please let me know 😄

Hello @sea-grass,
Sorry for the long time to reply and thank you for your contribution. After some long discussions, this project is currently in the process of being archived. However, the development of this project will continue as an independent community fork present at https://github.com/webassemblylabs/wasm-workers-server.
We would be grateful if you could recreate your contribution under that fork and we will work on applying it.

Thanks for letting me know @ereslibre. I'll re-open this PR against the other repository.


A314cp problems using a Pi4 or CM4 under emu68 (PiStorm/PiStorm32 Lite)

Using a Pi4 or CM4 (both versions) with a PiStorm or PiStorm32 under emu68 unfortunately makes the A314 clockport version unusable. All inputs, e.g. in the shell, are processed at a snail's pace and sometimes require multiple calls. It was tested on A500 with Lazarustorm and Clockport Adapter on the 68k socket and CM4 Lite/CM4 EMMC, second test with PiStorm32 Lite and Raspberry Pi4.
No problem using a Raspberry 3A+ on either configuration.

I was told that this issue has been fixed by an update to Emu68.


Transform patterns

Fixes https://github.com/KittyCAD/modeling-app/issues/2735
Adds a new KCL stdlib function pattern(numRepetitions, transform, whatToRepeat) which replicates some geometry the given number of times, applying the given transform to each repetition.
This is the the KCL stdlib function which accepts another function as input, so there's some small refactors to make that possible.
Still TODO:

Generate a KCL docs schema for the transform parameter (we've never had to generate docs for higher-order functions before)
Integrate it into the modeling-app (i.e. use the return value from the API to register the new replicated geometry with the rest of modeling app).

Got into merge conflict hell. Started again in https://github.com/KittyCAD/modeling-app/pull/2824


API docs formatting/review changes and scala tutorial added

Made code formatting and spelling correction changes to Python and Scala API docs. Added one scala tutorial.

why remove all the >>>? It seems to be the standard way of displaying mixed python code and output

We had some discussions around this and turned out that due to >>>, it becomes difficult to copy paste the code for users while trying those examples. So, removes those.

Has there been complaints about this?
This is standard used every where for mixed data and code.
for example: https://docs.scipy.org/doc/numpy/user/basics.broadcasting.html

If you see this new changed link https://github.com/Roshrini/mxnet/blob/b1cd636be561b5443b47046bc1614177a4eb62e6/docs/api/python/kvstore.md
You can see if there is separate function without >>>, it becomes easy to just see input code block and then output code block just below that.

new pull request opened


Add CameraRenderGraph::set

Objective
Some render plugins, like bevy-hikari require to set CameraRenderGraph. In order to switch between render graphs I need to insert a I need to insert a new CameraRenderGraph component which is not very ergonomic.
Solution
Add CameraRenderGraph::set like in Name.

Changelog
Added

CameraRenderGraph::set.

Good job. I use the pathtracer and I really need this Camera set. I suggest to merge it into 0.9

bors r+

The failure seems to be because of the lto = true enabled in #6461. #6472 should fix this.

bors retry


is @schedule_start_time in UTC or local time?

Please document if @schedule_start_time is in UTC or local time?

Document Details
⚠ Do not edit this section. It is required for docs.microsoft.com ➟ GitHub issue linking.

ID: 386f5b1f-e835-0a3b-8e3f-302a6bf5670e
Version Independent ID: 2470305f-0366-36ba-5365-adba61dd02b2
Content: Create and manage Elastic Database Jobs with Transact-SQL (T-SQL) - Azure SQL Database
Content Source: articles/sql-database/elastic-jobs-tsql.md
Service: sql-database
Sub-service: scale-out
GitHub Login: @jaredmoo
Microsoft Alias: jaredmoo

@HenrikStaunPoulsen We are checking this and will get back to you.

@HenrikStaunPoulsen The times in elastic jobs is always UTC. This has been confirmed by Product team as well.

@HenrikStaunPoulsen We are going ahead and closing this thread for now. If you have any further questions, please get back to us so that we can continue the discussion.


D3D12: Fix blit when MSAA is enabled and ResolveSubResource texture format parameter

I don't understand the dstLocation.pResource change. You're blitting to the resolve target of a different MSAA texture. If you need to blit from a normal texture or resolve an MSAA texture, you should just use a non-MSAA target texture to begin with, instead of abusing texture internals.

No I'm just using the blit command to copy one texture to another, as it is. This change is all about allowing this.

No I'm just using the blit command to copy one texture to another, as it is

You're blitting from non-MSAA or resolved-MSAA to MSAA, that logic seems nonsensical to me.
Imagine this blit scenario:

src is MSAA
dst is MSAA

Previously: With BGFX_TEXTURE_MSAA_SAMPLE on src, this correctly blits the raw MSAA texture*. Without it, you get an error because you're asking for a resolve, but your resolve target is MSAA (which makes no sense).
With your change: With BGFX_TEXTURE_MSAA_SAMPLE it now errors, so you can no longer blit between raw MSAA textures. For the second case, you now "resolve" src into dst. But it's now out of sync, because the next time you use it in a framebuffer, it still has the old content - you only overwrote the temporary resolve texture (m_singleMsaa).
The solution is straight-forward: blit to a non-MSAA texture if you want the resolved version.

* This BGFX_TEXTURE_MSAA_SAMPLE is a bit awkward and would ideally be detected automatically. So if you blit MSAA -> MSAA it should always copy the raw texture.

I don't understand why I should use another texture while this patch just auto-detects on which case are you, and uses the right way to approach the copy. All cases are covered and nothing was "broken".
Anyway, I'm open to suggestions for this patch, but at the moment the fact is that if you try to blit from an MSAA flagged texture to another MSAA one it will just not work. So which option do you give here? This works perfectly fine under all renderers but DX12 ( also, using exactly this logic ) at the current stage.

This works perfectly fine under all renderers but DX12 ( also, using exactly this logic ) at the current stage.

Just so I understand the issue exactly:
You blit from MSAA to MSAA.

This works with the Vulkan backend, but not D3D12?
Does it work when you add the BGFX_TEXTURE_MSAA_SAMPLE flag on the source texture?
What do you do with the destination texture after blitting? Sample as texture? Keep rendering into?

This works with the Vulkan backend, but not D3D12?
Yes, also OpenGL and DX11 works

Does it work when you add the BGFX_TEXTURE_MSAA_SAMPLE flag on the source texture?
Didn't try but I can try to replicate this situation on a bgfx demo

What do you do with the destination texture after blitting? Sample as texture? Keep rendering into?
It's the blit of a depth texture and we keep rendering on it


Update Posts “a-simple-post”

Automatically generated by Netlify CMS

👷 Deploy Preview for steady-cupcake-4564a9 processing.

Name
Link

🔨 Latest commit
53c0a97f096cd15947ef6f083918b59ec25007ee

🔍 Latest deploy log
https://app.netlify.com/sites/steady-cupcake-4564a9/deploys/63baa948247af900082c26b3


ARROW-3781: [C++] Implement BufferedOutputStream::SetBufferSize. Allocate buffer from MemoryPool

Since the BufferedOutputStream ctor was using std::string to allocate a buffer internally, I needed to use a static ctor that returns Status for creating the output stream with an arbitrary buffer size.

@pitrou @bkietz there's lots of places where we pass shared_ptr by const-reference, and others by value. In a lot of cases (e.g. object constructors) the intent is to retain a copy of the shared ptr (otherwise we should pass const T& or T*), so in these cases it might be better to pass by value so that a shared_ptr copy is not forced.
According to the C++ gurus https://stackoverflow.com/a/8741626/776560 there's no reason to pass by value unless you plan to make a copy. So I'm thinking we might start making our intent to retain ownership more clear by passing shared_ptr by value in more places, even though it may require us to use std::move more often in internal code

Mis-named my branch. I'll work on the BufferedReader (ARROW-3782) in a new branch

I don't think we should bother micro-optimizing shared_ptr parameter passing, except in critical routines. IO objects are heavy enough that paying the cost of a shared_ptr copy in a constructor call is unimportant.

The benefit isn't performance so much as semantics, and I agree that for taking/retaining ownership of the ptr by value is more clear. It isn't very important for shared_ptr but it is necessary for unique_ptr, so if we're going to transition to the latter it'd be useful to become accustomed to a convention which works for both

@pitrou agreed that it isn't worth making a decision on the basis of performance
@bkietz we use both unique_ptr and shared_ptr for object lifetime in the project, but anytime an object might become visible in a binding layer (e.g. Python, R, Ruby) it is usually necessary to use shared_ptr. For internal exclusive ownership of resources we try to use unique_ptr where we can

I'll submit a patch for ARROW-3782 (buffered input stream / random access file) in a separate patch, so this can be merged in the meantime

+1


Asset name of darwin/amd64 is missing the version

https://github.com/mattn/bsky/releases/tag/v0.0.63
bsky-darwin-.zip

https://github.com/mattn/bsky/actions/runs/12020901190/job/33510365063#step:5:140
make[1]: gobump: No such file or directory
make[1]: *** [show-version] Error 1

make show-version failed because gobump isn't found.

There are several ways to solve this issue.

Change GitHub Actions Runner from macos-latest to macos-13
Add $HOME/go/bin/darwin_amd64 to $PATH
Change the release asset architecture from darwin/amd64 to darwin/arm64
Release assets for both darwin/amd64 and darwin/arm64

About the option 3, I think the major architecture of macOS has been changed from amd64 to arm64.
Recent macOS uses apple silicon, and GitHub Actions runner macos-latest is also arm64.
So removing the support of darwin/amd64 is one option.
In my opinion, supporting both darwin/amd64 and darwin/arm64 is better.

I disagree with the option 2.

I have already created a pull request for the option 1, but if you prefer other options, I would address it.
What do you think? @mattn


NPCDeathEvent inherits from a wrong class

As of right now, NPCDeathEvent inherits from HurtEvent.
It should inherit from DeathEvent to maintain the style of the code, as PlayerDeathEvent inherits from DeathEvent.

Fixed.


Password checker

Maybe some sort of password checker/cracker to check with a password list like "lol123, 123lol" etc. :) or a moviepusher. botaccounts watching the movie you created and you get the money/fame!
otherwise great tool!

Maybe currently not a priority


Update HW04 makefile to use correct output#.txt files

As referenced by #4:

in the makefile of hw04, the new test cases 6 through 8 diff with output5.txt

fix: change outputn.txt to use the correct testcase output
also, add the extra test cases to testall if necessary

fix undefined reference to pow

info here (stackoverflow), but I couldn't compile without the -lm flag. This might be unnecessary for most people, and can be removed if the TAs decide.

Closing request as hw4 is already past due and members have probably seen this and deemed it unnecessary.


Remove extra dependencies from docs/Project.toml

Ref #37

Codecov Report

Merging #38 (5daa8fe) into master (29c7dae) will not change coverage.
The diff coverage is n/a.

@@           Coverage Diff           @@
##           master      #38   +/-   ##
=======================================
  Coverage   81.91%   81.91%
=======================================
  Files          32       32
  Lines         846      846
=======================================
  Hits          693      693
  Misses        153      153

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update 29c7dae...5daa8fe. Read the comment docs.

Grande!!


Can't subscribe to RSS feed

Describe the bug
Just downloaded the macos dmg and tried to subscribe to
https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss-analyzed.xml
UI indicates success, but nothing happens.

Seems like RDF isn't supported with parser we use. Reported it https://github.com/bobby-brennan/rss-parser/issues/107

Cool. Thanks for passing it along.

https://github.com/mrgodhani/raven-reader/releases/tag/v0.5.0 Added support for RDF

Sorry to say, but while it parses the feed now the content isn't very meaningful.

It should show description. Unfortunately, the parser won't work with all contents. If we find it is not able to parse it. We will replace it with a description excerpt.


Is this repo still maintained. If yes Please update the dependencies to the latest verson

@riginoommen Thank you for your interest in this project. I'll surely try and find time to update this project.


internal/core/eval: equal disjunction elements not eliminated

What version of CUE are you using (cue version)?

$ cue version
v0.4.3-beta.1

Does this issue reproduce with the latest release?
Yes.
What did you do?

I want to use value.FillPath, it works fine with a single value in struct:
var a = `filltest: {
	key1: *1 | int
}

var b = `{
	key1: *1 | int
}`

// a.FillPath(cue.ParsePath("filltest"), b)
filltest: {
        key1: *1 | int
}

Also works fine with different values in an array:
var a = `filltest: {
	key1: [1] | [...int]
}
`

var b = `{
	key1: [2] | [...int]
}`

// a.FillPath(cue.ParsePath("filltest"), b)
filltest: {
        key1: [1] | [2] | [...int]
}

But act strange with the same values in one array:
var a = `filltest: {
	key1: [1] | [...int]
}
`

var b = `{
	key1: [1] | [...int]
}`

// a.FillPath(cue.ParsePath("filltest"), b)
filltest: {
        key1: [1] | [1] | [1] | [...int]
}

What did you expect to see?
Expectation:
var a = `filltest: {
	key1: [1] | [...int]
}
`

var b = `{
	key1: [1] | [...int]
}`

// a.FillPath(cue.ParsePath("filltest"), b)
filltest: {
        key1: [1] | [...int]
}

What did you see instead?

Tried to add some unit tests in FillPath but got the same error:
            got:  {
            	a: {
            		b: [1] | [1] | [1] | []
            	}
            }
            want: {
            	a: {
            		b: [1]
            	}
            }

Here's a simpler version that also illustrates the problem:
exec cue eval x.cue
cmp stdout expect

-- x.cue --
y: [1] | [...int]
y: [1] | [...int]

-- expect --
y: [1] | []

This fails because the actual output is:
y: [1] | [1] | [1] | []

The output of cue def for this input is also not ideal:
y: ([1] | [...int]) & ([1] | [...int])

Thanks for the quick reply!
I found this bug when I tried to fill in my reference, maybe skipping filling the reference is a workaround to this problem for me 🤔. Still looking forward to the fix ASAP!
BTW, could you tell me where is the source code for this problem? Maybe I can try to fix this.

@FogDong just following up on this, please can you provide a repro comparing v0.2.2 (the version which I assume works for you) and tip, showing exactly what breaks for you?

In the meantime, I'm going to make a guess at what broke from your perspective:
# v0.2.2
go get<EMAIL_ADDRESS>go mod tidy
go run .
cmp stdout stdout.golden

# tip
go get<EMAIL_ADDRESS>go mod tidy
go run .
cmp stdout stdout.golden

-- go.mod --
module mod.com

go 1.18

require cuelang.org/go v0.2.2

-- main.go --
package main

import (
	"fmt"

	"cuelang.org/go/cue"
	"cuelang.org/go/cue/format"
	"cuelang.org/go/cue/load"
)

func main() {
	bps := load.Instances([]string{"."}, nil)
	v := cue.Build(bps)[0].Value().Lookup("z")
	node := v.Syntax(cue.ResolveReferences(true))
	b, err := format.Node(node)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%s\n", b)
}
-- x.cue --
package x

x: [1] | [...int]
y: [1] | [...int]
z: x & y
-- stdout.golden --
[...int]

I suspect your expectation is that this test should pass. However, it does not pass and gives:
# v0.2.2 (0.406s)
# tip (0.396s)
> go get<EMAIL_ADDRESS>[stderr]
go: upgraded cuelang.org/go v0.2.2 => v0.4.4-0.20220801114602-5a08d2f7a9a7
go: upgraded github.com/emicklei/proto v1.6.15 => v1.10.0
go: upgraded golang.org/x/net v0.0.0-20200226121028-0de0cce0169b => v0.0.0-20220425223048-2871e0cb64e4
go: upgraded golang.org/x/text v0.3.2 => v0.3.7
go: upgraded golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 => v0.0.0-20200804184101-5ec99f83aff1
go: upgraded gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71 => v3.0.1

> go mod tidy
> go run .
[stdout]
[1] | [1] | [1] | [...int]

> cmp stdout stdout.golden
--- stdout
+++ stdout.golden
@@ -1,1 +0,0 @@
-[1] | [1] | [1] | [...int]
@@ -0,0 +1,1 @@
+[...int]

FAIL: /tmp/testscript4139106237/repro.txtar/script.txt:11: stdout and stdout.golden differ

(Note there is a bug relating to the result [1] | [1] | [...int] above, but the duplicates in that disjunction can be ignored because they don't affect the semantic result).
The reason this test fails is that since v0.2.2 disjunction elimination behaviour has changed. The notes in https://github.com/cue-lang/cue/releases/tag/v0.3.0-beta.5 discuss this:

Note that v0.2 solved things differently: if in the result of a disjunction one term subsumes another, then the latter could be dropped. So in v0.2, the answer would be x: {a: int}, even in the absence of defaults. The problem with this approach is that it could lead to confusing answers and that the default could not always be accurately determined. A goal of v0.3 was to have a simpler story for disambiguation and let simplifications be merely a matter of optimization.

Relating this comment to my test case. In v0.2.2:
[1] | [...int]

reduced to:
[...int]

But in tip it remains as:
[1] | [...int]

@FogDong - it would still be good to understand how/why this breaks you. Thanks

Actually this is an issue in v0.2.2 and v0.4.3, I created this issue in April, it's not an issue preventing us from updating the cue version.

I create a repro here:
go mod tidy -compat=1.17
go run main.go
cmp stdout stdout.golden
-- go.mod --
module test-cue/and

go 1.17

require cuelang.org/go v0.4.4-0.20220729051708-0a46a1624353

require (
	github.com/cockroachdb/apd/v2 v2.0.1 // indirect
	github.com/emicklei/proto v1.10.0 // indirect
	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
	github.com/google/uuid v1.2.0 // indirect
	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
	github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect
	github.com/pkg/errors v0.8.1 // indirect
	github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b // indirect
	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
	golang.org/x/text v0.3.7 // indirect
	gopkg.in/yaml.v3 v3.0.1 // indirect
)
-- main.go --
package main

import (
	"fmt"

	"cuelang.org/go/cue"
	"cuelang.org/go/cue/cuecontext"
	"cuelang.org/go/cue/format"
)

var a = `filltest: {
	key1?: [1] | [...int]
}
`

var b = `{
	key1?: [1] | [...int]
}`

func main() {
	ctx := cuecontext.New()
	a := ctx.CompileString(a)
	b := ctx.CompileString(b)

	result := a
	for i := 0; i < 3; i++ {
		result = result.FillPath(cue.ParsePath("filltest"), b)
		b = result.LookupPath(cue.ParsePath("filltest"))
	}

	s, err := format.Node(result.Syntax(cue.All(), cue.ResolveReferences(true), cue.DisallowCycles(true), cue.Docs(true), cue.Attributes(true)))
	if err != nil {
		panic(err)
	}
	fmt.Println(string(s))
}
-- repro.txt --
-- stdout.golden --
{
        filltest: {
                key1?: [1] | [...int]
        }
}

And the result is like:

> go run main.go
[stdout]
{
        filltest: {
                key1?: ([1] | [...int]) & ([1] | [...int]) & ([1] | [...int]) & ([1] | [...int])
        }
}

> cmp stdout stdout.golden
--- stdout
+++ stdout.golden
@@ -1,5 +1,5 @@
 {
-       filltest: {
-               key1?: ([1] | [...int]) & ([1] | [...int]) & ([1] | [...int]) & ([1] | [...int])
-       }
+        filltest: {
+                key1?: [1] | [...int]
+        }
 }

FAIL: /var/folders/2n/0vszm3355n114dk2r_ln8vrh0000gp/T/testscript40035967/repro.txt/script.txt:3: stdout and stdout.golden differ
error running repro.txt in /var/folders/2n/0vszm3355n114dk2r_ln8vrh0000gp/T/testscript40035967/repro.txt

In my example code, I use a for loop to FillPath another value. In KubeVela, we'll have a user cue value as the base value, and KubeVela creates a new value, and try to fill the user base value. In every controller reconciliation, we'll try to fill the path for the base value if the step is still running. So if the base value is in the format [1] | [...int], the result will be complicated. And if the base value is complicated enough and as reconciliation grows, the cue value will explose.

Thanks for the repro. So the bug you are seeing is related to something I mentioned in passing earlier:

(Note there is a bug relating to the result [1] | [1] | [...int] above, but the duplicates in that disjunction can be ignored because they don't affect the semantic result).

However what I said there isn't quite correct.  [1] | [...int]
x: [1]
-- stdout.golden --
{
    "x": [
        1
    ]
}

should pass but gives:
> exec cue export x.cue
[stderr]
x: incomplete value [1] | [1]
[exit status 1]
FAIL: /tmp/testscript949714628/repro.txtar/script.txtar:1: unexpected command failure

Leaving this marked as "disjunctions" so we pick it up as part of that work.

@mpvl - to discuss as part of v0.6.0

This one's priority is high since there's no workaround.
This is a user-facing issue because we have a command called vela debug, in which all the cue values will be rendered and displayed to our users. This issue will make the debug output full of & with the same struct (if the original array is rather big, which is quite common in a k8s scenario, the output is a nightmare..)
Hope to be fixed in v0.6.0.


Blog page link on about.md (/about page) is broken.

Hi Isabel, I noticed that one of your links on your /about page points to possibly an old folder name and is therefore broken.  I will submit a PR to fix this.

Thank you so much for fixing this @SuzeShardlow


[BUG]: Content in "About Us" section not visible in light mode

Description:
When using the application in light mode, the content in the "About Us" section is not visible. It seems that the text color is not appropriately set for light mode, making the content appear to be invisible.
Steps to Reproduce:
Open the application in light mode.
Navigate to the "About Us" section.
Expected Behavior:
The content in the "About Us" section should be clearly visible and legible, irrespective of the selected color mode.
Screenshots
https://github.com/akshitagupta15june/PetMe/assets/81636517/2948925a-dcce-4fe0-b864-c05cf352e315
Desktop

OS: Window10
Browser: Chrome
Version: 115.0.5790.110

This issue was fixed

@akshitagupta15june, I am not sure this is fixed.
If not fixed, please re-open and assigned to me so I can work on it.
My suggestion would be to match colors in the top header:

Secondary task(s)
Remove hover on About Us title since this element is not a CTA. (image showing hover effect):


Issue: unclear processing of natural language query parsing + external api querying afterwards

Issue you'd like to raise.
I believe one of the most popular directions for langchain-based applications will be to parse the query in a natural language and to use some external api for the response generation. However, as soon as this external API doesn't have an OpenAISchema-based specification, the easiest way to do it will be via chain parsing + request.
I tried to implement it via a sequential chain, the example is provided below:
EXCHANGE_API_KEY = SOME_KEY

from langchain.chains import LLMRequestsChain, LLMChain, SimpleSequentialChain
from langchain.chains.openai_functions.openapi import get_openapi_chain
from langchain.prompts import PromptTemplate, SystemMessagePromptTemplate, ChatPromptTemplate, HumanMessagePromptTemplate

template = """You help to convert currencies.
You need to convert the input phrase into the query like from=EUR&to=GBP&amount=100"
You need to extract these parameters yourself from the query in natural language.
Please, return the string like https://api.currencylayer.com/convert?from=EUR&to=GBP&amount=100, where you need to fill in "from", "to", and "amount" based on previous instructions"
"""

system_prompt = SystemMessagePromptTemplate.from_template(template)

human_message_prompt = HumanMessagePromptTemplate.from_template("{question}")

first_prompt = ChatPromptTemplate(
messages=[
        system_prompt,
        human_message_prompt
    ]
)

## this one above may be done nicer, via PromptTemplate, but I don't think it's necessary here

template_2 = """
Query api with {synopsis} and return api response
"""

llm = OpenAI(temperature=0)
convert_chain = LLMChain(llm=llm, prompt=first_prompt, output_key="synopsis")

system_prompt_2 = PromptTemplate(input_variables=["synopsis"], template=template_2)
chain = LLMRequestsChain(llm_chain=LLMChain(llm=OpenAI(temperature=0), prompt=system_prompt_2))

overall_chain = SimpleSequentialChain(chains=[convert_chain, chain], verbose=True)

question = "Convert 1000 American Dollars to euros please"
inputs = {"input": question,
    #"url": "https://api.currencylayer.com/convert",
    "headers": {"access_key": EXCHANGE_API_KEY}
}

overall_chain(inputs)

The code above fails due to requests library stacktrace: InvalidSchema: No connection adapters were found for 'System: [https://api.currencylayer.com/convert?from=USD&to=EUR&amount=1000'](https://api.currencylayer.com/convert?from=USD&to=EUR&amount=1000%27)
It happens because the string "System: " is inevitably attached to the output of the first chain.
So, my questions are as follows:

Is the way I showed above right now the best option to interact with the APIs which have not been implemented yet to langchain and haven't got OpenAISchema-based docs?

If so, how to deal with my problem and strip the "System: " part for query? Is there a natural way to parse the natural language query into an API request via this chain?

Suggestion:
I think there should be a very clear way and documentation of how to deal with external APIs after natural language parsing done by generative network

+1


Add OpenAPI functionality

As of today, Fast api and Flask are more preferred framework to develop microservice and service mesh architecture.
Flask is more preferred for below points:

microservice architecture
more extension available in community than others as its bit matured.

I really like the simplicity flask has. But in recent days Flask is loosing to FastAPI.
Biggest CON of flask is its incompatibility with OpenAPI/documentation capability.
Being an Architect, my whole org has moved to FastAPI for a simple reason OpenAPI integration and documentation. As it helps us save on maintaining more than 30+ product API documentation. Earlier we were using 3rd party tools to maintain these documentation.
It would really help flask gain more ability if we can add openapi specification just like fast api.
I understand the same feature request was raised in back in 2021 but I believe being in technology domain we have keep ourselves updated. Same goes with the web framework as well.
To illustrate few, Django is adding capability to have microservice support. why? its to keep them updated with latest requirement.
Out of Django, Flask and FastAPI only Flask is not having OpenAPI support.
I know there are library available in community which can do this but having it natively available in flask bring more confidence.
Out of Django, Flask and FastAPI only Flask is not having OpenAPI support. I know there are library available in community which can do this but having it natively available in flask bring more confidence and usability. as there are open source repo which supports this functionality may slowly loose its contributor unlike Flask contributors.

To me it sounds like having a well-maintained Flask extension for this would fit much better in the spirit of how flask works...

There are already numerous extensions that provide this. It's outside the scope of Flask core.


Jobs page: Add the link to the Discourse forum post of rules for responding to a job and add the rules

We've had a higher rate of what seem to be 'copy-paste' responses to jobs posts this year - one way we're limiting that is introducing a rule that if you're posting a reply in our forum to a job opening from our jobs board that you avoid copy-paste like responses and you will get 3 warnings before being banned.
You can find info here: https://discourse.opensourcedesign.net/t/guidelines-on-posting-and-responding-to-jobs/3416
I think that another location to surface this information is the jobs page itself!
Let's link to this forum pots in the top of the jobs page and also consider having text of the rules in order to try to avoid copy-paste responses from designers.

I think that another location to surface this information is the jobs page itself!

Since there is no way to directly comment on the job page, I would probably not add the text here, since it diverts attention from things people can actually do there right now (posting jobs, browsing jobs)

I think that another location to surface this information is the jobs page itself!

Since there is no way to directly comment on the job page, I would probably not add the text here, since it diverts attention from things people can actually do there right now (posting jobs, browsing jobs)

I think my justification was that a lot of designers wanting to respond to a job do it via the individual jobs page and then click into the specific discussion for that job, bypassing the notices in the discourse forum.
My other suggestions for this would be adding it to the bottom of ever job posting right before the embedded forum discussion link and/or at the top of every created forum discussion individual job post. Thoughts?

My other suggestions for this would be adding it to the bottom of ever job posting right before the embedded forum discussion link…

Yes, before or after the link would be a good place. A short text like "Don’t use generic copy-pasted responses; they can get you banned (See Guidelines)" would not take much space.

After some (too long) investigation in the job repo I think the perma link to the 'reply to job post' info needs to be added just above where this .html talks about discourse stuff:
https://github.com/opensourcedesign/jobs/blob/master/_layouts/jobs.html

Hello,
Is this issue still active?
I would like to work on this.
Best Regards.

Hello,
Is this issue still active? I would like to work on this.
Best Regards.

I'll assign you this issue for 1 month and then un-assign if there's no progress - thank you 😄

Hey. I would like to work on this..
Best Regards


Examples do not seem to work properly

Hello,
Could you please indicate how to use it ? I tried to launch the HTML from examples folder, but nothing is displayed

Could you also tell me if this lib is able to render lattices geometries (with threejs) ?
Thank you

Hello,
The easiest way;
npm install
npm run dev

This should start a server with all examples, although opening the html files should have worked too...
Not sure what you mean exactly by lattices geometries, this library can polygonize implicit surfaces with certain conventions.
So some lattices geometries may be represented implicitely and polygonized for standard rendering, it depends on their representation.
Also, it may not be the best way to render those structures, depending on their shapes. Do you have a screenshot of what you need to render ?

Thank you
When running 'npm run dev', I have following error:

I would like to render lattices with TPMS unit cells, using implicit way (with threejs) in order to save  resources :

That's strange.
Remove node_modules folder, package.lock and re run:
npm install
So yes, you should be able to render those. With the proper implicit function you can polygonize that with the appropriate level of detail.
If your structures are only made of Capsules shapes then you can also generate those shapes individually and merge in a single mesh, or use special shaders for direct ray tracing rendering, like the one used on this tool (this one is proprietary though, contact me privately if interested).

In terms of performance, the benefit from an implicit polygonization over superposing capsules geometries will not be huged, if noticeable at all, as long as you merge those into a single Three.Geometry

ok thank you, it works fine now after running 'npm install' again.
Do you any example of rendering of multiple unit-cells arrangement (such as the one from the picture I sent you) ?
Because I'm not sure that even using one single three.geometry (containing all the cells) that the webGL visualization pipeline will not hang with hundreds of thousands cells.
That's why I'm looking for "implicit SBF" rendering systems.

My goal:
import a 3MF description of lattices structure (mainly described by cylinders/beams) in THREEJS, then render efficently in 3D, not with 'explicit' traditional triangles meshes.

Yes, those kind of geometries may be overwhelming, but three.js is not the bottleneck here, the mesh is and it depends on the GPU you are using to render that.
This library does not provide any special rendering pipeline, it's just representation and polygonization.
So in the end, from the blobtree you would get a mesh, if it's above say 2M polygons, you are likely to hit limitations.
SDF rendering shaders would help, but you will probably face limitations here too if you have hundreds of thousands of cells.
The best technique I know of is the one used by CapsuleSketch where you can raytrace each cylinders, cones and spheres with a special shader hack running on only 12 triangles per shape.
This one could probably be adapted and optimized to your usecase, although you would not be able to go under 2 triangles per shape, so say more than 1M shapes and you are likely to reach the limits.

ok thank you, it's clearer now !

How many cylinders do you have in your model ?

I have some models between 100K and 1M ; but I have one with nearly 2M..
I don't know about the model of Formula1 brake pedal (picture).

Maybe you can find some raytracing engine, although in your kind of situation, in general a custom solution is better because you can build on specificities of your problem.
For example here, you could render lines instead of cylinders when moving the mouse, then when the mouse stops you would render cylinders by batch progressively using the raytracing method I mentionned above.
If you find something, please post it here, it could interest people who stumble on this thread :)

As mentionned above, three.js will not be the bottleneck for a static object like yours, you just need to reduce the number of drawcalls then the GPU is the bottleneck.
It's difficult to give a number but in the forum you linked, the guy mentionning 20M is rendering billboards with impostors (this is the technique mentionned above use in Capsule Sketch, also it's a bit more fancy for cylinders or cones) and does not give details on his hardware.
Also, point clouds are not as expensive as meshes, in our projects we usually found that for similar performances you can render 3 to 4 times as much points as you can render triangles, but many parameters can influence.
If you test with just millions of cylinders, please let us know the performance you get and on which model.
The problem with raymarching SDF in shaders is the definition of the object. It works very well with small procedural definitions like in the artistic shaders from Iq, but you can't put that much data in a shader, you need smart float texture accesses and complex 3D data structures.
Exciting problem anyway, let us know what you find !


Add an 'op' operand to SPIRType

Continuing the discussion started in https://github.com/KhronosGroup/SPIRV-Cross/issues/2145#issuecomment-1545564097:
The current design of SPIRType in SPIRV-Cross is "flat": a given instance of SPIRType provides information on the number of stars (point depth), array size, vector size and even matrix size all in the same struct, in pseudocode:
Type(vectorwidth=2, ptr=1, as=Input, base=Int, bitwidth=32)
This is a departure from most compilers and the SPIR-V specification where a complex type is built up from a base type by applying a number of constructors:
PtrType(as=Input, pointee=VectorType(width=2, element_t=IntType(bitwidth=32)))
The flat design in use currently is however problematic:

Complex types such as structures cannot effectively be flattened - so the actual implementation uses a mixture of both designs
Pattern matching on type constructors is done in a ad-hoc way by looking at fields, for example type.array.empty(). This is unsafe because a pointer to an array passes this test too (1)
There is no way to disambiguate pointer-to-array and array-of-pointers. The former patterns historically were absent from Vulkan flavours of SPIR-V due to the initially restrictive capabilities, but with the addition of physical storage buffers this is no longer tenable.

(1) And in fact, a number of places in the implementation make such shortcuts, as I found out over the last few days.
This patch attempts to bring SPIRType back in line with this more traditional approach by introducing an op field to SPIRType, which corresponds to the SPIR-V opcode for the top-level type expression of any given type. This operand is set when constructing a type, either when parsing it or when created "inline".
Since there is still no validation that the op matches the various flattened fields, great care was taken to comb over the existing code and correctly set the operand wherever types are created. In the future it would be a great addition to move away from creating SPIRTypes inline and instead make proper constructors, that would also enable a smoother migration fully away from the flat design.
This patch is still a draft, but the tests all pass with no regressions currently. I'd like to refactor more code to use the operand as the ground truth rather than looking at the flat fields, but maybe it's better to split this up into smaller PRs.

 Thank you for your submission! We really appreciate it. Like many open source projects, we ask that you sign our Contributor License Agreement before we can accept your contribution.You have signed the CLA already but the status is still pending? Let us recheck it.

There are CI failures on MSL. The test files need to be updated.


Release/v1.1.0

Purpose

How does the solution address the problem

Changes made

prnet cleaned-up successfully


ListWidget with more capabilities: add/remove/modification

Hi @jschrewe,
In our project, we needed the ListField(FileField) structure to have more capabilities:

allow removal of elements (the current code is buggy)
have a better widget to allow removal + modification

I did some work on our fork to have this possibility, you can find it here: https://github.com/Work4Labs/django-mongodbforms/pull/8.
Some comments about what I did:

commits are not atomic, I used a trial / error approach
the code is not factorised as wanted, I didn't want to change your code too much
there are no tests (reason for number 1)

We are relying on this project a lot in our production environment, and I'm wondering several things: how active is your repo? Do you still use your code in production environment? Would you be inclined to let me be a contributor of the project? Can we setup tests (circleci / travis / other?) ?
Let me know what you think, I'm open for a quick chat :)

cc @olivierlefloch


Listen and let clients connect to us before we establish outgoing connection, so we simply pass through traffic

Implemented by 8bb3538761010fb6a9baa44b3935452bec35c3ef


Special case apply in pprint_tasks

Fixes #741.
Also only tests ResourceProfiler if psutil is installed (brought up in internal discussion).

Ping @mrocklin for review.


RandomGridCV - memory leak

Hi,
If I include data from 2011 to 2020, the RandomGridCV throws a memory leak. Therefore I would suggest to further limit the data set from 2016 to 2020 only.
Corresponding changes are done in split_data.py and crime_vancouver.Rmd. Pending final check of the script before PR.

Updated the literate code doc.

#44 closes this issue.


Profile request, @mzfr

I'd like to have my profile on ghuser.io, thanks!

here you go :) https://ghuser.io/mzfr


fix: Tooltip for long names in Entity Explorer

Description
Long names were not getting tooltips on the Entity explorer. This change will fix this by ensuring that the check for tooltips disabled is ran correctly and not with wrong values.

Fixes #6524
Type of change

Bug fix (non-breaking change which fixes an issue)

How Has This Been Tested?
Cypress Test:

Add a short name and a long name entity
Short name should not get a tooltip on hover
Long name should get a tooltip on hover
Renaming long name to a shorter name should not get a tooltip on hover

Checklist:

[x] My code follows the style guidelines of this project
[x] I have performed a self-review of my own code
[x] I have commented my code, particularly in hard-to-understand areas
[ ] I have made corresponding changes to the documentation
[x] My changes generate no new warnings
[x] I have added tests that prove my fix is effective or that my feature works
[ ] New and existing unit tests pass locally with my changes

/ok-to-test sha=37924e7

/ok-to-test sha=2bc0673

/ok-to-test sha=01f573d

/ok-to-test sha=c8af044


Does not register with mos

Hello,
Thank you for the plugin. This looks great. However I am trying to use this plugin alongside mos and when it is included via the tag:
`
`
mos generates the following error:
execute('echo', ['works']) Error: Failed to execute template code at line 240 with 'execute is not defined'
Can you tell me how to make this plugin work?

Hey folks. I figured this out. In your package.json you need to add:
"mos": {   "plugins": ["execute"] }


WIP: Mount Windows layers as volumes on the host

This is a work in progress implementation of mounting Windows volumes on the host. It currently is failing the Basic snapshotter test due to a few assumptions made by the tests. The first issue is that the root directory of the layer is a link, which is tripping up the filesystem differ. The second is mounting a parent and child layer at the same time is currently failing.
This PR is to share the current progress.
/cc @johnstep @stevvooe @dmcgowan

It is nice to see the tests running though. The base related changes and changes to tests look good so far.
What is the impact of this target + ":layerid" file? They will just need to be created anywhere that is mounted or is created as metadata on the original target?

target + ":layerid" is metadata on the mounted folder to correlate back to which HCS layer is being used to mount so that it can be unmounted. See above about Alternate Data Streams for more details on what is being used to store the metadata.

Pushed more commits that get the following tests passing on Windows:
t.Run("Basic", makeTest(name, snapshotterFn, checkSnapshotterBasic))
t.Run("StatActive", makeTest(name, snapshotterFn, checkSnapshotterStatActive))
t.Run("StatComitted", makeTest(name, snapshotterFn, checkSnapshotterStatCommitted))
t.Run("TransitivityTest", makeTest(name, snapshotterFn, checkSnapshotterTransitivity))
t.Run("PreareViewFailingtest", makeTest(name, snapshotterFn, checkSnapshotterPrepareView))
t.Run("Update", makeTest(name, snapshotterFn, checkUpdate))

Can you rebase, also there is a vet complaint
gometalinter --config .gometalinter.json ./...
snapshots\testsuite\testsuite.go:51::error: unreachable code (vet)
Makefile:109: recipe for target 'check' failed
mingw32-make: *** [check] Error 1

Vet failure is due to enabling certain tests while in development. I'm adding proper test skipping for tests that will not work on Windows in a change soon.
Will rebase.

Locally tests are passing. Debugging why they're failing in CI.

This PR is ready for review pending https://github.com/containerd/continuity/pull/113

Codecov Report

Merging #2287 into master will decrease coverage by 3.96%.
The diff coverage is 75%.

@@            Coverage Diff             @@
##           master    #2287      +/-   ##
==========================================
- Coverage   49.28%   45.31%   -3.97%
==========================================
  Files          84       93       +9
  Lines        7424     9572    +2148
==========================================
+ Hits         3659     4338     +679
- Misses       3089     4532    +1443
- Partials      676      702      +26

Flag
Coverage Δ

#linux
49.28% <ø> (ø)
:arrow_up:

#windows
41.75% <75%> (?)

Impacted Files
Coverage Δ

snapshots/windows/windows.go
58.62% <75%> (ø)

oci/spec.go
40% <0%> (-10%)
:arrow_down:

snapshots/native/native.go
43.89% <0%> (-10%)
:arrow_down:

metadata/snapshot.go
45.8% <0%> (-8.96%)
:arrow_down:

archive/compression/compression.go
43.93% <0%> (-8.9%)
:arrow_down:

remotes/docker/fetcher.go
41.42% <0%> (-7.6%)
:arrow_down:

content/local/writer.go
52.63% <0%> (-7.37%)
:arrow_down:

archive/tar.go
43.05% <0%> (-6.95%)
:arrow_down:

metadata/containers.go
47.97% <0%> (-6.62%)
:arrow_down:

metadata/buckets.go
54.66% <0%> (-5.04%)
:arrow_down:

... and 55 more

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update 6eee2a0...a12b082. Read the comment docs.

Going to close this one, we have a carry PR open and windows work is still ongoing


Analysis of similar projects

lendoit
https://lendoit.com/
colendi
https://www.colendi.com/

Possible other inspirations as well:
Bitbond
https://www.bitbond.com/

@tspoff I think it's a good idea for this to be subsumed into #90. I do wonder whether we can get a MBA volunteer to help us out with a lot of the business + market research stuff.


Improve build system

Added dedicated CMake target with the main function and Makefile target to run it
Added automatic discovery of test source files

On a separate note, the build of Catch2 tests is tens of times slower than GTest. I am disappointed.