CRD tasks will not be deleted completly (alias list kubeteach.geberl.io/v1alpha1)

# INFO ABOUT COMPLETE NEW VANILLA k8s v1.22 (ubuntu + containerd + calico)
root@f2k8s1p:~# kubectl get nodes -o wide
NAME      STATUS   ROLES                  AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION     CONTAINER-RUNTIME
f2k8s1p   Ready    control-plane,master   15m   v1.22.0   <IP_ADDRESS>    <none>        Ubuntu 20.04.2 LTS   5.4.0-80-generic   containerd://1.5.2
f2k8s2p   Ready    <none>                 14m   v1.22.0   <IP_ADDRESS>    <none>        Ubuntu 20.04.2 LTS   5.4.0-80-generic   containerd://1.5.2
f2k8s3p   Ready    <none>                 14m   v1.22.0   <IP_ADDRESS>    <none>        Ubuntu 20.04.2 LTS   5.4.0-80-generic   containerd://1.5.2

root@f2k8s1p:~# kubectl get pods --all-namespaces -o wide
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE     IP              NODE      NOMINATED NODE   READINESS GATES
kube-system   calico-kube-controllers-58497c65d5-ctpqk   1/1     Running   0          2m29s   <IP_ADDRESS>   f2k8s3p   <none>           <none>
kube-system   calico-node-5zfn9                          1/1     Running   0          2m29s   <IP_ADDRESS>      f2k8s3p   <none>           <none>
kube-system   calico-node-8f7xs                          1/1     Running   0          2m29s   <IP_ADDRESS>      f2k8s2p   <none>           <none>
kube-system   calico-node-bkdsq                          1/1     Running   0          2m29s   <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   coredns-78fcd69978-qqzbd                   1/1     Running   0          5m      <IP_ADDRESS>   f2k8s1p   <none>           <none>
kube-system   coredns-78fcd69978-x5rrx                   1/1     Running   0          5m      <IP_ADDRESS>   f2k8s1p   <none>           <none>
kube-system   etcd-f2k8s1p                               1/1     Running   0          5m7s    <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   kube-apiserver-f2k8s1p                     1/1     Running   0          5m16s   <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   kube-controller-manager-f2k8s1p            1/1     Running   0          5m7s    <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   kube-proxy-2bqxx                           1/1     Running   0          5m      <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   kube-proxy-dztqt                           1/1     Running   0          4m37s   <IP_ADDRESS>      f2k8s2p   <none>           <none>
kube-system   kube-proxy-nc8np                           1/1     Running   0          4m33s   <IP_ADDRESS>      f2k8s3p   <none>           <none>
kube-system   kube-scheduler-f2k8s1p                     1/1     Running   0          5m7s    <IP_ADDRESS>      f2k8s1p   <none>           <none>

root@f2k8s1p:~# kubectl get tasks
error: the server doesn't have a resource type "tasks"

root@f2k8s1p:~# kubectl apply -f https://github.com/dergeberl/kubeteach/releases/latest/download/deployment.yaml
namespace/kubeteach-system created
customresourcedefinition.apiextensions.k8s.io/exercisesets.kubeteach.geberl.io created
customresourcedefinition.apiextensions.k8s.io/taskdefinitions.kubeteach.geberl.io created
customresourcedefinition.apiextensions.k8s.io/tasks.kubeteach.geberl.io created
serviceaccount/kubeteach-controller-manager created
role.rbac.authorization.k8s.io/kubeteach-leader-election-role created
clusterrole.rbac.authorization.k8s.io/kubeteach-manager-role created
clusterrole.rbac.authorization.k8s.io/kubeteach-metrics-reader created
clusterrole.rbac.authorization.k8s.io/kubeteach-proxy-role created
rolebinding.rbac.authorization.k8s.io/kubeteach-leader-election-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/kubeteach-manager-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/kubeteach-proxy-rolebinding created
configmap/kubeteach-manager-config created
service/kubeteach-controller-manager-metrics-service created
deployment.apps/kubeteach-controller-manager created

root@f2k8s1p:~# kubectl apply -f  https://github.com/dergeberl/kubeteach/releases/latest/download/exerciseset1.yaml
namespace/kubeteach-troubleshoot created
deployment.apps/fix-me created
exerciseset.kubeteach.geberl.io/set1 created
clusterrole.rbac.authorization.k8s.io/set1 created
clusterrolebinding.rbac.authorization.k8s.io/set1 created

root@f2k8s1p:~# kubectl get pods --all-namespaces -o wide
NAMESPACE                NAME                                            READY   STATUS         RESTARTS   AGE     IP              NODE      NOMINATED NODE   READINESS GATES
kube-system              calico-kube-controllers-58497c65d5-ctpqk        1/1     Running        0          6m1s    <IP_ADDRESS>   f2k8s3p   <none>           <none>
kube-system              calico-node-5zfn9                               1/1     Running        0          6m1s    <IP_ADDRESS>      f2k8s3p   <none>           <none>
kube-system              calico-node-8f7xs                               1/1     Running        0          6m1s    <IP_ADDRESS>      f2k8s2p   <none>           <none>
kube-system              calico-node-bkdsq                               1/1     Running        0          6m1s    <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system              coredns-78fcd69978-qqzbd                        1/1     Running        0          8m32s   <IP_ADDRESS>   f2k8s1p   <none>           <none>
kube-system              coredns-78fcd69978-x5rrx                        1/1     Running        0          8m32s   <IP_ADDRESS>   f2k8s1p   <none>           <none>
kube-system              etcd-f2k8s1p                                    1/1     Running        0          8m39s   <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system              kube-apiserver-f2k8s1p                          1/1     Running        0          8m48s   <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system              kube-controller-manager-f2k8s1p                 1/1     Running        0          8m39s   <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system              kube-proxy-2bqxx                                1/1     Running        0          8m32s   <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system              kube-proxy-dztqt                                1/1     Running        0          8m9s    <IP_ADDRESS>      f2k8s2p   <none>           <none>
kube-system              kube-proxy-nc8np                                1/1     Running        0          8m5s    <IP_ADDRESS>      f2k8s3p   <none>           <none>
kube-system              kube-scheduler-f2k8s1p                          1/1     Running        0          8m39s   <IP_ADDRESS>      f2k8s1p   <none>           <none>
kubeteach-system         kubeteach-controller-manager-84cff55968-wtx2q   1/1     Running        0          3m14s   <IP_ADDRESS>    f2k8s2p   <none>           <none>
kubeteach-troubleshoot   fix-me-94dc7844c-jbtx2                          0/1     ErrImagePull   0          31s     <IP_ADDRESS>    f2k8s2p   <none>           <none>

root@f2k8s1p:~# kubectl get tasks --all-namespaces -o wide
NAMESPACE   NAME     TITLE                           DESCRIPTION                                                                                STATUS
default     task01   Create namespace                Create a new namespace with the name kubeteach                                                                                active
default     task02   Create pod                      Create a pod in namespace kubeteach, name it pod1 and use nginx:latest as image                                                                pending
default     task03   Create deployment               Create a deployment with the name kubeteach-webserver in namespace kubeteach, 3 replicas and nginx:latest as image                             pending
default     task04   Scale deployment                Scale the deployment from task3 to 5 replicas                                                                                pending
default     task05   Add port to deployment          Add a containerPort (TCP port 80) to the deployment from task3                                                                                pending
default     task06   Create service for deployment   Create a service of type NodePort to export port 80 of the deployment from task3                                                               pending
default     task07   Create configMap                Create a configMap with name colorconf in namespace kubeteach with value color=blue                                                            pending
default     task08   Bind configmap data to pod      Create a new nginx:latest pod with name colorconf-pod in namespace kubeteach and use the configmap value color as environment variable COLOR   pending
default     task09   Create serviceaccount           Create a new serviceaccount named kubeteach-sa in kubeteach namespace                                                                          pending
default     task10   Use serviceaccount              Use the serviceaccount kubeteach-sa in a new deployment named nginx-sa with image nginx:latest in kubeteach namespace                          pending
default     task11   Create role                     Create a new role named kubeteach-pod-role in kubeteach namespace with permissions on pods (get, list, watch)                                  pending
default     task12   Create rolebinding              Connect the role kubeteach-pod-role (task11) and serviceaccount kubeteach-sa (task9) in a rolebinding named kubeteach-rolebinding              pending
default     task13   Fix broken deployment           In namespace kubeteach-troubleshoot is a deployment named fix-me, find the problem and fix this deployment                                     active

##### EVERYTHING UP AND RUNNING.
#####
##### LET DELETE kubeteach now

root@f2k8s1p:~# kubectl delete -f  https://github.com/dergeberl/kubeteach/releases/latest/download/exerciseset1.yaml
namespace "kubeteach-troubleshoot" deleted
deployment.apps "fix-me" deleted
exerciseset.kubeteach.geberl.io "set1" deleted
clusterrole.rbac.authorization.k8s.io "set1" deleted
clusterrolebinding.rbac.authorization.k8s.io "set1" deleted

root@f2k8s1p:~# kubectl delete -f https://github.com/dergeberl/kubeteach/releases/latest/download/deployment.yaml
namespace "kubeteach-system" deleted
customresourcedefinition.apiextensions.k8s.io "exercisesets.kubeteach.geberl.io" deleted
customresourcedefinition.apiextensions.k8s.io "taskdefinitions.kubeteach.geberl.io" deleted
customresourcedefinition.apiextensions.k8s.io "tasks.kubeteach.geberl.io" deleted
serviceaccount "kubeteach-controller-manager" deleted
role.rbac.authorization.k8s.io "kubeteach-leader-election-role" deleted
clusterrole.rbac.authorization.k8s.io "kubeteach-manager-role" deleted
clusterrole.rbac.authorization.k8s.io "kubeteach-metrics-reader" deleted
clusterrole.rbac.authorization.k8s.io "kubeteach-proxy-role" deleted
rolebinding.rbac.authorization.k8s.io "kubeteach-leader-election-rolebinding" deleted
clusterrolebinding.rbac.authorization.k8s.io "kubeteach-manager-rolebinding" deleted
clusterrolebinding.rbac.authorization.k8s.io "kubeteach-proxy-rolebinding" deleted
configmap "kubeteach-manager-config" deleted
service "kubeteach-controller-manager-metrics-service" deleted
deployment.apps "kubeteach-controller-manager" deleted

### CHECK

root@f2k8s1p:~# kubectl get pods --all-namespaces -o wide
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE    IP              NODE      NOMINATED NODE   READINESS GATES
kube-system   calico-kube-controllers-58497c65d5-ctpqk   1/1     Running   0          9m3s   <IP_ADDRESS>   f2k8s3p   <none>           <none>
kube-system   calico-node-5zfn9                          1/1     Running   0          9m3s   <IP_ADDRESS>      f2k8s3p   <none>           <none>
kube-system   calico-node-8f7xs                          1/1     Running   0          9m3s   <IP_ADDRESS>      f2k8s2p   <none>           <none>
kube-system   calico-node-bkdsq                          1/1     Running   0          9m3s   <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   coredns-78fcd69978-qqzbd                   1/1     Running   0          11m    <IP_ADDRESS>   f2k8s1p   <none>           <none>
kube-system   coredns-78fcd69978-x5rrx                   1/1     Running   0          11m    <IP_ADDRESS>   f2k8s1p   <none>           <none>
kube-system   etcd-f2k8s1p                               1/1     Running   0          11m    <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   kube-apiserver-f2k8s1p                     1/1     Running   0          11m    <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   kube-controller-manager-f2k8s1p            1/1     Running   0          11m    <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   kube-proxy-2bqxx                           1/1     Running   0          11m    <IP_ADDRESS>      f2k8s1p   <none>           <none>
kube-system   kube-proxy-dztqt                           1/1     Running   0          11m    <IP_ADDRESS>      f2k8s2p   <none>           <none>
kube-system   kube-proxy-nc8np                           1/1     Running   0          11m    <IP_ADDRESS>      f2k8s3p   <none>           <none>
kube-system   kube-scheduler-f2k8s1p                     1/1     Running   0          11m    <IP_ADDRESS>      f2k8s1p   <none>           <none>

### AND NOW check for Tasks:

root@f2k8s1p:~# kubectl get tasks --all-namespaces -o wide
Error from server (NotFound): Unable to list "kubeteach.geberl.io/v1alpha1, Resource=tasks": the server could not find the requested resource (get tasks.kubeteach.geberl.io)

We see, that the list kubeteach.geberl.io/v1alpha1 will remain after deleting "everything".
This is a known behavior (https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/) but should be fixed in future.

Thanks for opening an issue.
I can reproduce this behavior, but as you said i did not find any solution to fix this right now. Did I miss something?
In general I think this is not  a big issue in kubeteach. Because it is designed to run in local or test cluster. There is also no cleanup job for created objects during tasks.

for kubeteach this is only a minor open topic after clearing an existing k8s.
As mentioned from you, kubeteach is for teaching purposes. Therefore we may also close this bug. This bug need to be solved upstream.

I think close this bug should be fine.
If it is fixed upstream than there should be no task to fix it for kubeteach.
Feel free to open this issue again if something can be done from kubeteach side.


Quantitative evaluation

Hi, I'm trying PaGE-Link on another dataset (movielens).
I ran the train_linkpred.py and the pagelink.py codes without big problems, but i have some issues with the eval_explanation.py since it requires as input 2 extra data structures:

movielens_pred_pair_to_edge_labels
movielens_pred_pair_to_path_labels.

I'd like to know how the aug_citation_pred_pair_to_edge_labels and aug_citation_pred_pair_to_path_labels files in /datasets are constructed so that i can replicate them with my data and run the eval_explantion.py file.
Also, I saw that the 2 structures are loaded with the load_dataset function in the data_processing.py, but these files are used only in the eval_explantion.py. I proceeded putting as a comment the lines in which the structures are loaded for running the train_linkpred.py and the pagelink.py since the same function is used for loading the data even in these.
As a last thing I'd like to know if there are some src_nodes (author) and tgt_nodes (paper) that have both the edges 'likes' and 'writes' in your graph. It seems not to me but i wanted a confirmation.
Thank you for the attention,
ES

Thank you for your interest in our work and sorry for the late reply. Please find my answers to your questions below.

The aug_citation_pred_pair_to_edge_labels and aug_citation_pred_pair_to_path_labels are the ground truth labels for evaluating the explanations. When you are dealing with a new dataset, you may get the corresponding labels by gathering domain knowledge, or having humans label them, or using pre-defined rules to generate them (our case). The first and the second approaches are actually better if you are able to perform them. In our case, we proceed with the third and defined two rules to generate these labels, i.e., concise and informative (please refer to section 6.1 for more details). I will try to update the corresponding code to the repo as soon as I can.

For the load_dataset function, your suggestion is great, which will help to save unnecessary IO. I will definitely consider make such changes in our next update.

No we don't have both "likes" and "writes" between the same pair of author and paper, so your understanding is correct. This is because we do not want to predict an existing edge between two nodes (even in different edge types), so we explicitly avoid creating those "likes" edges for prediction. In practice, you can think of the "likes" relation as an recommendation of new papers for authors to read, so there is no need to recommend their own papers.

Thank you for your answer, I think it could be useful if you can provide an example of how you generated theese files for a better comprehension


Closing Paymetheus doesn't close the background instance of DCRD

This prevents the full version of Paymetheus ("Decred") from starting up properly next time, throwing an error: "Consensus server exited with code 1"
dcrd.exe has to be manually stopped in task viewer for Decred to successfully launch again.
Windows 7 64 bit
Paymetheus version: decred_0.8.2-beta_x64.msi

We've definitely tested this so I'm unsure why it is not working for you.  Can you grab the end of the dcrd logs (in %localappdata%/dcrd/logs/mainnet/dcrd.log)?

Sure, the log is attached.
My case isn't really the normal end user case, so maybe I uncovered something not specifically tested for. I had only fired up Paymetheus to try helping a forum user.  Normally I use the CLI versions so my AppData/Local had existing data from my dcrd.exe instance.  Also, I didn't create a wallet in Paymetheus, so that could be it too.  I am closing Paymetheus (the "Decred" version) from the "Create a new wallet" and "Restore a wallet from seed" screen.
dcrd[4-11].txt

If you can reproduce this without any other dcrd running then feel free to reopen, but I'm going to close this now since running both the Decred shortcut and your own node separately isn't supported (this is what the standalone version is for).  From those logs it appears that there are multiple dcrd instances that were writing to the log file all at the same time.


[SPARK-29054][SS] Invalidate Kafka consumer when new delegation token available

What changes were proposed in this pull request?
Kafka consumers are cached. If delegation token is used and the token is expired, then exception is thrown. Such case new consumer is created in a Task retry with the latest delegation token. This can be enhanced by detecting the existence of a new delegation token. In this PR I'm detecting whether the token in the consumer is the same as the latest stored in the UGI (targetServersRegex must match not to create a consumer with another cluster's token).
Why are the changes needed?
It would be good to avoid Task retry to pick up the latest delegation token.
Does this PR introduce any user-facing change?
No.
How was this patch tested?
Existing + new unit tests.
Additionally executed the following code snippet to measure ensureConsumerHasLatestToken time consumption:
    val startTimeNs = System.nanoTime()
    for (i <- 0 until 10000) {
      consumer.ensureConsumerHasLatestToken()
    }
    logInfo(s"It took ${TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs)} ms" +
      " to call ensureConsumerHasLatestToken 10000 times")

And here are the results:
19/09/11 14:58:22 INFO KafkaDataConsumerSuite: It took 1058 ms to call ensureConsumerHasLatestToken 10000 times
...
19/09/11 14:58:23 INFO KafkaDataConsumerSuite: It took 780 ms to call ensureConsumerHasLatestToken 10000 times
...
19/09/11 15:12:11 INFO KafkaDataConsumerSuite: It took 1032 ms to call ensureConsumerHasLatestToken 10000 times
...
19/09/11 15:12:11 INFO KafkaDataConsumerSuite: It took 679 ms to call ensureConsumerHasLatestToken 10000 times

Test build #110476 has started for PR 25760 at commit 7db58e8.

Can one of the admins verify this patch?

Test build #110476 has finished for PR 25760 at commit 7db58e8.

This patch fails Spark unit tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test FAILed.

Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/110476/
Test FAILed.

Test build #110478 has started for PR 25760 at commit 32ff76f.

Test build #110478 has finished for PR 25760 at commit 32ff76f.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/110478/
Test PASSed.

Ahhh, there is a not necessary change on the branch. Doing the resurrection...

Can one of the admins verify this patch?

Test build #110641 has started for PR 25760 at commit f872b92.

Test build #110641 has finished for PR 25760 at commit f872b92.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/110641/
Test PASSed.

Thanks guys, valid comments. Let me think it though and update it soon...

Test build #111012 has started for PR 25760 at commit 381db79.

Not sure if that's what you've meant by the comments but after the restructuring big step ahead. Please have another round and share your thoughts.

Test build #111018 has started for PR 25760 at commit 703cf9c.

Test build #111020 has started for PR 25760 at commit e4d01bc.

Test build #111012 has finished for PR 25760 at commit 381db79.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111012/
Test PASSed.

Test build #111018 has finished for PR 25760 at commit 703cf9c.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111018/
Test PASSed.

Test build #111020 has finished for PR 25760 at commit e4d01bc.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111020/
Test PASSed.

Test build #111068 has started for PR 25760 at commit 205c19a.

Test build #111069 has started for PR 25760 at commit dcaac2b.

Test build #111068 has finished for PR 25760 at commit 205c19a.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111068/
Test PASSed.

Test build #111069 has finished for PR 25760 at commit dcaac2b.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111069/
Test PASSed.

Test build #111431 has started for PR 25760 at commit 120f634.

Test build #111432 has started for PR 25760 at commit 71d7223.

Test build #111431 has finished for PR 25760 at commit 120f634.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111431/
Test PASSed.

Test build #111432 has finished for PR 25760 at commit 71d7223.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111432/
Test PASSed.

Test build #111479 has started for PR 25760 at commit d5379fa.

Test build #111479 has finished for PR 25760 at commit d5379fa.

This patch fails Spark unit tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test FAILed.

Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111479/
Test FAILed.

Test build #111483 has started for PR 25760 at commit f23c533.

Test build #111483 has finished for PR 25760 at commit f23c533.

This patch fails Spark unit tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test FAILed.

Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111483/
Test FAILed.

Test build #111487 has started for PR 25760 at commit 23fd53e.

Test build #111487 has finished for PR 25760 at commit 23fd53e.

This patch fails Spark unit tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test FAILed.

Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111487/
Test FAILed.

retest this please

Test build #111495 has started for PR 25760 at commit 23fd53e.

Test build #111495 has finished for PR 25760 at commit 23fd53e.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111495/
Test PASSed.

Test build #111604 has started for PR 25760 at commit 75e6e95.

Test build #111604 has finished for PR 25760 at commit 75e6e95.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111604/
Test PASSed.

Test build #111624 has started for PR 25760 at commit e36cc13.

Test build #111624 has finished for PR 25760 at commit e36cc13.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111624/
Test PASSed.

Test build #111668 has started for PR 25760 at commit a7181a5.

Test build #111668 has finished for PR 25760 at commit a7181a5.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111668/
Test PASSed.

Test build #111717 has started for PR 25760 at commit b1b2287.

Test build #111717 has finished for PR 25760 at commit b1b2287.

This patch fails due to an unknown error code, -9.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test FAILed.

Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111717/
Test FAILed.

retest this please

Test build #111725 has started for PR 25760 at commit b1b2287.

Test build #111725 has finished for PR 25760 at commit b1b2287.

This patch passes all tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test PASSed.

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/111725/
Test PASSed.

Merging to master.


Plugin author mailing list and notifications

I think it's a great idea to spin the platforms out into separate repositories -- it keeps the core clean and enables plugin authors to handle issues without polluting your core codebase.
Is it possible to have a mailing list or forum for those of us who are building plugins so we can keep up to date on changes which impact platform? I would like to keep isy-js up to date but I find it difficult to keep checking back looking for changes. You're moving fast and I'd like to keep up. :)

So we have a Slack channel that we could use more; I just added that to the README. We could create an "announcements" channel that people could sign up for email notifications on - or do you think an email group is also needed?

Let's try using Slack for now - I made the #general channel readonly, so you can sign up for email notifications there and chat about stuff in the #plugins channel.


Updates needed for mesos 1.8-rc2 with a AMI release

Summary:
In order to do seccomp we need to run against Mesos 1.8.   This brings us up to 1.8-rc2 which is the latest.
JIRA issues:

issue with mesos on this version of OS
WARN [18:10:56 LaunchQueueIntegrationTest-MesosMaster-32824] mesos: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.27' not found (required by mesos)
WARN [18:10:56 LaunchQueueIntegrationTest-MesosAgent-32825] mesos: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.27' not found (required by mesos)

Failing because Mesos 1.8 is installed correctly on Debian 9
blocked by https://jira.mesosphere.com/browse/DCOS_OSS-5096

Till/Benno and team will provide a new release of mesos for debian with seccomp

Somehow our subnet to build the AMI was removed. I filed a ticket https://jira.mesosphere.com/browse/DCOS-54243.

What's the state of this @kensipe and @ANeumann82?


chore: Release 1.5.0

Describe your changes
Issue ticket number and link
Checklist before requesting a review

[x] I updated CHANGELOG.MD with a description of my changes
[x] If the change was to a module, I ran the code validation script (scripts/validate.sh)
[x] If the change was to a module, I have added thorough tests
[x] If the change was to a module, I have added/updated the module's README.md
[x] If a module was added, I added a reference to the module to the repository's README.md
[x] I verified that my code deploys successfully using seedfarmer apply

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

AWS CodeBuild CI Report

CodeBuild project: AIOpsModuleTestsInfrastruct-doMLXEYsnmxr
Commit ID: c9b51d5346d287ff2a9a51b84084efb1809e95e2
Result: SUCCEEDED
Build Logs (available for 30 days)

Powered by github-codebuild-logs, available on the AWS Serverless Application Repository


Simplify 'processConstraint' and 'addMissingValueSpent'

(follow-up from https://github.com/input-output-hk/plutus/pull/2534)

Replace valueSpentRequired and valueSpentActual by a single field valueSpentBalance
valueSpentBalance holds the difference (valueSpentRequired - valueSpentActual)
I decided not to follow the second suggestion of adding up inputs and outputs at the end. Transaction inputs (of type TxIn) don't have a value attached to them, so we'd have to use lookupTxOutRef in order to get the value. But we already do this lookup when we handle MustSpendPubKeyOutput in processConstraint, so we can just add modify valueSpentBalance in there.

Pre-submit checklist:

Branch

[x] Commit sequence broadly makes sense
[x] Key commits have useful messages
[x] Relevant tickets are mentioned in commit messages

PR

[ ] Self-reviewed the diff
[x] Useful pull request description
[x] Reviewer requested

If you updated any cabal files or added Haskell packages:

[ ] $(nix-build default.nix -A pkgsLocal.updateMaterialized) to update the materialized Nix files
[ ] Update hie-*.yaml files if needed

If you changed any Haskell files:

[x] $(nix-shell shell.nix --run fix-stylish-haskell) to fix any formatting issues

If you changed any Purescript files:

[ ] $(nix-shell shell.nix --run fix-purty) to fix any formatting issues

Pre-merge checklist:

[ ] Someone approved it
[ ] Commits have useful messages
[ ] Review clarifications made it into the code
[ ] History is moderately tidy; or going to squash-merge

Going to merge this now as the change is mechanical (applying valueSpentBalance = valueSpentRequired - valueSpentActual)
Regardless of this PR we need to have another look at the constraints library. I created an issue in JIRA for that.


Hierarchical Edge Bundling With a tree

Hello is there an interactive gist to this Hierarchical Edge Bundling in a matrix/ tree layout , the link for Dan's interactive sample/code is broken

Hello can you please help with this.


moved netcdf-4 tests from nc_test to nc_test4

In this PR I moved three tests, which are only run if netcdf-4 is built, from nc_test to nc_test4.
This is necessary for fully splitting HDF5 from libsrc4, but makes sense also because it simplifies the build systems while still running the same tests as it has been running.
Part of #856.
Part of #738.

The failures in Travis are due to the HDF5 plugin file copy issues.

Thanks, @edhartnett ! I've also merged a change reducing the number of travis tests in half, on master.  This should speed things up.  We sacrifice testing clang and gcc, but until we have another demonstrated error, the speedup is worth it.

Awesome, thanks!


Implement puppy CRUD

Fixes #310
Review of colleague's PR #283
Changes proposed in this PR:

Implement a Puppy struct containing ID, Breed, Colour, Value
Create Storer interface with crud methods for Puppy
Implement MapStore a Storer backed by a map
implementation SyncStore a Storer backed by a sync.Map
Add test suite for Storer interface testing both implementations

Codecov Report

Merging #309 into master will not change coverage.
The diff coverage is 100%.

@@          Coverage Diff          @@
##           master   #309   +/-   ##
=====================================
  Coverage     100%   100%
=====================================
  Files          20     21    +1
  Lines         328    363   +35
=====================================
+ Hits          328    363   +35

Impacted Files
Coverage Δ

06_puppy/nicholascross/main.go
100% <100%> (ø)

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update b64a6e9...d9a3b69. Read the comment docs.

The go-course is now closed. Thank you very much for participating.


Not owned tracks

It would be nice to see how many of my fav series use the same tracks.
So you know which ones to buy.

I think this is now covered by the purchase guide 👍


Handle SIGINT to stop the OpenCR if we suddenly close the program

Related to #60

SIGINT is handled by default and throws the KeyboardInterrupt exception. Therefore we don't need to modify the way it's handled : we only need to catch the exception.


Add calibrate_decision_threshold

Issue #, if available:
resolves #1038, #1515
pre-req for #3112
Description of changes:
Add TabularPredictor.calibrate_decision_threshold which allows to optimize a given metric's decision threshold for predictions to strongly enhance the metric score.
The below code and output showcase how this logic can reliably improve metric scores on a variety of metrics, particularly f1 and balanced_accuracy.

Example Code
from autogluon.tabular import TabularPredictor, TabularDataset

if __name__ == '__main__':
    path_prefix = 'https://autogluon.s3.amazonaws.com/datasets/Inc/'
    label = 'class'

    path_train = path_prefix + 'train.csv'
    path_test = path_prefix + 'test.csv'

    train_data = TabularDataset(path_train)
    train_data = train_data.sample(25000, random_state=0)
    test_data = TabularDataset(path_test)

    predictor: TabularPredictor = TabularPredictor(
        label=label,
        verbosity=3,
    ).fit(
        train_data=train_data,
        hyperparameters={'GBM': {}},
        # fit_weighted_ensemble=False,
        num_bag_folds=2,
        num_bag_sets=1,
    )
    predictor.leaderboard(test_data)

    from autogluon.core.metrics import accuracy, balanced_accuracy, f1, mcc, precision, recall
    y_pred = predictor.predict(data=test_data)
    result = predictor.evaluate_predictions(y_true=test_data[label], y_pred=y_pred)
    decision_threshold_dict = dict()
    for eval_metric in [accuracy, balanced_accuracy, f1, mcc, precision, recall]:
        decision_threshold = predictor.calibrate_decision_threshold(metric=eval_metric)
        decision_threshold_dict[eval_metric.name] = decision_threshold

        y_pred_w_decision_threshold = predictor.predict(data=test_data, decision_threshold=decision_threshold)
        result_calibrated = predictor.evaluate_predictions(y_true=test_data[label], y_pred=y_pred_w_decision_threshold)

        metric_score = result[eval_metric.name]
        metric_score_calibrated = result_calibrated[eval_metric.name]

        decision_threshold_dict[eval_metric.name] = dict(
            decision_threshold=decision_threshold,
            metric_score=metric_score,
            metric_score_calibrated=metric_score_calibrated,
        )

    for name, result_dict in decision_threshold_dict.items():
        decision_threshold = result_dict['decision_threshold']
        metric_score = result_dict['metric_score']
        metric_score_calibrated = result_dict['metric_score_calibrated']
        print(f'decision_threshold={decision_threshold:.3f}\t| metric="{name}"'
              f'\n\ttest_score uncalibrated: {metric_score:.4f}'
              f'\n\ttest_score   calibrated: {metric_score_calibrated:.4f}'
              f'\n\ttest_score        delta: {metric_score_calibrated-metric_score:.4f}')

Example Output
decision_threshold=0.500	| metric="accuracy"
	test_score uncalibrated: 0.8703
	test_score   calibrated: 0.8703
	test_score        delta: 0.0000
decision_threshold=0.220	| metric="balanced_accuracy"
	test_score uncalibrated: 0.7925
	test_score   calibrated: 0.8476
	test_score        delta: 0.0551
decision_threshold=0.380	| metric="f1"
	test_score uncalibrated: 0.7022
	test_score   calibrated: 0.7245
	test_score        delta: 0.0222
decision_threshold=0.380	| metric="mcc"
	test_score uncalibrated: 0.6243
	test_score   calibrated: 0.6347
	test_score        delta: 0.0105
decision_threshold=0.990	| metric="precision"
	test_score uncalibrated: 0.7713
	test_score   calibrated: 1.0000
	test_score        delta: 0.2287
decision_threshold=0.000	| metric="recall"
	test_score uncalibrated: 0.6445
	test_score   calibrated: 1.0000
	test_score        delta: 0.3555

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

I've added a major update to the PR.
Added logic:

Support for leaderboard
Support for evaluate / evaluate_predictions
Support for predict_multi
Support for .fit(…, calibrate_decision_threshold=True)
Support for updating the default decision_threshold via predictor.set_decision_threshold
Support for refit_full models
Support for get_oof_pred
Tests for all of the above
Various logging / docstrings

Benchmark results show no changes from this PR, as expected.

Would be great if this was included in MultiModalPredictor


[Bugfix] Handle mixed block and inline content in submit & compare feedback

The reported bug was that there wasn't enough spacing between paragraphs here:

Upon investigation, I found that the content in that feedback consisted of a text node, a paragraph, and a text node. That is not valid slate-js content.
Side Note: upon opening it in authoring, the content would get normalized and fixed if the user made any change whatsoever to the page.
The feedback was being passed through ensureParagraphs in questions/common.ts which I think was meant to prevent things like this, but ensureParagraphs only catches the case where all the children are inline, not when they're partially inline and partially block elements.
In this PR, there is a new wrapInlinesWithParagraphs that handles any combination of inline & block elements, wrapping all the inline, and keeping adjacent inlines in the same paragraph, but that's only used in one very specific case. It technically fixes this bug, and in the most surgical way possible.
If we were early in the process, I would advocate for swapping out  the implementation of ensureParagraphs with wrapInlinesWithParagraphs for all content.
But when I tried that, it resulted in over 14,000 changes in the chemistry course. The chances of this causing unexpected content changes is a near certainty.
A few options on where to go next:

Only apply this new method as surgically as possible like this PR does as issues come up.
Apply this more broadly, but selectively. Such as to all feedback, all hints, or some subset of elements.
Apply it everywhere, and deal with the consequences.

Fore reference, the input XML we get is:
<explanation>Because the atomic mass on the periodic table is very close to 39, you
                    can conclude that the most abundant of the 3 isotopes is K-39.
In fact the
                        percent abundances are: K-39 is 93.3%. K-41 is 6.7%. K-40 is 0.012%.
 It
                    is common that elements have one isotope that is far more abundant than the
                    other (although not always the case). If the atomic mass on the periodic table
                    is very close to a whole number, it is safe to assume that if you rounded that
                    value to the nearest whole number you would have the mass number of the most
                    common isotope for that element. Like we see with potassium.</explanation>

For reference, the input XML we get is:
<explanation>Because the atomic mass on the periodic table is very close to 39, you
                    can conclude that the most abundant of the 3 isotopes is K-39.
In fact the
                        percent abundances are: K-39 is 93.3%. K-41 is 6.7%. K-40 is 0.012%.
 It
                    is common that elements have one isotope that is far more abundant than the
                    other (although not always the case). If the atomic mass on the periodic table
                    is very close to a whole number, it is safe to assume that if you rounded that
                    value to the nearest whole number you would have the mass number of the most
                    common isotope for that element. Like we see with potassium.</explanation>

Is it possible that the visual case that folks are complaining about is a special case of explanation (which should, but isn't, enforcing paragraphs in tag)? Or do we think this is showing up in multiple places?
Is it possible to implement your suggested fix ("If we were early in the process, I would advocate for swapping out the implementation of ensureParagraphs with wrapInlinesWithParagraphs for all content.") AFTER we ingest Chemistry? IE if this is generally the right answer, but is problematic in the chem process, should we consider that reimplementation for future ingestion?

@normanbier Yes, it's possible. So far, all of the changes I tracked down and looked at were visually identical before/after except for these feedback items.  With so many potential changes, I'm cautious on claiming that's always the case.
I have since found that we are following this approach in some other targeted cases, such as list-items and question stems. (there's a similar function wrapLooseText I had overlooked). That doesn't change the core issue, but probably lends some weight to the targeted-approach method.
We could certainly change how course-digest works for different courses over time.

Since normalization logic would effectively be doing the same thing as the proposed change here, would it not be better to just do it up front during course-digest so that we know what changes this might cause rather than wait until someone edits the page?

@eliknebel That's definitely another thing to consider. Right now, on most chemistry pages, every every time we open any page in authoring, we're getting a whole bunch of normalization rules firing and modifying things. I just opened a sample page and hit 14 of these top-level text rules fired.
Having a full-compliant output that didn't need any additional normalization would be a great goal and prevent any of the "I edited the doc, and this weird thing happened" bugs.

I first dealt with this issue in the context of question stems in chemistry, adding wrapLooseText routine and applying wrapLooseText(ensureParagraphs(<content>)). Later applied that also to fixup of list items when block content was added to list items.  Because of that history, this "fixup" operates on the javascript content representation formed after common restructuring transformations of the XML DOM, and also uses the two routines rather than one. It might be more appropriate to do it as part of DOM restructuring, but that is minor -- still have to target all elements to which it must be applied,

As a general matter, better not to leave in two different routines doing nearly the same thing (wrapLooseText + ensureParagraphs vs wrapInlinesWithParagraphs).

Looks like wrapInlinesWithParagraphs duplicates collectTextsIntoParagraphs. Shouldn't we have only one? The former uses more accurate terminology -- more general "inline" as opposed to "text" -- but I think they have exactly the same effect.

@andersweinstein yes, I'm going to combine them. Didn't find wrapLooseText until my comment above mentioning it.

I did an export with option 3 above, where we run this for all ensureParagraph calls. I've been looking through all the differences in two ways. I scanned through the diff of hundreds of output files, and visually compared the rendered output of dozens of them.
Overall, I only found 3 types of unexpected visual changes and fixed those three. They all followed a pattern of putting in extra line breaks around elements that should have been inline and were obvious to catch in the diff scan I did (example attached).
I have more confidence in that option now. There still is a risk I missed something in my review, but that might be our best option now.


Add JSON serialization to big number types

This PR:

Closes #7856

Summary of Changes:

Allow serialisation: and .to_json to all Big* structs (BigInt,BigFloat, BigDecimal)

Serialisation Big*.to_json is enabled through adding JSON::Builder.number methods for each Big* type, serialising into Number types

Allow deserialisation of BigFloat from Int or Float

Allow all Big* structs to be used as JSON keys: add .to_json_object_key

Add missing pending_win32 "does for BigDecimal" to yaml/serialization_spec.cr

Further Discussion:

This PR seeks to resolve #7856, and is built from the exchanges in https://github.com/crystal-lang/crystal/pull/8070 .

The reason for opening this PR is because issue #7856 should be resolved swiftly as Big* finds lots of usage in terms of serialisation, yet https://github.com/crystal-lang/crystal/pull/8070 has been open and without activity for almost a year. In this PR, all comments from PR #8070 are addressed, e.g. disallowing BigInt and BigFloat to be deserialised from a String, allowing Big* to only serialise into Number and not String.

With regards to RX14's suggestion on building a custom converter to serialize/ deserialize Big* structs into/ from String (https://github.com/crystal-lang/crystal/pull/8070#issuecomment-525241993), this can be done as a separate PR, or if the maintainers would like this to be addressed here, I would not mind adding it in. However, IMHO, I think the commit in this PR currently is adequate to resolve the original issue, and any other modification can be addressed in a separate PR.

Thank you @dukeraphaelng

Seems now we're just missing the same thing for YAML...


Using a newer version of pymapdl reader.

Not forcing any particular version, just lower bound.
This close #979 by updating pymapdl reader version.

Using 0.51.6 because it is when the deprecation warnings were fixed.

I recall that our doc build fails when using the latest. Try modifying the version just to test. Something like …dev1

Interestingly...
the nightly docs build fails with this branch: https://github.com/pyansys/pymapdl/runs/5585437476?check_suite_focus=true
however the github actions workflow seems to work: https://github.com/pyansys/pymapdl/actions/runs/1998389547/attempts/1
I'm double checking this by re-running this again.

@germa89, looks like it's fixed in ansys-mapdl-reader. I'm going to still pin it just so we're sure it's why our docs are failing.

@germa89, looks like a full doc rebuild works as well.

Thank you @akaszynski for this fix.


beta 1

[x] move result from console to UI
[x] make click button bigger
[x] show cps
[ ] add timer (moved to beta 2)

https://github.com/sickpancake/cps-tester/commit/4e86a524248ead968d556bce387eb48b7bbdbdb4


[SPARK-11080] [SQL] Throw exception when NamedExpression.newExprId is called inside tasks

My understanding of NamedExpression.newExprId is that it is only intended to be called on the driver. If it is called on executors, then this may lead to scenarios where the same expression id is re-used in two different NamedExpressions.
More generally, I think that calling NamedExpression.newExprId within tasks may be an indicator of potential attribute binding bugs. Therefore, I think that we should throw an exception when this method is called inside of tasks.

Merged build triggered.

Merged build started.

Test build #43632 has started for   PR 9093 at commit 48e3d1c.

Test build #43632 has finished for   PR 9093 at commit 48e3d1c.

This patch fails Spark unit tests.
This patch merges cleanly.
This patch adds no public classes.

Merged build finished. Test FAILed.

Test FAILed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/43632/
Test FAILed.

Digging into the test failures here, it looks like a bunch of them are caused by the BindReferences call inside of AggregationIterator:
[info]   Caused by: org.apache.spark.sql.catalyst.errors.package$TreeNodeException: makeCopy, tree: hyperloglogplusplus(a#4,0.04)
[info]   	at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:49)
[info]   	at org.apache.spark.sql.catalyst.trees.TreeNode.makeCopy(TreeNode.scala:315)
[info]   	at org.apache.spark.sql.catalyst.trees.TreeNode.transformChildren(TreeNode.scala:280)
[info]   	at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:232)
[info]   	at org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:217)
[info]   	at org.apache.spark.sql.catalyst.expressions.BindReferences$.bindReference(BoundAttribute.scala:85)
[info]   	at org.apache.spark.sql.execution.aggregate.AggregationIterator.<init>(AggregationIterator.scala:93)
[info]   	at org.apache.spark.sql.execution.aggregate.SortBasedAggregationIterator.<init>(SortBasedAggregationIterator.scala:29)
[info]   	at org.apache.spark.sql.execution.aggregate.SortBasedAggregate$$anonfun$doExecute$1$$anonfun$2.apply(SortBasedAggregate.scala:86)
[info]   	at org.apache.spark.sql.execution.aggregate.SortBasedAggregate$$anonfun$doExecute$1$$anonfun$2.apply(SortBasedAggregate.scala:72)
[info]   	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$20.apply(RDD.scala:700)
[info]   	at org.apache.spark.rdd.RDD$$anonfun$mapPartitions$1$$anonfun$apply$20.apply(RDD.scala:700)
[info]   	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
[info]   	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
[info]   	at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
[info]   	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
[info]   	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
[info]   	at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
[info]   	at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
[info]   	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:297)
[info]   	at org.apache.spark.rdd.RDD.iterator(RDD.scala:264)
[info]   	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
[info]   	at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
[info]   	at org.apache.spark.scheduler.Task.run(Task.scala:88)
[info]   	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
[info]   	... 3 more
[info]   Caused by: java.lang.reflect.InvocationTargetException
[info]   	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
[info]   	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
[info]   	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
[info]   	at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
[info]   	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$makeCopy$1$$anonfun$apply$10.apply(TreeNode.scala:326)
[info]   	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$makeCopy$1$$anonfun$apply$10.apply(TreeNode.scala:325)
[info]   	at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:51)
[info]   	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$makeCopy$1.apply(TreeNode.scala:323)
[info]   	at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$makeCopy$1.apply(TreeNode.scala:315)
[info]   	at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:48)
[info]   	... 27 more
[info]   Caused by: java.lang.IllegalStateException: Expression ids should not be allocated inside of tasks
[info]   	at org.apache.spark.sql.catalyst.expressions.NamedExpression$.newExprId(namedExpressions.scala:30)
[info]   	at org.apache.spark.sql.catalyst.expressions.AttributeReference$.apply$default$5(namedExpressions.scala:184)
[info]   	at org.apache.spark.sql.catalyst.expressions.aggregate.HyperLogLogPlusPlus$$anonfun$2.apply(functions.scala:546)
[info]   	at org.apache.spark.sql.catalyst.expressions.aggregate.HyperLogLogPlusPlus$$anonfun$2.apply(functions.scala:545)
[info]   	at scala.collection.generic.GenTraversableFactory.tabulate(GenTraversableFactory.scala:149)
[info]   	at org.apache.spark.sql.catalyst.expressions.aggregate.HyperLogLogPlusPlus.<init>(functions.scala:545)
[info]   	... 37 more (QueryTest.scala:78)

Here, it looks like the copying of HyperLogLogPlus is causing problems because that ends up changing the expressionIds of its aggBufferAttributes.

Merged build triggered.

Merged build started.

Test build #43662 has started for   PR 9093 at commit 955a1a8.

Now what this PR did doesn't conform to the title, can you update title and description?

@cloud-fan, yep, planning to update shortly.

Test build #43662 has finished for   PR 9093 at commit 955a1a8.

This patch passes all tests.
This patch merges cleanly.
This patch adds the following public classes (experimental):
class ChildProcAppHandle implements SparkAppHandle
abstract class LauncherConnection implements Closeable, Runnable
final class LauncherProtocol
 static class Message implements Serializable
 static class Hello extends Message
 static class SetAppId extends Message
 static class SetState extends Message
 static class Stop extends Message
class LauncherServer implements Closeable
class NamedThreadFactory implements ThreadFactory
class OutputRedirector
case class ExprId(id: Long, jvmId: UUID)

Alright, updating now....

Test PASSed.
Refer to this link for build results (access rights to CI server needed):
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/43662/
Test PASSed.

Merged build finished. Test PASSed.

Updated; PTAL @marmbrus.

LGTM pending tests

It already passed tests as of the latest commit.

Merging to master.

Cherry-picked into branch-1.5, to fix https://issues.apache.org/jira/browse/SPARK-11885


Edge detection struggles on non-credit cards objects when preset resolution is higher than 640x480

I am attempting to repurpose card.io to be able to perform some basic edge and corner detection on paper documents. By setting detectionMode to CardIODetectionModeCardImageOnly, I'm able to successfully capture the edges and corners of paper documents by yoking the UIImage out from the CardIOCreditCardInfo object.
That being said, the output quality is low, and I gather from @dgoldman-ebay's comment https://github.com/card-io/card.io-iOS-source/issues/16#issuecomment-77111035 that this is expected. I upped the resolution to AVCaptureSessionPreset1920x1080 and it fails to detect the borders and edges of paper documents in the green enclosure.
Is this due to the nature of the edge detection pipeline? How feasible is it to implement scanning paper documents at high preset resolutions without losing its border/edge detection prowess that's available at lower preset resolutions?

I haven't thought hard about this in a long time. That said, I strongly suspect that what is going on is that there are a bunch of magic constants in the edge and line detection code that are tuned for the number of pixels found in 640x480 images. There was a lot of fine-tuning that went into getting the existing system to be highly responsive. The relevant numbers were determined experimentally, not mathematically, so if my suspicion is correct, the next step here would be to run similar experiments using higher resolution. If you're interested in tackling this, off the top of my head, I'd say the numbers to investigate are mainly the various canny and hough thresholds.

I worked on higher resolutions at one point, and IIRC these days we can indeed detect a card-shaped rectangle at any camera resolution.
You can increase the camera resolution inside CardIOVideoStream.mm - there's commented-out code in there about this.
However, eventually you'll probably run into preset expectations about the width/height ratio of the desired rectangle, as @josharian indicates. If your paper documents have a predictable shape, you might be able to adjust those expectations accordingly. Or remove them, perhaps.

@irvingruan, did you find a possible solution for this ? I am running into similar issues with card scanning of non-payment cards.
Any help is appreciated.

@biksappdev: I did, but it was in another SDK that I found. Given the complexity of modifying card.io's library for arbitrary polygons, I decided to not use it.
FWIW, the wonderful folks at Scanbot (https://scanbot.io/) have a packaged SDK that does precisely what I originally wanted. It's closed source and costs money, though.

@irvingruan , thanks a lot for the reply. I dropped a mail to you regarding some queries, could you please respond when you have appropriate time for it, Thanks.

Hi all I'm trying to reuse this source code to scan and crop another card type (warranty card). I'm trying to increase camera resolution for better result. But i got stuck on it. So could you please help me to increase camera resolution to higher! Or please told me what have i do to change it. thanks you very much :)


Update documented time complexity.

According to #144, this documentation was a holdover from the previous implementation.

Thanks!


Error while reading world files for modded maps

I'm seeing this lately for some of my modded maps.
java.lang.UnsupportedOperationException: Found unexpected Value 2 at 1438
at qowyn.ark.ArkSavegame.readBinary(ArkSavegame.java:145)
at qowyn.ark.ArkSavegame.(ArkSavegame.java:78)
at qowyn.ark.tools.CreatureListCommands.listImpl(CreatureListCommands.java:109)
at qowyn.ark.tools.CreatureListCommands.wild(CreatureListCommands.java:67)
at qowyn.ark.tools.App.main(App.java:114)
It was working for the world, however, after a wild dino wipe, it stopped working, and still continues to not work even after players have logged into the world and the world has been re-launched and saved to see if it helps.
Here's the command arguments used:
wild \ark-3\ShooterGame\Saved\SavedArks\ShigoIslands.ark \Wild\ark-3\ -p --pretty-printing
Attached is one of the world files exhibiting the issue. Hope it helps.
ShigoIslands.zip

Apologies, this is for Ark-Tools, not ark-savegame-toolkit. If you're able to move it to that issue tracker, that would be good :)

Thats actually been a bug in ark-savegame-tookit :) and it's now fixed with the new releases of both.


.NET Core 3.0/3.1 support

Are there any plans to port this project to .NET Core 3.1? Seeing as .NET Framework is slowing down, and the future of the framework (.NET 5) is an extension of .NET Core 3.1, I think it's the logical move to do a port.

In theory this should be pretty straightforward - the bulk of the code is DLL imports and, some simple classes to wrap GL objects and then Windows controls. All of this functionality is available in .NET Core 3.0 onwards.
As it's been a few years since I've been doing enterprise .NET, one question I would have is on compatibility - is this likely to cause issues with people who are already using the code in large .NET Framework-based codebases, or would they be able to add a .NET Core based dependency?
Even if the legacy issue remains, it would probably still make sense to make a new Major release targeting .NET Core, and then maintain a legacy release branch for anyone who needs .NET framework support.
WDYT?

You can target multiple frameworks in a csproj file when using the new SDK style. The resulting nuget package then contains .NET Core and traditional .NET Framework or .NET Standard assemblies. This way, you don't need to maintain separate code branches.

OK I'm working on this now in a new branch, hopefully will have updates very soon!

OK it's done! I've written up the experiences here 😄
https://dwmkerr.com/modernising-dotnet-projects/
If anyone has any problems open up an issue and I'll look into them. Am planning to spend some time cleaning up the library a bit and looking at support for MacOS and Linux.


Expand usage of Advisory Speed Limit field

The Advisory Speed Limit field is now shown either by default or in More Fields for the Motorway Link, Trunk Link, Primary Link, Secondary Link, and Tertiary Link presets.
In Canada and the United States, if not elsewhere, advisory speed limits are normally posted at entrance and exit ramps but regulatory speed limit signs are rarely posted in these situations. Tagging maxspeed on a ramp could potentially lead to a more dangerous experience for users reliant on navigation UI to remind them about the most relevant speed limit. Mappers unaware of the distinction or unaware of the separate maxspeed:advisory key have been inaccurately setting maxspeed to the advisory speed limit. Presenting the normal field by default will hopefully make such errors less common.
When this point was originally raised in openstreetmap/iD#4870, it generated some controversy, which this PR attempts to sidestep by limiting the changes to places where it’s known to be correct. Unfortunately, the schema lacks an affordance for saying that a field should be under fields in some locations but moreFields in other locations. As an ugly workaround, this PR duplicates the Speed Limit and Advisory Speed Limit fields everywhere they’re used. At some point, we’ll need to come up with a real solution before we can consider other variations of speed limits, such as maxspeed:hgv, to avoid a proliferation of similar fields.
This PR also adds the Advisory Speed Limit field to the Motorway, Trunk Road, Primary Road, Secondary Road, and Tertiary Road presets, but under More Fields. Advisory speed limits are tagged on main roads in cases where road geometry requires extra care, such as around curves or hidden driveways.

Did you consider to instead duplicate the *_link presets themselves instead of the the maxspeed* fields?

Oh, good idea. It would still be messy but not as messy.

PS: not sure if mutcd is a universally known acronym (I had to google it 😅). Perhaps one could use _ca_us as a suffix, since that is what the locationSet setting actually uses?

I think you’re right about that being a better suffix. Variations of the MUTCD are used in some other countries too, but as far as I know, ramp advisory speed limit signs aren’t used in all of them.


IPv4/single: applied MISRA to FreeRTOS_ARP.c

From time to time we check if the code is still complying to the MISRA rules.
This PR presents some related change for FreeRTOS_ARP.c.
By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

The changes proposed in this PR also appear in PR #357, and they will be merged to main from there.
Closing this PR now.


[SPARK-47431][SQL][FOLLOWUP] Do not access SQLConf in Literal.apply

What changes were proposed in this pull request?
Refactor of literals.scala.
Why are the changes needed?
Literal expression is too low-level and should not access SQLConf, otherwise it's hard to reason about when/where the default collation is applied.
Does this PR introduce any user-facing change?
No.
How was this patch tested?
Tests already available from initial PR.
Was this patch authored or co-authored using generative AI tooling?
No.

thanks, merging to master!


Check_DB2_HADR_sh.log - file size becoming large

Hi,
We are using this to monitor DB2_HADR however now the log file have become large and consuming lots of storage. Can we delete this log file or back it up at some different location.
Please let me know impact of moving the log file from /tmp location as the storage have nearly become full. Please advice.
Thanks
Kunal

Hi,
I just want to be sure you are using the script "check_hadr_status" to check your HADR cluster. It seems to me that you are using another one.
I ask that, because this script does not generate the file you are indicating, the "Check_DB2_HADR_sh.log" file.
Instead, it creates a file called "check_hadr_status.log" in the /tmp file.
The only case that this file can have a name like the one you are telling me, is if you renamed the original script (check_hadr_status -> Check_DB2_HADR_sh). Is it the case?
In order to be sure about the file, and if it is safe to delete it, can you put some part of this file in this ticket?


Always send a rotation hype message in response to the hype comamnd.

Even if nothing interesting has happened.
(tbh the skipping of reports when nothing has happened is confusing and should
probably just be removed - #general is buy enough now it's never just the
bot talking to itself and if it is does it matter?)

Travis tests have failed
Hey @bakert,
Please read the following log in order to understand the failure reason.
It'll be awesome if you fix what's wrong and commit the changes.
TravisBuddy Request Identifier: 3d11e0e0-0152-11eb-9be6-0d9634a5fbe7

Abandoning this. There is a ticket open to rethink the hype command.


report: add get maintainers function

Tested locally. Requires path to the linux kernel checkout directory, no sure if adding this config into manager makes sense now.
Fixes #242

We don't hava ./scripts/get_maintainer.pl in syzkaller, it is in linux kernel project.
I think it's better to explain this in some where.

Added a comment.

PTAL

PTAL

LGTM


[Snyk] Security upgrade golang from 1.16.4-alpine to 1.23.3-alpine

Snyk has created this PR to fix 4 vulnerabilities in the dockerfile dependencies of this project.
Keeping your Docker base image up-to-date means you’ll benefit from security fixes in the latest version of your chosen image.
Snyk changed the following file(s):

scenarios/ecs_takeover/terraform/website/Dockerfile

We recommend upgrading to golang:1.23.3-alpine, as this image has only 1 known vulnerabilities. To do this, merge this pull request, then verify your application still works as expected.
Vulnerabilities that will be fixed with an upgrade:

Issue
Score

Out-of-bounds Read SNYK-ALPINE313-APKTOOLS-1533754
  714

Buffer Overflow SNYK-ALPINE313-OPENSSL-1569448
  714

Buffer Overflow SNYK-ALPINE313-OPENSSL-1569448
  714

Out-of-bounds Write SNYK-ALPINE313-ZLIB-2976175
  714

Use After Free SNYK-ALPINE313-BUSYBOX-1920751
  614

[!IMPORTANT]

Check the changes in this PR to ensure they won't cause issues with your project.
Max score is 1000. Note that the real score may have changed since the PR was raised.
This PR was automatically created by Snyk using the credentials of a real user.

Note: You are seeing this because you or someone else with access to this repository has authorized Snyk to open fix PRs.
For more information:
🧐 View latest project report
📜 Customise PR templates
🛠 Adjust project settings
📚 Read about Snyk's upgrade logic

Learn how to fix vulnerabilities with free interactive lessons:
🦉 Use After Free

Checkmarx One – Scan Summary & Details – b862dfff-ff03-4ab8-88db-880712a2417d
New Issues

Severity
Issue
Source File / Package
Checkmarx Insight

ALB Listening on HTTP
/lb.tf: 74
AWS Application Load Balancer (alb) should not listen on HTTP

Authentication Without MFA
/rw_iam_user.tf: 8
Users should authenticate with MFA (Multi-factor Authentication) to ensure an extra layer of protection when authenticating

Authentication Without MFA
/initial_iam_user.tf: 8
Users should authenticate with MFA (Multi-factor Authentication) to ensure an extra layer of protection when authenticating

Authentication Without MFA
/dev_machine.tf: 78
Users should authenticate with MFA (Multi-factor Authentication) to ensure an extra layer of protection when authenticating

Authentication Without MFA
/iam.tf: 21
Users should authenticate with MFA (Multi-factor Authentication) to ensure an extra layer of protection when authenticating

CVE-2021-41092
Go-github.com/docker/cli-v20.10.7
Vulnerable Package

CVE-2022-24778
Go-github.com/containerd/imgcrypt-v1.1.1
Vulnerable Package

CVE-2023-39325
Go-golang.org/x/net-v0.0.0-20211015210444-4f30a5c0130f
Vulnerable Package

CVE-2023-44487
Go-k8s.io/apiserver-v0.20.6
Vulnerable Package

CVE-2023-44487
Go-google.golang.org/grpc-v1.38.0
Vulnerable Package

CVE-2023-44487
Go-k8s.io/apimachinery-v0.20.6
Vulnerable Package

CVE-2023-45133
<EMAIL_ADDRESS>Vulnerable Package

CVE-2023-45288
Go-golang.org/x/net-v0.0.0-20211015210444-4f30a5c0130f
Vulnerable Package

CVE-2023-45311
Npm-fsevents-1.2.9
Vulnerable Package

CVE-2023-49568
Go-github.com/go-git/go-git/v5-v5.4.2
Vulnerable Package

CVE-2023-49569
Go-github.com/go-git/go-git/v5-v5.4.2
Vulnerable Package

CVE-2024-21538
Npm-cross-spawn-6.0.5
Vulnerable Package

CVE-2024-21538
Npm-cross-spawn-5.1.0
Vulnerable Package

CVE-2024-21626
Go-github.com/opencontainers/runc-v1.0.0-rc93
Vulnerable Package

CVE-2024-24786
Go-google.golang.org/protobuf-v1.26.0
Vulnerable Package

CVE-2024-3817
Go-github.com/hashicorp/go-getter-v1.5.9
Vulnerable Package

CVE-2024-4068
Npm-braces-2.3.2
Vulnerable Package

CVE-2024-41110
Go-github.com/docker/docker-v20.10.7
Vulnerable Package

CVE-2024-6257
Go-github.com/hashicorp/go-getter-v1.5.9
Vulnerable Package

CloudTrail Log Files S3 Bucket with Logging Disabled
/cloudtrail.tf: 49
CloudTrail Log Files S3 Bucket should have 'logging' enabled

CloudWatch Console Sign-in Without MFA Alarm Missing
/cloudwatch.tf: 1
Ensure a log metric filter and alarm exist for management console sign-in without MFA

CloudWatch IAM Policy Changes Alarm Missing
/cloudwatch.tf: 1
Ensure a log metric filter and alarm exist for IAM policy changes

CloudWatch Root Account Use Missing
/cloudwatch.tf: 1
Ensure a log metric filter and alarm exist for root acount usage

CloudWatch Unauthorized Access Alarm Missing
/cloudwatch.tf: 1
Ensure a log metric filter and alarm exist for unauthorized API calls

Cx477c10a4-5717
Go-go.etcd.io/etcd-v0.5.0-alpha.5.0.20200910180754-dd1b699fc489
Vulnerable Package

Cx9c51b693-b977
Go-github.com/containerd/containerd-v1.5.2
Vulnerable Package

Cxb421a484-d05a
Go-github.com/cyphar/filepath-securejoin-v0.2.2
Vulnerable Package

Cxb6dee8d5-b814
Go-gopkg.in/square/go-jose.v2-v2.5.1
Vulnerable Package

DB Instance Storage Not Encrypted
/rds.tf: 46
AWS DB Instance should have its storage encrypted by setting the parameter to 'true'. The storage_encrypted default value is 'false'.

DB Instance Storage Not Encrypted
/rds.tf: 50
AWS DB Instance should have its storage encrypted by setting the parameter to 'true'. The storage_encrypted default value is 'false'.

EC2 Instance Has Public IP
/dev_machine.tf: 52
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 137
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 183
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 25
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 98
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 43
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 119
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 61
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 122
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 29
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 96
EC2 Instance should not have a public IP address.

EC2 Instance Has Public IP
/ec2.tf: 6
EC2 Instance should not have a public IP address.

ECS Task Definition Network Mode Not Recommended
/ecs.tf: 45
Network_Mode should be 'awsvpc' in ecs_task_definition. AWS VPCs provides the controls to facilitate a formal process for approving and testing all...

ECS Task Definition Volume Not Encrypted
/ecs.tf: 71
Amazon ECS Task Definition does not have encryption for data at transit enabled. To prevent such a scenario, enable the attribute 'transit_encryption'

EFS Not Encrypted
/efs.tf: 1
Elastic File System (EFS) must be encrypted

EFS Without KMS
/efs.tf: 1
Amazon Elastic Filesystem should have filesystem encryption enabled using KMS CMK customer-managed keys instead of AWS managed-keys

IAM Database Auth Not Enabled
/rds.tf: 46
IAM Database Auth Enabled should be configured to true when using compatible engine and version

IAM Database Auth Not Enabled
/rds.tf: 50
IAM Database Auth Enabled should be configured to true when using compatible engine and version

IAM Policies With Full Privileges
/iam.tf: 108
IAM policies shouldn't allow full administrative privileges (for all resources)

IAM Policy Grants Full Permissions
/iam.tf: 108
IAM policy should not grant full permissions to resources from the get-go, instead of granting permissions gradually as necessary.

Missing User Instruction
/Dockerfile: 1
A user should be specified in the dockerfile, otherwise the image will run as root

Missing User Instruction
/Dockerfile: 1
A user should be specified in the dockerfile, otherwise the image will run as root

Missing User Instruction
/Dockerfile: 1
A user should be specified in the dockerfile, otherwise the image will run as root

Passwords And Secrets - Generic Token
/cognito.tf: 84
Query to find passwords and secrets in infrastructure code.

Passwords And Secrets - Generic Token
/cognito.tf: 86
Query to find passwords and secrets in infrastructure code.

Passwords And Secrets - Generic Token
/cognito.tf: 85
Query to find passwords and secrets in infrastructure code.

S3 Bucket Object Not Encrypted
/s3.tf: 51
S3 Bucket Object should have server-side encryption enabled

S3 Bucket Object Not Encrypted
/s3.tf: 63
S3 Bucket Object should have server-side encryption enabled

S3 Bucket Object Not Encrypted
/s3.tf: 49
S3 Bucket Object should have server-side encryption enabled

S3 Bucket Object Not Encrypted
/s3.tf: 91
S3 Bucket Object should have server-side encryption enabled

S3 Bucket Object Not Encrypted
/s3.tf: 71
S3 Bucket Object should have server-side encryption enabled

S3 Bucket Object Not Encrypted
/s3.tf: 19
S3 Bucket Object should have server-side encryption enabled

S3 Bucket Object Not Encrypted
/s3.tf: 39
S3 Bucket Object should have server-side encryption enabled

S3 Bucket Object Not Encrypted
/s3.tf: 29
S3 Bucket Object should have server-side encryption enabled

S3 Bucket Object Not Encrypted
/s3.tf: 61
S3 Bucket Object should have server-side encryption enabled

S3 Bucket Object Not Encrypted
/s3.tf: 81
S3 Bucket Object should have server-side encryption enabled

Sensitive Port Is Exposed To Entire Network
/ec2.tf: 77
A sensitive port, such as port 23 or port 110, is open for the whole network in either TCP or UDP protocol

Sensitive Port Is Exposed To Entire Network
/ec2.tf: 53
A sensitive port, such as port 23 or port 110, is open for the whole network in either TCP or UDP protocol

Sensitive Port Is Exposed To Entire Network
/ec2.tf: 102
A sensitive port, such as port 23 or port 110, is open for the whole network in either TCP or UDP protocol

Unknown Port Exposed To Internet
/ec2.tf: 78
AWS Security Group should not have an unknown port exposed to the entire Internet

Unrestricted Security Group Ingress
/ec2.tf: 106
Security groups allow ingress from <IP_ADDRESS>:0 and/or ::/0

Unrestricted Security Group Ingress
/ec2.tf: 78
Security groups allow ingress from <IP_ADDRESS>:0 and/or ::/0

Unrestricted Security Group Ingress
/ec2.tf: 58
Security groups allow ingress from <IP_ADDRESS>:0 and/or ::/0

Unrestricted Security Group Ingress
/ec2.tf: 82
Security groups allow ingress from <IP_ADDRESS>:0 and/or ::/0

ALB Is Not Integrated With WAF
/lb.tf: 27
All Application Load Balancers (ALB) must be protected with Web Application Firewall (WAF) service

ALB Not Dropping Invalid Headers
/lb.tf: 27
It's considered a best practice when using Application Load Balancers to drop invalid header fields

API Gateway Access Logging Disabled
/apigateway.tf: 38
API Gateway should have Access Log Settings defined

API Gateway Access Logging Disabled
/apig.tf: 316
API Gateway should have Access Log Settings defined

API Gateway Access Logging Disabled
/apigateway.tf: 38
API Gateway Stage should have Access Logging Settings defined

API Gateway Access Logging Disabled
/apig.tf: 316
API Gateway Stage should have Access Logging Settings defined

API Gateway Deployment Without Access Log Setting
/apig.tf: 309
API Gateway Deployment should have access log setting defined when connected to an API Gateway Stage.

API Gateway Method Does Not Contains An API Key
/apig.tf: 99
An API Key should be required on a method request.

API Gateway Method Does Not Contains An API Key
/apig.tf: 89
An API Key should be required on a method request.

API Gateway Method Does Not Contains An API Key
/apig.tf: 82
An API Key should be required on a method request.

API Gateway With CloudWatch Logging Disabled
/apig.tf: 316
AWS CloudWatch Logs for APIs should be enabled and using the naming convention described in documentation

API Gateway With Invalid Compression
/apig.tf: 65
API Gateway should have valid compression, which means attribute 'minimum_compression_size' should be set and its value should be greater than -1 a...

API Gateway With Open Access
/apig.tf: 85
API Gateway Method should restrict the authorization type, except for the HTTP OPTIONS method.

API Gateway With Open Access
/apig.tf: 92
API Gateway Method should restrict the authorization type, except for the HTTP OPTIONS method.

API Gateway With Open Access
/apig.tf: 102
API Gateway Method should restrict the authorization type, except for the HTTP OPTIONS method.

API Gateway Without Configured Authorizer
/apig.tf: 65
API Gateway REST API should have an API Gateway Authorizer

API Gateway Without SSL Certificate
/apig.tf: 316
SSL Client Certificate should be enabled

API Gateway X-Ray Disabled
/apig.tf: 316
API Gateway should have X-Ray Tracing enabled

API Gateway without WAF
/apig.tf: 316
API Gateway should have WAF (Web Application Firewall) enabled

CVE-2021-41190
Go-github.com/docker/docker-v20.10.7
Vulnerable Package

CVE-2023-3978
Go-golang.org/x/net-v0.0.0-20211015210444-4f30a5c0130f
Vulnerable Package

CVE-2023-48795
Go-golang.org/x/crypto-v0.0.0-20210513164829-c07d793c2f9a
Vulnerable Package

CVE-2024-28863
Npm-tar-4.4.19
Vulnerable Package

CVE-2024-28863
Npm-tar-4.4.8
Vulnerable Package

CVE-2024-29018
Go-github.com/docker/docker-v20.10.7
Vulnerable Package

CVE-2024-35195
Python-requests-2.26.0
Vulnerable Package

CVE-2024-4067
Npm-micromatch-3.1.10
Vulnerable Package

CloudTrail SNS Topic Name Undefined
/cloudtrail.tf: 34
Check if SNS topic name is set for CloudTrail

CloudWatch AWS Organizations Changes Missing Alarm
/cloudwatch.tf: 1
Ensure a log metric filter and alarm exist for AWS organizations changes

CloudWatch Disabling Or Scheduled Deletion Of Customer Created CMK Alarm Missing
/cloudwatch.tf: 1
Ensure a log metric filter and alarm exist for disabling or scheduled deletion of customer created CMK

CloudWatch Log Group Without KMS
/cloudwatch.tf: 1
AWS CloudWatch Log groups should be encrypted using KMS

CloudWatch Management Console Auth Failed Alarm Missing
/cloudwatch.tf: 1
Ensure a log metric filter and alarm exist for AWS Management Console authentication failures

CloudWatch S3 policy Change Alarm Missing
/cloudwatch.tf: 11
Ensure a log metric filter and alarm exist for S3 bucket policy changes

CloudWatch S3 policy Change Alarm Missing
/cloudwatch.tf: 43
Ensure a log metric filter and alarm exist for S3 bucket policy changes

CloudWatch Without Retention Period Specified
/cloudwatch.tf: 1
AWS CloudWatch Log groups should have retention days specified

Cloudwatch Cloudtrail Configuration Changes Alarm Missing
/cloudwatch.tf: 1
Ensure a log metric filter and alarm exist for CloudTrail configuration changes

Cloudwatch Security Group Changes Alarm Missing
/cloudwatch.tf: 1
Ensure a log metric filter and alarm exist for security group changes

Cognito UserPool Without MFA
/cognito.tf: 1
AWS Cognito UserPool should have MFA (Multi-Factor Authentication) defined to users

Cx47e4310e-3167
Go-go.etcd.io/etcd-v0.5.0-alpha.5.0.20200910180754-dd1b699fc489
Vulnerable Package

Cx68162182-3f7b
Go-github.com/containerd/containerd-v1.5.2
Vulnerable Package

ECR Image Tag Not Immutable
/sdlc.tf: 58
ECR should have an image tag be immutable. This prevents image tags from being overwritten.

Healthcheck Not Set
/docker_stack.yml: 4
Check containers periodically to see if they are running properly.

IAM Policies Attached To User
/iam.tf: 55
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 24
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/rw_iam_user.tf: 6
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 119
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 106
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 19
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 98
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 120
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/initial_iam_user.tf: 6
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 102
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 89
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 85
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/dev_machine.tf: 76
IAM policies should be attached only to groups or roles

IAM Policies Attached To User
/iam.tf: 115
IAM policies should be attached only to groups or roles

Instance With No VPC
/dev_machine.tf: 52
EC2 Instances should be configured under a VPC network. AWS VPCs provide the controls to facilitate a formal process for approving and testing all ...

Memory Not Limited
/docker_stack.yml: 4
Memory limits should be defined for each container. This prevents potential resource exhaustion by ensuring that containers consume not more than t...

Missing_HSTS_Header
/scenarios/ecs_takeover/terraform/website/main.go: 51
Attack Vector

Pip install Keeping Cached Packages
/Dockerfile: 9
When installing packages with pip, the '--no-cache-dir' flag should be set to make Docker images smaller

Pip install Keeping Cached Packages
/Dockerfile: 18
When installing packages with pip, the '--no-cache-dir' flag should be set to make Docker images smaller

Pip install Keeping Cached Packages
/Dockerfile: 4
When installing packages with pip, the '--no-cache-dir' flag should be set to make Docker images smaller

RDS With Backup Disabled
/rds.tf: 50
Make sure the AWS RDS configuration has automatic backup configured. If the retention period is equal to 0 there is no backup

RDS With Backup Disabled
/rds.tf: 46
Make sure the AWS RDS configuration has automatic backup configured. If the retention period is equal to 0 there is no backup

REST API With Vulnerable Policy
/apig.tf: 325
REST API policy should avoid wildcard in 'Action' and 'Principal'

Relative_Path_Traversal
/scenarios/rce_web_app/assets/rce_app/index.js: 35
Attack Vector

Relative_Path_Traversal
/scenarios/rce_web_app/assets/rce_app/index.js: 33
Attack Vector

Relative_Path_Traversal
/scenarios/rce_web_app/assets/rce_app/index.js: 35
Attack Vector

Relative_Path_Traversal
/scenarios/rce_web_app/assets/rce_app/index.js: 33
Attack Vector

Relative_Path_Traversal
/scenarios/rce_web_app/assets/rce_app/index.js: 33
Attack Vector

Relative_Path_Traversal
/scenarios/rce_web_app/assets/rce_app/index.js: 35
Attack Vector

S3 Bucket Logging Disabled
/s3.tf: 27
Server Access Logging should be enabled on S3 Buckets so that all changes are logged and trackable

S3 Bucket Logging Disabled
/s3.tf: 49
Server Access Logging should be enabled on S3 Buckets so that all changes are logged and trackable

S3 Bucket Logging Disabled
/s3.tf: 6
Server Access Logging should be enabled on S3 Buckets so that all changes are logged and trackable

S3 Bucket Logging Disabled
/s3.tf: 38
Server Access Logging should be enabled on S3 Buckets so that all changes are logged and trackable

S3 Bucket Logging Disabled
/codepipeline.tf: 8
Server Access Logging should be enabled on S3 Buckets so that all changes are logged and trackable

S3 Bucket Logging Disabled
/cloudtrail.tf: 49
Server Access Logging should be enabled on S3 Buckets so that all changes are logged and trackable

S3 Bucket Logging Disabled
/s3.tf: 9
Server Access Logging should be enabled on S3 Buckets so that all changes are logged and trackable

S3 Bucket Logging Disabled
/s3.tf: 9
Server Access Logging should be enabled on S3 Buckets so that all changes are logged and trackable

S3 Bucket Policy Accepts HTTP Requests
/s3.tf: 15
S3 Bucket policy should not accept HTTP Requests

S3 Bucket Policy Accepts HTTP Requests
/s3.tf: 4
S3 Bucket policy should not accept HTTP Requests

S3 Bucket Policy Accepts HTTP Requests
/cloudtrail.tf: 56
S3 Bucket policy should not accept HTTP Requests

S3 Bucket Without Versioning
/codepipeline.tf: 8
S3 bucket should have versioning enabled

S3 Bucket Without Versioning
/s3.tf: 38
S3 bucket should have versioning enabled

S3 Bucket Without Versioning
/s3.tf: 9
S3 bucket should have versioning enabled

S3 Bucket Without Versioning
/s3.tf: 49
S3 bucket should have versioning enabled

S3 Bucket Without Versioning
/cloudtrail.tf: 49
S3 bucket should have versioning enabled

S3 Bucket Without Versioning
/s3.tf: 6
S3 bucket should have versioning enabled

S3 Bucket Without Versioning
/s3.tf: 9
S3 bucket should have versioning enabled

S3 Bucket Without Versioning
/s3.tf: 27
S3 bucket should have versioning enabled

SNS Topic Not Encrypted
/sns.tf: 2
SNS (Simple Notification Service) Topic should be encrypted

SNS Topic Not Encrypted
/sns.tf: 12
SNS (Simple Notification Service) Topic should be encrypted

Secretsmanager Secret Without KMS
/secretsmanager.tf: 1
AWS Secretmanager should use AWS KMS customer master key (CMK) to encrypt the secret values in the versions stored in the secret

Secretsmanager Secret Without KMS
/secrets_manager.tf: 1
AWS Secretmanager should use AWS KMS customer master key (CMK) to encrypt the secret values in the versions stored in the secret

Secretsmanager Secret Without KMS
/secrets_manager.tf: 12
AWS Secretmanager should use AWS KMS customer master key (CMK) to encrypt the secret values in the versions stored in the secret

Security Opt Not Set
/docker_stack.yml: 4
Attribute 'security_opt' should be defined.

Sensitive Port Is Exposed To Wide Private Network
/rds.tf: 6
A sensitive port, such as port 23 or port 110, is open for a wide private network in either TCP or UDP protocol

Unpinned Package Version in Apk Add
/Dockerfile: 6
Package version pinning reduces the range of versions that can be installed, reducing the chances of failure due to unanticipated changes

Unpinned Package Version in Apk Add
/Dockerfile: 3

More results are available on AST platform


New namespace support

I started working on new approach to namespace support.
My ideas:

Contexts:
You can create namespaces for contexts by just declaring it in the Jenny.properties or the Jenny preferences window

Game, Input, MySharedStuff.Common
This should generate

CommonAttribute
CommonComponentsLookup
CommonContext
CommonEntity
CommonMatcher
etc.

and all of them are wrapped in MySharedStuff namespace.
You can still access the contexts like contexts.game, contexts.mySharedstuffCommon
You can still use the attributes
[MySharedStuff.Common]
public sealed class Position3Int : IComponent
{
    public Vector3Int Value;
}

Components

You can add namespaces to components
namespace Shmup.Core
{
    public sealed class Player : IComponent
    {
    }
}

var e = contexts.game.CreateEntity(); // e is GameEntity
contexts.game.CreateEntity().isPlayer = true;

It's work in progress, I'm currently working on the details and see what works well and what doesn't. I try to keep the api the same, but in some cases it might be necessary to be more explicit to avoid name clashes.
I'll keep you updated and add more examples once it takes shapes more.

Update: I'm making some progress on this while at the same time tackling one of my favourite issue #307
This requires changes to the current code gen which generates partial classes for entities etc.
This will be replaced with C# extension methods.
More infos coming soon

Will close in favour of #1005
You can follow the progress there


FABC-881 Use earlier revision of bccsp

need to revert an earlier commit as we need to
support plugins in the 1.4.x stream

Signed-off-by: Gari Singh<EMAIL_ADDRESS>

@sykesm  - I had to use an earlier revision of bccsp (still from master) as we still need to support plugins in 1.4.x  :-(


Enh/neuropixels rig

Adds neuropixels rig mapping for mpe sync config sync.yml and mvr config mvr.ini. An additional non-mpe config mvr.mapping.json is used to map mvr camera names to camera assembly names in the rig. Aside from sync and camera assembly information, the rest of the rig information is static and rehydrated from rig.partial.json.

@mochic let's please also make the name of the module more specific.  maybe: neuropixels-mvr.

@dyf This has beeb moved to #28


fix dashboard activity for tier0

@hufnagel, Dirk please review.
Jen found out that Tier0 workflows are reported as “production” activity in dashboard.
http://dashb-cms-job.cern.ch/dashboard/templates/web-job2/#user=&refresh=60&table=Jobs&p=1&records=25&activemenu=1&usr=&site=&submissiontool=&application=&activity=production&status=&check=submitted&tier=&date1=2017-06-08+19%3A53&date2=2017-06-09+19%3A53&sortby=task&scale=linear&bars=20&ce=&rb=&grid=&jobtype=&submissionui=&dataset=&submissiontype=&task=&subtoolver=&genactivity=&outputse=&appexitcode=&accesstype=&inputse=&cores=
It seems it is hard coded for express, repack and promptreco.
https://github.com/dmwm/WMCore/blob/master/src/python/WMCore/WMSpec/StdSpecs/Express.py#L51
https://github.com/dmwm/WMCore/blob/master/src/python/WMCore/WMSpec/StdSpecs/Repack.py#L46
https://github.com/dmwm/WMCore/blob/master/src/python/WMCore/WMSpec/StdSpecs/PromptReco.py#L34
But it gets overwritten in assignment stage.

If Dirk has put a code in place to call the assignment arguments, then he has to provide the correct Dashboard activity, otherwise it will get the default "production" value:
https://github.com/ticoann/WMCore/blob/efbd47fe5105548f8e9807593a72114ada941724/src/python/WMCore/WMSpec/StdSpecs/StdBase.py#L1114
and won't work even with this patch.
I think this patch is not needed, T0 just needs to pass that value when tier0 workflows are getting updated/assigned. @hufnagel btw, make sure to set it to t0, not "tier0" nor something else, see this issue:
https://github.com/dmwm/WMCore/issues/7256

@amaltaro I can fix this in the Tier0 specs, no problem.

@amaltaro, @hufnagel, I think it is better to set the Dashboard default value to "tier0" or "t0" which ever is the correct form, instead of hard coding in the code. (In case testingT0 workflow should have different activities, it can be controlled by spec parameter)

Yes, it could be done either during creation (we'd then need to override the createArgs on every T0 spec) or during assignmnent (when T0 updates the workload).
If there isn't any difference, then I suggest we leave Dashboard settings to the assignment stage, leaving it hard-coded for creation, as it is for all the other specs.

@amaltaro suggested to change the creation and the assignment step, which I did. If there is a way to just change one default, that would be easier. These specs are Tier0 specific.

closing it. - replaced by #7924


SLO. Is there feature that sends logout request to all clients?

Is there feature that sends logout requests to all clients or I should implement it by myself by using CasUserLogoutEvent?

I didn't implement SLO because of the specification of the protocol doesn't describe it in detail.
Yes, you should listen to CasUserLogoutEvent

I'm not sure, but I will try


qx contrib publish: allow overriding of version number

From @cboulanger on November 28, 2017 8:17
qx contrib publish at the moment increments the existing version number according to the release type (major, minor, patch ...). We should  add a -V/--version option that allows to manually set the version number.
Copied from original issue: qooxdoo/qooxdoo-cli#47

Already implemented, closing.


feat(SPV-1284): add admin method to create contact between two parties

This pull request introduces a new feature to create contacts through an admin interface, along with associated error handling and testing. The most important changes include the addition of the createContact function, updates to the CreateContact model, and corresponding tests.
New feature for creating contacts:

actions/admin/contact.go: Added the createContact function, which handles the creation of a contact using the provided paymail and other details, and responds with the appropriate status.
actions/admin/models.go: Introduced the CreateContact struct to represent the data model for creating a contact, including fields for paymail, full name, and metadata.
engine/action_contact.go: Added the AdminCreateContact method in the Client struct to encapsulate the logic for creating a contact, including validation and error handling.
engine/interface.go: Updated the ContactService interface to include the AdminCreateContact method.

Error handling:

engine/spverrors/definitions.go: Defined new error types ErrMissingContactPaymailParam, ErrContactAlreadyExists, and ErrGettingPKIFailed to handle specific error scenarios during contact creation. [1] [2]

Testing:

engine/contact_service_test.go: Added comprehensive tests for the AdminCreateContact method, covering various scenarios including successful creation, missing paymail, and duplicate contact cases.

Pull Request Checklist

[ ] 📖 I created my PR using provided  : CODE_STANDARDS
[ ] 📖 I have read the short Code of Conduct: CODE_OF_CONDUCT
[ ] 🏠 I tested my changes locally.
[ ] ✅ I have provided tests for my changes.
[ ] 📝 I have used conventional commits.
[ ] 📗 I have updated any related documentation.
[ ] 💾 PR was issued based on the Github or Jira issue.

:warning: Please install the  to ensure uploads and comments are reliably processed by Codecov.
Codecov Report
Attention: Patch coverage is 47.69231% with 34 lines in your changes missing coverage. Please review.

Project coverage is 46.02%. Comparing base (93b9929) to head (0ffc099).
Report is 2 commits behind head on main.

Files with missing lines
Patch %
Lines

actions/admin/contact.go
0.00%
19 Missing :warning:

engine/action_contact.go
66.66%
10 Missing and 5 partials :warning:

Additional details and impacted files

@@            Coverage Diff             @@
##             main     #807      +/-   ##
==========================================
+ Coverage   45.95%   46.02%   +0.06%
==========================================
  Files         364      365       +1
  Lines       17595    17690      +95
==========================================
+ Hits         8086     8142      +56
- Misses       8912     8945      +33
- Partials      597      603       +6

Flag
Coverage Δ

unittests
46.02% <47.69%> (+0.06%)
:arrow_up:

Flags with carried forward coverage won't be shown. Click here to find out more.

Files with missing lines
Coverage Δ

actions/sharedconfig/routes.go
100.00% <100.00%> (ø)

engine/action_contact.go
42.45% <66.66%> (+3.56%)
:arrow_up:

actions/admin/contact.go
0.00% <0.00%> (ø)

... and 18 files with indirect coverage changes

Continue to review full report in Codecov by Sentry.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update 93b9929...0ffc099. Read the comment docs.