[do not merge] server: Do NOT cleanup dhcp and dns when stop a vm

Description

According comment in PR #3608, dhcp and dns entries are cleaned up only when a VM is expunged.
Revert part of commit 8fb388e9312b917a8f36c7d7e3f45985a95ce773.

Types of changes

[ ] Breaking change (fix or feature that would cause existing functionality to change)
[ ] New feature (non-breaking change which adds functionality)
[X] Bug fix (non-breaking change which fixes an issue)
[ ] Enhancement (improves an existing feature and functionality)
[ ] Cleanup (Code refactoring and cleanup, that may add test cases)

Screenshots (if appropriate):
How Has This Been Tested?

@rhtyd

@blueorangutan package

@rhtyd a Jenkins job has been kicked to build packages. I'll keep you posted as I make progress.

Packaging result: ✔centos6 ✔centos7 ✔debian. JID-311

@blueorangutan test

@rhtyd a Trillian-Jenkins test job (centos7 mgmt + kvm-centos7) has been kicked to run smoke tests

@blueorangutan test

@rhtyd a Trillian-Jenkins test job (centos7 mgmt + kvm-centos7) has been kicked to run smoke tests

This absolutely requires manual testing (that also means #3608).
No merging until tested thoroughly, please!
(I'll test myself as well)

In short - NOTHING IS CLEANED at all, whatsoever...
Test:
ENV built based from packages built from this PR (VMware 6.5u, CenOS7 mgmt) - clean install.
Created Isolated network (R21 in examples below)
Created Shared network (R22 in examples below)
Created a "vm1" (and some more VMs, but irrelevant) and joined vm1 to:

Isolated network (default one) , IP <IP_ADDRESS>
Shared network (additional network), IP <IP_ADDRESS>

While VM running, inside R21 (Isolated)
root@r-21-VM:~# grep -ir "10\.1\.1\.101" /var/; grep -ir "10\.1\.1\.101" /etc/
/var/www/html/userdata/<IP_ADDRESS>/.htaccess:Allow from <IP_ADDRESS>
/var/www/html/metadata/<IP_ADDRESS>/local-ipv4:<IP_ADDRESS>
/var/www/html/metadata/<IP_ADDRESS>/.htaccess:Allow from <IP_ADDRESS>
/var/cache/cloud/passwords-<IP_ADDRESS>:<IP_ADDRESS>=saved_password
1","default_gateway":"<IP_ADDRESS>","default_entry":true,"remove":false,"type":"dhcpentry"}
/var/cache/cloud/processed/VR-df1e8224-8c07-4e8d-9006-191c1f48224a.cfg:{"vm_ip_address":"<IP_ADDRESS>","vm_metadata":[["userdata","user-data",null],["metadata","service-offering","Small Instance"],["metadata","availability-zone","pr3627-t408-vmware-65u2"],["metadata","local-ipv4","<IP_ADDRESS>"],["metadata","local-hostname","vm1"],["metadata","public-ipv4","<IP_ADDRESS>"],["metadata","public-hostname","<IP_ADDRESS>"],["metadata","instance-id","75c43c29-a101-458f-81c6-8fe6e7546890"],["metadata","vm-id","75c43c29-a101-458f-81c6-8fe6e7546890"],["metadata","public-keys",null],["metadata","cloud-identifier","CloudStack-{f36badac-df80-40f2-a1f2-ab3a9ea41259}"]],"type":"vmdata"}
/var/lib/misc/dnsmasq.leases:1573462858 02:00:2d:24:00:0e <IP_ADDRESS> vm1 *
/var/log/cloud.log:2019-10-10 20:52:55,009 INFO     Processing metadata for <IP_ADDRESS>
/var/log/cloud.log:2019-10-10 20:53:08,194 INFO     Processing metadata for <IP_ADDRESS>
/var/log/cloud.log:2019-10-10 20:59:14,675  configure.py process:330 Processing metadata for <IP_ADDRESS>
/var/log/dnsmasq.log:Oct 10 21:00:58 dnsmasq-dhcp[2826]: DHCPDISCOVER(eth0) <IP_ADDRESS> 02:00:2d:24:00:0e
/var/log/dnsmasq.log:Oct 10 21:00:58 dnsmasq-dhcp[2826]: DHCPOFFER(eth0) <IP_ADDRESS> 02:00:2d:24:00:0e
/var/log/dnsmasq.log:Oct 10 21:00:58 dnsmasq-dhcp[2826]: DHCPREQUEST(eth0) <IP_ADDRESS> 02:00:2d:24:00:0e
/var/log/dnsmasq.log:Oct 10 21:00:58 dnsmasq-dhcp[2826]: DHCPACK(eth0) <IP_ADDRESS> 02:00:2d:24:00:0e vm1
/var/log/messages:Oct 10 20:59:13 systemvm passwd_server_ip.py: serve_password: password saved for VM IP <IP_ADDRESS>
/etc/dhcphosts.txt:02:00:2d:24:00:0e,<IP_ADDRESS>,vm1,756h
/etc/cloudstack/vmdata.json:  "<IP_ADDRESS>": [
/etc/cloudstack/vmdata.json:      "<IP_ADDRESS>"
/etc/cloudstack/vmpassword.json:  "<IP_ADDRESS>": "saved_password",
/etc/cloudstack/dhcpentry.json:  "<IP_ADDRESS>": {
/etc/cloudstack/dhcpentry.json:    "ipv4_address": "<IP_ADDRESS>",
/etc/hosts:<IP_ADDRESS>   vm1

vm1 expunged, inside R21 (Isolated)
root@r-21-VM:~# grep -ir "10\.1\.1\.101" /var/; grep -ir "10\.1\.1\.101" /etc/
/var/www/html/userdata/<IP_ADDRESS>/.htaccess:Allow from <IP_ADDRESS>
/var/www/html/metadata/<IP_ADDRESS>/local-ipv4:<IP_ADDRESS>
/var/www/html/metadata/<IP_ADDRESS>/.htaccess:Allow from <IP_ADDRESS>
/var/cache/cloud/passwords-<IP_ADDRESS>:<IP_ADDRESS>=saved_password
/var/cache/cloud/processed/VR-df1e8224-8c07-4e8d-9006-191c1f48224a.cfg:{"host_name":"vm1","mac_address":"02:00:2d:24:00:0e","ipv4_address":"<IP_ADDRESS>","ipv6_duid":"00:03:00:01:02:00:2d:24:00:0e","dns_addresses":"<IP_ADDRESS>","default_gateway":"<IP_ADDRESS>","default_entry":true,"remove":false,"type":"dhcpentry"}
/var/cache/cloud/processed/VR-df1e8224-8c07-4e8d-9006-191c1f48224a.cfg:{"vm_ip_address":"<IP_ADDRESS>","vm_metadata":[["userdata","user-data",null],["metadata","service-offering","Small Instance"],["metadata","availability-zone","pr3627-t408-vmware-65u2"],["metadata","local-ipv4","<IP_ADDRESS>"],["metadata","local-hostname","vm1"],["metadata","public-ipv4","<IP_ADDRESS>"],["metadata","public-hostname","<IP_ADDRESS>"],["metadata","instance-id","75c43c29-a101-458f-81c6-8fe6e7546890"],["metadata","vm-id","75c43c29-a101-458f-81c6-8fe6e7546890"],["metadata","public-keys",null],["metadata","cloud-identifier","CloudStack-{f36badac-df80-40f2-a1f2-ab3a9ea41259}"]],"type":"vmdata"}
/var/lib/misc/dnsmasq.leases:1573462858 02:00:2d:24:00:0e <IP_ADDRESS> vm1 *
/var/log/cloud.log:2019-10-10 20:52:55,009 INFO     Processing metadata for <IP_ADDRESS>
/var/log/cloud.log:2019-10-10 20:53:08,194 INFO     Processing metadata for <IP_ADDRESS>
/var/log/cloud.log:2019-10-10 20:59:14,675  configure.py process:330 Processing metadata for <IP_ADDRESS>
/var/log/dnsmasq.log:Oct 10 21:00:58 dnsmasq-dhcp[2826]: DHCPDISCOVER(eth0) <IP_ADDRESS> 02:00:2d:24:00:0e
/var/log/dnsmasq.log:Oct 10 21:00:58 dnsmasq-dhcp[2826]: DHCPOFFER(eth0) <IP_ADDRESS> 02:00:2d:24:00:0e
/var/log/dnsmasq.log:Oct 10 21:00:58 dnsmasq-dhcp[2826]: DHCPREQUEST(eth0) <IP_ADDRESS> 02:00:2d:24:00:0e
/var/log/dnsmasq.log:Oct 10 21:00:58 dnsmasq-dhcp[2826]: DHCPACK(eth0) <IP_ADDRESS> 02:00:2d:24:00:0e vm1
/var/log/messages:Oct 10 20:59:13 systemvm passwd_server_ip.py: serve_password: password saved for VM IP <IP_ADDRESS>
/etc/dhcphosts.txt:02:00:2d:24:00:0e,<IP_ADDRESS>,vm1,756h
/etc/cloudstack/vmdata.json:  "<IP_ADDRESS>": [
/etc/cloudstack/vmdata.json:      "<IP_ADDRESS>"
/etc/cloudstack/vmpassword.json:  "<IP_ADDRESS>": "saved_password",
/etc/cloudstack/dhcpentry.json:  "<IP_ADDRESS>": {
/etc/cloudstack/dhcpentry.json:    "ipv4_address": "<IP_ADDRESS>",
/etc/hosts:<IP_ADDRESS>   vm1

Identical results are for R22 (Shared network) - zero difference in grep results (not sharing, for brevity)

Additional thing I saw - when you restart network with cleanup - dhcp.leases file is completely empty (probably some more issues in other dns and dhcp related files)

Tested with a VM attached just to Isolated network - same results.

Trillian test result (tid-406)
Environment: kvm-centos7 (x2), Advanced Networking with Mgmt server 7
Total time taken: 35774 seconds
Marvin logs: https://github.com/blueorangutan/acs-prs/releases/download/trillian/pr3627-t406-kvm-centos7.zip
Intermittent failure detected: /marvin/tests/smoke/test_vpc_redundant.py
Intermittent failure detected: /marvin/tests/smoke/test_vpc_vpn.py
Smoke tests completed. 76 look OK, 1 have error(s)
Only failed tests results shown below:

Test
Result
Time (s)
Test File

test_05_rvpc_multi_tiers
Failure
414.14
test_vpc_redundant.py

test_05_rvpc_multi_tiers
Error
443.62
test_vpc_redundant.py

@andrijapanicsb thanks a lot for testing.
the issue is not caused by this PR, but PR 3608 which has been merged.
in finalizeExpunge part, nics has already been removed so no rules will be applied in virtual routers.
we should move it to _networkMgr.cleanupNics
I will test it and update this PR if it is tested ok.

@andrijapanicsb I have updated this PR and tested ok. could you please test it ?

Before expunging vm

root@r-39-VM:~# cat /var/lib/misc/dnsmasq.leases
1573859468 1e:00:33:00:00:09 <IP_ADDRESS> vm1 *
root@r-39-VM:~# cat /etc/dhcphosts.txt
1e:00:33:00:00:09,<IP_ADDRESS>,vm1,758h

nothing changes after stopping vm

after expunging vm

root@r-39-VM:~# cat /var/lib/misc/dnsmasq.leases
root@r-39-VM:~# cat /etc/dhcphosts.txt

@andrijapanicsb I did not review your test, the test should be to check the dnsmasq related config file and even maybe the leases file.

Thx @ustcweizhou @rhtyd  - will test is as soon as possible.

@andrijapanicsb that's good, thanks for testing.
@rhtyd can you set up a trillian test ?

@blueorangutan package

@andrijapanicsb a Jenkins job has been kicked to build packages. I'll keep you posted as I make progress.

Packaging result: ✔centos6 ✔centos7 ✔debian. JID-317

@ustcweizhou @andrijapanicsb - have you tested the case that when a VM is stopped, its rules are not removed from the VR? I'll kick the test
@blueorangutan test

@rhtyd a Trillian-Jenkins test job (centos7 mgmt + kvm-centos7) has been kicked to run smoke tests

yes @rhtyd , tested pretty thoroughly. Also tested when VR is stopped (and VM expunge automatically kicks in, etc). Looks good to me. Was even restarting networks with and without cleanup - all good.

Trillian test result (tid-413)
Environment: kvm-centos7 (x2), Advanced Networking with Mgmt server 7
Total time taken: 34026 seconds
Marvin logs: https://github.com/blueorangutan/acs-prs/releases/download/trillian/pr3627-t413-kvm-centos7.zip
Intermittent failure detected: /marvin/tests/smoke/test_vpc_redundant.py
Smoke tests completed. 76 look OK, 1 have error(s)
Only failed tests results shown below:

Test
Result
Time (s)
Test File

test_05_rvpc_multi_tiers
Failure
433.28
test_vpc_redundant.py

test_05_rvpc_multi_tiers
Error
462.09
test_vpc_redundant.py

@weizhouapache @rhtyd I think I see a bit of a problem with this one (somewhat edge case).
VM1 has default network NET1
VM1 is attached to NET2 as an additional network - DNS and DHCP configs are created for this in the VR of NET2 (/etc/hosts and /etc/dhcphosts.txt
We later detach VM1 from NET2 (and potentially never again attach it) - garbage is left forever in VR of NET2 (this consumes IP if nothing else).
Can you think of any other consequence here?
I see that when attaching AGAIN VM1 to NET2, a NEW config (new IP) is used in DNS and DHCP config, while the OLD IP is still present (in /etc/hosts and /etc/dhcphosts.txt) - so this will make issues for sure.

So... when an additional network is detached from the VM, and we later stop/delete/expunge that VM - nothing happens, since we only clean DHCP/DNS from the VRs/networks to which the VM is still attached.
Possibly a correct fix would be to ALSO do the same cleanup (that we otherwise do only when VM is expunging) when DETACHING from an ADDITIONAL network. Well, when detaching from ANY network - since after VM is detached, its reference(s) to its previous network(s) is lost and will never be later cleaned up (dhcp/dns config) when VM is destroyed
Makes sense @ustcweizhou @rhtyd  /CC @PaulAngus  ?

This PR is closed, we might lose the discussion @andrijapanicsb - can you create a ticket with your concerns? (i.e. to cleanup on vm nic remove/detach?)

To be continued via: https://github.com/apache/cloudstack/issues/3706
thx both!


Unable to fetch manifests from private harbor registry

Is there an existing issue for this?

[X] I have searched the existing issues

Current Behavior
When trying to verify the signature of a container pulled from a private harbor registry, the following error is returned by the CLI:
Error from server: error when creating "pull-harbor-success.yaml": admission webhook "clusterwide-verify-image.kubewarden.admission" denied the request: Pod verify-image-valid is not accepted: verification of image harbor.devops.<company>.com/<repo>/<project>:develop failed: Host error: Callback evaluation failure: Cannot fetch manifest of harbor.devops.<company>.com/<repo>/<project>:develop: Not authorized: url https://harbor.devops.<company>.com/v2/<repo>/<project>/manifests/develop

and this one by the policy-server log:
2024-03-15T15:50:38.846810Z ERROR request{method=POST uri=/validate/clusterwide-verify-image?timeout=10s version=HTTP/1.1}:validation{host="policy-server-default-647b6c8c56-lbfq7" policy_id="clusterwide-verify-image" kind="Pod" kind_group="" kind_version="v1" name="verify-image-valid" namespace="test" operation="CREATE" request_uid="d0fa899d-a975-4c47-ac27-16a7a5044c0a" resource="pods" resource_group="" resource_version="v1" subresource=""}:validate{self=PolicyEvaluator { runtime: "wapc" } settings={"modifyImagesWithDigest": Bool(true), "rule": String("PublicKey"), "signatures": Array [Object {"image": String("harbor.devops.<company>.com/<repo>/*"), "pubKeys": Array [String("-----BEGIN PUBLIC KEY-----\n<Public key data>\n-----END PUBLIC KEY-----")]}]}}: policy_evaluator::runtimes::callback: callback evaluation failed policy_id="clusterwide-verify-image" binding="kubewarden" operation="v2/verify" error="Cannot fetch manifest of harbor.devops.<company>.com/<repo>/<project>:develop: Not authorized: url https://harbor.devops.<company>.com/v2/<repo>/<project>/manifests/develop"
2024-03-15T15:50:38.856695Z  INFO request{method=POST uri=/validate/clusterwide-verify-image?timeout=10s version=HTTP/1.1}: tower_http::trace::on_response: finished processing request latency=81 ms status=200

Expected Behavior
Pull the image and either accept if verified or reject if not verified.
Steps To Reproduce

Generate private/public key pair with cosign
sign docker image
submit signed image to private harbor registry
create robot account for rancher access
On rancher, install kubewarden-crds and kubewarden-controller
create docker-registry secret in kubewarden namespace referencing robot account from step 4
install kubewarden-defaults, modifying helm chart to reference imagePullSecret as in this link: https://docs.kubewarden.io/operator-manual/policy-servers/private-registry
install verify-image-signatures policy, set it to use public key used (yaml of latest attempt at solving this on my own in Anything else)
Provide harbor secret to namespace you want image in
kubectl apply -f pull-harbor-success.yaml (contents in anything else)

Environment
Cosign version: 2.2.3
Harbor version: v2.10.0-6abb4eab
K8s implementation: EKS
cert-manager version: 1.14.2
Kubewarden version info:
 - kubewarden-controller: 2.0.8
 - kubewarden-crds: 1.4.5
 - kubewarden-defaults: 1.9.3
 - policy-server: 1.10.0
 - verify-image-signatures: 0.2.8
Rancher version info:
 - Rancher: 2.8.2
 - Dashboard: 2.8.0
 - Helm: 2.16.8-rancher2
 - Machine: 0.15.0-rancher106

Anything else?
Policy works when following along with this: https://www.kubewarden.io/blog/2022/05/verifying-image-signatures/
None of the default policies have had this issue.
Mutation setting seems to have no effect.
I've tried generating a robot account on harbor that has all possible permissions to no avail.
verify-image.yaml
includes things added by rancher. Timestamp reflects latest attempt at solving, I've been working this issue for the last month or so.
apiVersion: policies.kubewarden.io/v1
kind: ClusterAdmissionPolicy
metadata:
  annotations:
    artifacthub/pkg: verify-image-signatures/verify-image-signatures/0.2.8
  creationTimestamp: '2024-03-13T19:34:33Z'
  finalizers:
    - kubewarden
  generation: 1
  managedFields:
    - apiVersion: policies.kubewarden.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:metadata:
          f:annotations:
            .: {}
            f:artifacthub/pkg: {}
        f:spec:
          .: {}
          f:backgroundAudit: {}
          f:mode: {}
          f:module: {}
          f:mutating: {}
          f:namespaceSelector:
            .: {}
            f:matchExpressions: {}
          f:policyServer: {}
          f:rules: {}
          f:settings:
            .: {}
            f:modifyImagesWithDigest: {}
            f:rule: {}
            f:signatures: {}
          f:timeoutSeconds: {}
      manager: agent
      operation: Update
      time: '2024-03-13T19:34:33Z'
    - apiVersion: policies.kubewarden.io/v1
      fieldsType: FieldsV1
      fieldsV1:
        f:status:
          .: {}
          f:conditions:
            .: {}
            k:{"type":"PolicyActive"}:
              .: {}
              f:lastTransitionTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"PolicyServerConfigurationUpToDate"}:
              .: {}
              f:lastTransitionTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
            k:{"type":"PolicyUniquelyReachable"}:
              .: {}
              f:lastTransitionTime: {}
              f:message: {}
              f:reason: {}
              f:status: {}
              f:type: {}
          f:mode: {}
          f:policyStatus: {}
      manager: manager
      operation: Update
      subresource: status
      time: '2024-03-15T13:16:53Z'
  name: verify-image
  resourceVersion: '59372597'
  uid: 097ac356-ca34-4073-b648-6e9f49b2a030
spec:
  backgroundAudit: true
  mode: protect
  module: ghcr.io/kubewarden/policies/verify-image-signatures:v0.2.8
  mutating: true
  namespaceSelector:
    matchExpressions:
      - key: kubernetes.io/metadata.name
        operator: In
        values:
          - opa-test
  policyServer: default
  rules:
    - apiGroups:
        - ''
      apiVersions:
        - v1
      operations:
        - CREATE
        - UPDATE
      resources:
        - pods
  settings:
    modifyImagesWithDigest: true
    rule: PublicKey
    signatures:
      - image: harbor.devops.<company>.com/<repo>/*
        pubKeys:
          - |-
            -----BEGIN PUBLIC KEY-----
            <Public key data>
            -----END PUBLIC KEY-----
  timeoutSeconds: 10
status:
  conditions:
    - lastTransitionTime: '2024-03-15T13:16:53Z'
      message: The policy webhook has been created
      reason: PolicyActive
      status: 'True'
      type: PolicyActive
    - lastTransitionTime: '2024-03-13T19:35:28Z'
      message: The latest replica set is uniquely reachable
      reason: LatestReplicaSetIsUniquelyReachable
      status: 'True'
      type: PolicyUniquelyReachable
    - lastTransitionTime: '2024-03-13T19:34:33Z'
      message: Configuration for this policy is up to date
      reason: ConfigurationVersionMatch
      status: 'True'
      type: PolicyServerConfigurationUpToDate
  mode: protect
  policyStatus: active

pull-harbor-success
apiVersion: v1
kind: Pod
metadata:
  name: verify-image-valid
  namespace: test
spec:
  containers:
  - name: test-verify-image
    image: harbor.devops.<company>.com/<repo>/<project>:develop
  imagePullSecrets:
  - name: harbor-regcreds

I can reproduce this problem and the issue is not in the policy. The bug is in the policy-fetcher library. To be more precise in this line:
https://github.com/kubewarden/policy-fetcher/blob/1075c0eddc2d1bbb9c6f9af17d073a1c661d5e91/src/verify/mod.rs#L337-L340
Here we are getting the auth for the registry in the docker config.json file. However, in this file we should look for the registry. And in this line, we are passing the full image url. Therefore, none credential is found and the fallback is to use anonymous authentication. We can see that using kwctl with verbose output:
2024-04-11T02:08:28.216223Z DEBUG policy_fetcher::registry: Couldn't fetch credentials. Using anonymous instead error=NoCredentialConfigured registry=demo.goharbor.io/testing/policy-server:v1.11.0

Note the registry in use. It should be only demo.goharbor.io. This leads to a not authorized request:
2024-04-11T02:08:32.085327Z ERROR validate{self=PolicyEvaluator { runtime: "wapc" } settings={"signatures": Array [Object {"image": String("demo.goharbor.io/testing/*"), "pubKeys": Array [String("-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj
0CAQYIKoZIzj0DAQcDQgAEVIq8ZbAJYntgr/suvQFnuM/dbWiE\noFRemT6fQeINVlxw9WGdBQiLVUmwPVclv07qscx3lEUQp/D8X3kBzqqWQw==\n-----END PUBLIC KEY-----\n")]}]}}: policy_evaluator::runtimes::callback: callback evaluation failed policy_id="verify-image-
signatures" binding="kubewarden" operation="v2/verify" error="failed to get image trusted layers: Cannot fetch manifest of demo.goharbor.io/testing/policy-server:v1.11.0: Not authorized: url https://demo.goharbor.io/v2/testing/policy-serv
er/manifests/v1.11.0"

I'm moving this issue to the policy-fetcher repository


change agora to Firecloud methods repository

Fixes #959

I'm not loving how the length of the text affects the layout. Assuming it fits, I'd rather the entirety of "FireCloud Methods Repository" be on one line below "Source:" (which should then also be done with "dockstore" for consistency, which itself should probably be rendered as "Dockstore" for consistency) or indent by the width of "Source:" when wrapping.

Thanks for the feedback. I changed the name to just 'FireCloud' for now, which eliminates the formatting problems. If we want the longer name, we can rethink the design a bit.


try domvm

hey @creationix,
i noticed your comment [1] and wanted to see if you'd like to try out domvm. you may remember that i actually came from Mithril to your https://github.com/creationix/domchanger library [2]. then used both as inspiration for writing domvm.
domvm is spiritually close to domchanger in API (v1 also used JSONML) and does not have built-in routing, ajax, promises, streams, or global auto-redraw - it's just a view layer.
cheers!
[1] https://github.com/MithrilJS/mithril.js/issues/1886#issuecomment-323184646
[2] https://github.com/creationix/domchanger/issues/5

cc @andraaspar
i saw your lifecycle/redraw convo in Mithril. domvm supports isolated redraw and take a look at [1] and [2], perhaps you'll find domvm's hooks more agreeable, too.
[1] http://leeoniya.github.io/domvm/demos/playground/#lifecycle-hooks
[2] https://github.com/leeoniya/domvm#lifecycle-hooks

@leeoniya Thanks, I have looked at the docs, and tried out the playground. Feedback:

el vs vw & can't see component classes: I assume this will be difficult to hook up with JSX.
I actually like not having to call redraw after each event handler.
It's hard to translate a Mithril problem to domvm, but:

A component needs to alter the model in a lifecycle hook.
The model may be used to render other components outside the one that altered the model, hence the entire UI needs to be redrawn.
We are in midst of a redraw sequence, in a lifecycle event. We initiate a redraw for the entire UI.
What will happen? Will a new redraw sequence start immediately? Will redraw sequences be nested? Or will the redraw mechanism wait for all lifecycle calls to finish before starting a new redraw sequence? That's what my PR to Mithril was about. How does that work with domvm?

i'm not sure why a JSX hookup would be hard. there's no universal h, but that'd just be a wrapper around a switch/case that creates explicit vnode types anyhow. there are some other things that would need to be adapted, but no showstoppers. JSX is much more limited than domvm's factories, so you may lose quite a bit of power by introducing a JSX layer.
explicit redraw from events can be avoided at either the global level or per-view level: https://github.com/leeoniya/domvm#autoredraw

What will happen? Will a new redraw sequence start immediately? Will redraw sequences be nested? Or will the redraw mechanism wait for all lifecycle calls to finish before starting a new redraw sequence? That's what my PR to Mithril was about. How does that work with domvm?

domvm does not have a redraw queue. redraws are async as a side-effect being rAF-debounced per-view, but you can call them synchronously if needed via vm.redraw(true). there's no reason why you can't call a redraw from a lifecycle hook, though there's nothing that'll prevent you from triggering an infinite redraw->hook->redraw loop, given the proper conditions.

OK, one more helper to write.
Sweet! I missed that somehow.

domvm does not have a redraw queue

My issue exactly. I'm still thinking that templates should be static. I should be able to tell which lifecycle is going to follow which one. If renders get nested, it's all messed up.
Lifecycle methods populate part of my model. I need to keep track of what is rendered and what is not, so that I know it is ready to be refreshed from the server. (It was hidden and now it is rendered, so the user expects it to be freshly populated.) That system is messed up by nested lifecycle events.
Also, I find it very important to render stuff correctly for the first time the user sees it. I cannot have flashes of intermediate / old data.
It's difficult. I use workarounds, but a queue in the virtual DOM engine seems like such a simple thing. I don't know why noone does that.

virtual-dom engines are reactive to the data you feed them. if you set data.hidden = true; vm.redraw(); then you will get the appropriate dom without any flashes of old data. if i'm not mistaken, what you seem to be trying to accomplish is the inverse - have the view fetch the data and mutate the model, which will then re-refresh the view afterwards. this type of flow is quite difficult to reason about and i doubt many vdom engines can or want to accommodate it.
do you have a concrete example of when this pattern would be better than a top-down data->render approach?

Maybe I'll just fork my own then! ;)
I have lots, including the one I described above. But for a more obvious one: you're rendering SVG and you need to measure the size of text to draw something at the correct size / position. Or, you actually want to initiate an action when something appears. Scheduling a redraw before the next frame does not sound evil to me.

But for a more obvious one: you're rendering SVG and you need to measure the size of text to draw something at the correct size / position.

https://jsfiddle.net/83kg80qf/

@andraaspar :

will the redraw mechanism wait for all lifecycle calls to finish before starting a new redraw sequence?

It's pretty trivial to debounce redraws to the event queue:
function viewUpdater(vm) {
    let pending=false;
    return function() {
        if(!pending) {
            pending=true;
            setTimeout(function() {
                pending=false;
                if(vm.render) { vm.redraw(true); }                                                  // ignore redraws before view is actualized
                },0);
            }
        }
    }

@lawrence-dol setTimeout allows a frame to be drawn, so the redraw is not immediate. Please see this Mithril example for the effect I'm trying to avoid.

@andraaspar https://jsfiddle.net/jc90m7cy/

@andraaspar :
Then, I think, the standard redraw, which debounces with RAF should do the trick.

@andraaspar
Looking at the example, and drawing on the context you've given here, specifically, "It was hidden and now it is rendered, so the user expects it to be freshly populated"...
This seems to me like a standard UI/UX problem in web programming. You want to show "hidden" data, using data fresh from the host when the user clicks "show". But you have an essential disparity: the reveal action is to all intents instantaneous, and the data fetch is necessarily async and requires an indeterminate amount of time.
This need is nothing specific to DOMVM, Mithril, or any other way that one builds web apps. In essence, what you want to do, logically, is defer the rendering of the data portion of the view until the data is fetched, leaving it blank while the data is pending.
However, if the network is slow, this will suffer from a UX problem of "nothing happened when I clicked!!" if the network response is not nearly instantaneous; the common solution is to display some sort of "throbber" while the network request is pending. This can cause a flicker of the throbber in the normal case if the network is usually fast, in which case it's pretty easy to defer the display of the throbber for a few hundred ms; however I prefer just to have an identically sized icon for the show, throbber and hide actions, so the flicker is not noticeable. You also may elect to render in such a way that only the "show" icon is changed until the data fetch completes.
I apologize if I am stating the obvious, but to do this:

Set the model data in a way that the the View renders a throbber in place of the "show" icon and blank for the data part of the view.
Initiate the request and trigger a redraw.
When the request completes update the model and trigger a redraw.
View redraws now showing a "hide" icon and the data view.

Am I missing some detail?


Update torch dependency pin to <2.1

Description of changes:
Updates torch package dependency to allow versions torch<2.1.
Merge Checklist:
Put an x without space in the boxes that apply. If you are unsure about any checklist, please don't hesitate to ask. We are here to help! This is simply a reminder of what we are going to look for before merging your pull request.
General

[x] I have read the contributor guidelines
[ ] This is a documentation change or typo fix. If so, skip the rest of this checklist.
[ ] I certify that the changes I am introducing will be backward compatible, and I have discussed concerns about this, if any, with the MosaicML team.
[ ] I have updated any necessary documentation, including README and API docs (if appropriate).

Tests

[x] I ran pre-commit on my change. (check out the pre-commit section of prerequisites)
[ ] I have added tests that prove my fix is effective or that my feature works (if appropriate).
[ ] I ran the tests locally to make sure it pass. (check out testing)
[ ] I have added unit and/or integration tests as appropriate to ensure backward compatibility of the changes.

Closing since this duplicates #234


[manuf] Bind perso firmware with PERSO_INITIAL MANUF_STATE

This PR has two commits that:

Separate the provisioning of the MANUF_STATE OTP field from the rest of the CREATOR_SW_CFG partition and provision it at the end of perso firmware.
Bind the personalization firmware with the PERSO_INITIAL manufacturing state.

Define PERSO_INITIAL creator manufacturing state.
Create a new manifest_perso with Initial manufacturing state binding.

This PR addresses #24610 partially.
Note: this depends on #24783. Only review the last 2 commits.

rebase to cleanup conflicts?

rebase to cleanup conflicts?

Rebased.


Permissions dropdown doesn't scroll

For apps that have been granted access to write to many archives, the permissions prompt overflows the screen and can't be scrolled.

Also we should handle archives that don't have titles

Also we should handle archives that don't have titles

almosted missed that little guy!  move it to the top! :)


drivers: nrf*: drop -pin properties support

This patch series drop support for the deprecated -pin properties. Zephyr 3.4, the next release, will remove pinmux as well as legacy pin configuration mechanisms.
Ref https://github.com/zephyrproject-rtos/zephyr/pull/55025

(otherwise LGTM)

I think it makes sense to keep pinctrl_nrf_migrate for one more release.
Remember that many people won't take action on deprecations until it is absolutely necessary. Therefore, I consider it likely that we will see some subset of users only finding out about this script and wanting to run it once their -pin based DTS files no longer work.
Keeping the script around for one more release is thus useful for them since they will still have it in the tree that they want to test against. It can then be removed in v3.5.
Basically I am saying I think it makes sense to keep the migration mechanism around in the first release that makes migration mandatory.

Removed patch dropping migration tool. Will mention it again on the release notes when we reach 3.4-rc1.


split into two projects

one project that can be opened without errors by vs and one that can be opened by xamarin studio

FHMOBSDK-3


automatic brightness  not stable

Finally got this beautiful clock running. But run into another little problem. Got an ldr connected to gpio 34. In the clockwise settings changed the pin to 34. But whatever value I put in the settings page at automatic bright. The brightness keeps switching back and forth when I turn off the lights. The display changes brightness every couple off seconds. What am I doing wrong? Maybe a wrong ldr? Have tried several types.  But get stuck every time.

Hi! This is probably happening because the value read by the LDR is oscillating and it's enough to change the bright of display.  Have you tried to read the value when the room is dark and light? This is my suggestion, with the room dark, click on the "Read Pin" and get the value, do the same with the light on. Put the values in the automatic bright settings page. If it doesn't work, I think using a masking tape to block a little bit of the light can avoid this oscillation.
Reference here
https://github.com/jnthas/clockwise/wiki/Connecting-the-LDR

Im sorry jonathas, this dummy had not seen that there was also a wiki. Had already googled on the internet how to connect an ldr to an esp. But that was already in your wiki. Tried what you said, the values at dark were around 1100 (though my computer screen was on) And at lights on 4095. If I enter these values, it is still not right. But fill in the value 0 and 800, it does work satisfactorily. So how it works exactly I don't know, but it works now. Of course there is also a difference in the ldrs. I have a few types lying around. Must have something to do with the resistance. Beautifully made jonathas! Nice of you to share that with fellow tinkerers.
Again sorry, reading comprehension is also a profession!
Greetzz from holland
Harrie


About data loading

Hello, thank for sharing the great work. I have a quick question. When I train the CLAN model, I find that randomly mirror the images and randomly scale the crop_size are used fo r preprocessing data, which is a little different from previous works.
When training, I can think of it as data augmentation. But, why apply the above preprocessing to the test phase?
Thanks!

https://github.com/RoyalVane/CLAN/blob/d66fc258f71adb09a00fcdf854d20453e8263cd2/CLAN_evaluate.py#L91
The scale and mirror were set to "False" in test phase. Where did you find this problem?

Thanks for your reply.
In deed, the scale and mirror are set to False in test phase. But I still want to know how much performance improvement with the scale and mirror in training phase?
By the way, due to the scale process, sometimes  11G memory might be not enough.

Thank you for your question. In fact, the improvement of using "scale" and "mirror" augmentation is not significant in this UDA scenary. You could  simply set these parameters to false and evaluate the performance.


UNet1DUncondWrapper Initialization error due to missing in_channels argument

UNet1DUncondWrapper Initialization error due to missing in_channels argument
Description
I encountered an issue where the diffusion config does not appear to be passed correctly to UNet1DUncondWrapper from create_diffusion_uncond_from_config.
Despite specifying the in_channels in the diffusion configuration, the initialization of UNet1DUncondWrapper fails with an argument missing error.
Steps to Reproduce

Use the following model configuration:
{
  "model_type": "diffusion_uncond",
  "sample_size": 524288,
  "sample_rate": 44100,
  "audio_channels": 1,
  "model": {
    "type": "adp_uncond_1d",
    "diffusion": {
      "config": {
        "in_channels": 64
      }
    },
    "pretransform": {
      "type": "dac_pretrained",
      "config": {}
    },
    "training": {
      "learning_rate": 0.00004,
      "demo": {
        "demo_every": 1500,
        "demo_steps": 100,
        "num_demos": 1
      }
    }
  }
}

Pass this config to train.py (this configuration is incomplete, but offered as a minimal example as it produces the error reliably.

Expected Behavior
The model initialises and the in_channels parameter is set from the config.
Actual Behavior
The model initialization fails with the following error message:
```plaintext
File "/content/stable-audio-tools/stable_audio_tools/models/diffusion.py", line 568, in create_diffusion_uncond_from_config
    model = UNet1DUncondWrapper(
TypeError: __init__() missing 1 required positional argument: 'in_channels'
```

It seems that the diffusion:config may not be being properly passed to UNet1DUncondWrapper. Despite in_channels being explicitly defined and set, the value is not recognised.

Note, this can be bypassed by removing the diffusion key in the JSON and moving the config up a level in the structure. However, this is a different configuration to what seems 'standardised' across other configs, such as the conditional config.


implement test_recurring_jobs_for_detached_volume

https://github.com/longhorn/longhorn/issues/1509

This currently failed at step 15.

This currently failed at step 15.

@PhanLe1010 Can you help to review this?

Sure

Oops. I missed this comment

This currently failed at step 15.

I will retest this feature.


Update osu.py to add threading support and fix disconnects on unhandled exceptions

I've updated osu.py to add threading support for tasks and also fix disconnects on unhandled exceptions.
Here are the changelogs for that:

1.2.1
1.2.2

Good job as always!


Many backticks in same slide merges slides

If I attempt to use multiple backtick operators within the same slide, slides get merged. Please view the following example:
http://utdrmac.github.io/backtick_issue.html
This should be 3 separate slides. My only goal is to get those certain words to use the monospace font, like any other "console" type keywords. Using 3 backticks would be consistent behavior; using span or other would not be consistent.

Well, I discovered how to "fix" this by reading the lexer code. Seems I should only be using single backticks and not 3 of them.


Migrate Documents

Migrate documents from: https://github.com/OpenBCca/airbnb-regulation/wiki

Work with Reena to populate wikis.


[BUG] i was not able import the file

Describe the bug
run-as: package not debuggable: zeroxfourf.wristkey
I have installed the app on my wearOS watch [Samsung watch 7]. using the below command
adb -s <IP_ADDRESS>:46287 install app-release.apk
I have copied the file aegis app to my watch using the below command
% adb -s <IP_ADDRESS>:46287  shell run-as run-as zeroxfourf.wristkey cp /data/local/tmp/aegis-export-plain-6211495091.json  files/
now the issue is the app is not debuggable
run-as: package not debuggable: zeroxfourf.wristkey
this worked in the the previous version of the app

You'll need to clone the project and build it as a debug app in Android Studio. I am planning to add a feature to detect this in the next version, and a pipeline to build both debug and stable versions of the app for this purpose. For now, I can't seem to find the time to add all these things.
Feel free to make a pull request if it's urgent.


Make rules identified by abstract string rule id

Why you made the change:
I did this so that rules would exist and be identified by an abstract identifier
while still being modified.

👍


This project is currently not under active development

Hello everyone, we here at OneLogin wanted to let you know that this project is currently not under active development. We apologize for recent silence and continued wait, but we intend to resume maintenance in the future.
Note that I am unable to make any more changes to this repository, and I don't have someone I can forward you to at this time.

For others who stumbled here, see https://github.com/onelogin/java-saml/issues/388 for @pitbulk offering to take over maintenance of the various OneLogin open source SAML client repos.

Development Update. OneLogin is releasing these projects to a new organization with @pitbulk. This migration is actively happening and the priority is to make the transition as seamless as possible for end users of these repo/packages.
We expect that there are several questions that you all have and we are working with @pitbulk to answer those in our next update. Below are some answers we have for you now.
What is being changed?
The repos/packages will no longer be officially supported and hosted by OneLogin. This means that they will not be in the OneLogin Github org but in a new org, SAML Tools. References to the repos being provided and supported by OneLogin Inc will be removed.
Which projects are being moved?
All SAML repos will be moved. This includes: java-saml, python3-saml, wordpress-saml, moodle-saml, joomla-saml, drupal-saml, and dotnet-saml
When will this transfer happen?
We expect this to be completed by the end of the year, Dec 31 2022.
Why is this transfer happening?
OneLogin is releasing control of these open source repos so that these repos can maintained by community instead.
When will the next update be?
To keep you all informed of status, we will give monthly updates of how the transfer is proceeding.

@pitbulk - congratulations, Sixto

Development Update. Although it may not seem like it, the last month had a lot of progress and the primary SAML Toolkit repos and packages have been transferred from OneLogin to this new SAML Toolkit Org.
@pitbulk now has all the access needed to maintain these toolkits and will be providing his own update.
There has been a lot of pent up demand for support on these repos and now that this transfer is finished you should expect to see a lot more progress on that!
There are still four repos (wordpress-saml, moodle-saml, joomla-saml, drupal-saml) left to transfer and these will be finished in the upcoming weeks.
Cheers!

I gonna maintain the project, so this warning is not required anymore ;)


Define SUPPORTED_COUNTRIES docker env variable

We do not want to deliver default values for supported countries, as not defining them should result in an error. However, during development, we want those values to be available—especially for anybody new to the project. Solution: Define them in the docker script.
This PR solves:
submission_1         | ***************************
submission_1         | APPLICATION FAILED TO START
submission_1         | ***************************
submission_1         |
submission_1         | Description:
submission_1         |
submission_1         | Binding to target org.springframework.boot.context.properties.bind.BindException: Failed to bind properties under 'services.submission' to app.coronawarn.server.services.submission.config.SubmissionServiceConfig failed:
submission_1         |
submission_1         |     Property: services.submission.supportedCountries
submission_1         |     Value: null
submission_1         |     Reason: null
submission_1         |
submission_1         |
submission_1         | Action:
submission_1         |
submission_1         | Update your application's configuration

@roesslerj: It seems like your commit isn't signed by your GitHub user. Make sure the commits have the same e-mail from your Github user so we can easily track changes.

@fredrb: Thanks for pointing that out. I now added both email addresses to my GitHub account and will change the local one. How do I check that?


Update DXUT port and add Effects11 (FX) port

This updates the existing DXUT port to the latest version (it was last updated to the September 2016 release). The upstream project was updated with it's own CMake project and package support to simplify this port.
Also added a new Effects11 port for the Effects system for Direct3D 11. Again, the upstream project now provides a CMake.

Seems the format for vcpkg.json is not correct.
Could you please use vcpkg format-manifest ./ports/xxx/vcpkg.json to format it?

@NancyLi1013 It didn't like having a port-version of 0 in the vcpkg.json... Fixed.

@NancyLi1013 So I've fixed all the issues with dxut and effects11, sync'd master, and rebased.
The remaining failure isn't my port:
Error: While reading versions for port log4cpp from file: C:\a\1\s\versions\l-\log4cpp.json
        Version `2.9.1#4` was not found in versions file.
        Run:

           vcpkg x-add-version log4cpp

        to add the new port version.

Merged !16138 so that should fix that failure...

@ras0219-msft @NancyLi1013 I believe at this point all your feedback has been integrated. Please approve.

Yep, this is great! Thanks for the new versions and the update PR!


feat(core): maybe allow fields to be passed as map as well

in this format name would be the key and value would be rest of fields parameters

How would this work with field names? Is it then same kind of issue that arrays have?

Not going to proceed with this since

This would make fields more readable when working with map groupings, but would not help with array groupings
Defining value for the key (name) is difficult. It cannot be initial value in all cases since if it would be an object, it should contain the other field properties
Does not provide value for CMS setups, only when manually developing fields
In the end having only 1 way is the best solution


983 add visible anchors

Add visible anchors on hover
Summary
Add an Id Generator that provides ID to titles : "Features, Scenarios, Background, Rules..." and anchors that allow the
user to access to certain parts easily.
Details

Add a function Id Generator
Add Anchors on components
Add Components
Modify styles.scss

Motivation and Context
Resolve this issue :
https://github.com/cucumber/cucumber/issues/983
Screenshots
Anchor that appear when hovering :

URL when you clicked on the anchor :

Types of changes

[ ] Bug fix (non-breaking change which fixes an issue).
[x] New feature (non-breaking change which adds functionality).
[ ] Breaking change (fix or feature that would cause existing functionality to not work as expected).

Checklist:

[ ] The change has been ported to Java.
[ ] The change has been ported to Ruby.
[x] The change has been ported to JavaScript.
[ ] The change has been ported to Go.
[ ] The change has been ported to .NET.
[ ] I've added tests for my code.
[ ] My change requires a change to the documentation.
[ ] I have updated the documentation accordingly.
[ ] I have updated the CHANGELOG accordingly.

Right now the Feature header is farther to the right than its child headers. It should be the other way around.
Can you try to fix this? Otherwise looks great!

Yep ! I try to fix this !

It's fixed !


Replace broken link about use as a job queue

The author of "Generating Thousands of PDFs on EC2 with Ruby" reposted it on his blog.

Thank you. Boy does that page need a major overhaul. Links from 2009 😱.


Allow more capital characters and numbers in flag keys and variant keys

Allowing capital letters and numbers in flag keys and variant keys. This helps with use cases where keys are being migrated from legacy systems to flagr.
Description
Change flag key regex to accommodate to allow any word character, digits and hyphen.
Motivation and Context
This resolves issue 254.
https://github.com/checkr/flagr/issues/254
How Has This Been Tested?
Changed unit tests and tested locally by creating keys with capital letters and digits and other combinations.
Types of changes

[ ] Bug fix (non-breaking change which fixes an issue)
[ ] New feature (non-breaking change which adds functionality)
[ x] Breaking change (fix or feature that would cause existing functionality to change)

Checklist:

[ x] My code follows the code style of this project.
[ ] My change requires a change to the documentation.
[ ] I have updated the documentation accordingly.
[ x] I have added tests to cover my changes.
[x ] All new and existing tests passed.

Codecov Report

Merging #257 into master will not change coverage.
The diff coverage is n/a.

@@          Coverage Diff          @@
##           master   #257   +/-   ##
=====================================
  Coverage      82%    82%
=====================================
  Files          26     26
  Lines        1528   1528
=====================================
  Hits         1253   1253
  Misses        207    207
  Partials       68     68

Impacted Files
Coverage Δ

pkg/util/util.go
100% <ø> (ø)
:arrow_up:

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update 11a33a8...09aaad6. Read the comment docs.


Adds command aad user recyclebinitem list

Closes #4498

Phenomenal job! We'll try to review it ASAP!

Hi @MathijsVerbeeck
Seems like there is a merge conflict here, could you rebase with the latest main please? Thanks!

@milanholemans  These should be resolved. 😄

Seems like something went wrong with the pipelines after resolving some merge conflicts. Could you retrigger them @milanholemans ?

Merged manually. Awesome work! Thank you for this contribution! 👏


Fix: forbid repeated deepspeed.initialize on training objects

Previously closed PR:
https://github.com/microsoft/DeepSpeed/pull/6848
Partially Fixes: https://github.com/microsoft/DeepSpeed/issues/6772 https://github.com/microsoft/DeepSpeed/issues/6771 https://github.com/microsoft/DeepSpeed/issues/6770 by forbidding repeated initialization.
What are changed:

Marking 'model', 'optimizer' and 'lr_scheduler' in the arguments of deepspeed.initialize with the flag ds_is_inited = True.
Marking 'engine', 'engine.optimizer' and 'engine.lr_scheduler' in the return values of deepspeed.initialize with the flag ds_is_inited = True.
When calling deepspeed.initialize, raise an exception if detected ds_is_inited == True in the input model, optimizer or lr_scheduler

Expected Behavior:
Forbid repeated deepspeed.initialize invocations on model, optimizer and lr_scheduler objects.

@microsoft-github-policy-service agree

This fix still has interference with existing unit tests. Let me double check before we proceed.

The unit tests in tests/unit/runtime/test_ds_initialize.py all passed.
I am not able to check other unit tests due to GPU memory constraint.

Thanks for the review @tjruwase.

I added handling for callable types for optimizer and lr_scheduler. The handling is to only mark is_ds_inited for objects instead of callables, as the callables are not stateful and reuse should be allowed.

Regarding,

I think this call should be moved into _mark_trainobjs_initialized()
I think _assert_trainobjs_not_inited should still be separated from _mark_trainobjs_initialized as _mark_trainobjs_initialized is also called before deepspeed.initialize return the wrapped model and optimizers. The wrapped models may already have is_ds_inited being True since in DeepSpeedEngine all model flags will be passed through on the wrapper.

If we still want to keep _assert_trainobjs_not_inited inside _mark_trainobjs_initialized, we can do either of the three:

Thanks for raising this important design issue. Below are my thoughts and/or clarifications.

DeepSpeed is designed to optimize PyTorch, and currently does this by wrapping PyTorch constructs, e.g., nn.Module, optim.Optimizer, optim.lr_scheduler, etc.
To avoid nesting issues, DeepSpeed should not wrap DeepSpeed data structures, such as DeepSpeedEngine, DeepSpeedOptimizer, DeepSpeedDataLoader, etc.

Based on the above, I am aligned with your 3rd suggestion (below) of leveraging type information to simplify this PR.

3. check whether inited use type information as well, i.e. for types of DeepSpeedEngine we directly return inited == True instead of checking for flags.

What do you think?

Got it. Yeah 3 is technically what should be implemented instead of letting flags just flying around. Will get the update EOD.

@tjruwase Thanks for the suggestion. Please check the updated PR.


This patch allows FlagRemoveDotSegments to work with relative links --

i.e. URL's that have the Host field set to the empty string.
Just to be clear, relative links will be stored in URL's,
typically, if the user expects to later call URL.ResolveReference
to create a full URL. That seems to be a relatively common use-case.

Mentioning @PuerkitoBio so he can see this fork.

Thanks for the PR!

Hi!  This PR has caused some new errors to show up in the [Hugo](https://github.com/spf13/hugoTravis CI test runninggo test -v ./...`, e.g.:
https://travis-ci.org/spf13/hugo/jobs/47995348

I am not saying that there is anything wrong with this PR (I haven't looked at it yet, and I have to go outside in a few minutes), but just a note that some behaviour has changed that needs investigation.  :-)

Thanks, will revert for not until I have time to investigate further.
On Thu, Jan 22, 2015, 16:53 Anthony Fok<EMAIL_ADDRESS>wrote:

Hi! This PR has caused some new errors to show up in the [Hugo](
https://github.com/spf13/hugo` Travis CI test running go test -v ./...,
e.g.:
https://travis-ci.org/spf13/hugo/jobs/47995348
I am not saying that there is anything wrong with this PR (I haven't
looked at it yet, and I have to go outside in a few minutes), but just a
note that some behaviour has changed that needs investigation. :-)
—
Reply to this email directly or view it on GitHub
https://github.com/PuerkitoBio/purell/pull/5#issuecomment-71129767.

Hi @PuerkitoBio,
Wow, I didn't expect such a quick response!  Actually, perhaps this "patch to allows FlagRemoveDotSegments to work with relative links" is good, and perhaps it is Hugo that needs to modified for the potentially improved behaviour in purell.  :-)  Anyhow, I think I will leave this to people more capable than me like you, @bjornerik, @tatsushid and @owenwaller to look into.  :-)
Thanks again!

Oh sure , that revert is absolutely not a blame of the pr, it's just that I
can't take care of it right now so until then I'd rather not break people's
stuff until we understand what causes the change of behaviour.
On Thu, Jan 22, 2015 at 22:04 Anthony Fok<EMAIL_ADDRESS>wrote:

Hi @PuerkitoBio https://github.com/PuerkitoBio,
Wow, I didn't expect such a quick response! Actually, perhaps this "patch
to allows FlagRemoveDotSegments to work with relative links" is good, and
perhaps it is Hugo that needs to modified for the potentially improved
behaviour in purell. :-) Anyhow, I think I will leave this to people more
capable than me like you, @bjornerik https://github.com/bjornerik,
@tatsushid https://github.com/tatsushid and @owenwaller
https://github.com/owenwaller to look into. :-)
Thanks again!
—
Reply to this email directly or view it on GitHub
https://github.com/PuerkitoBio/purell/pull/5#issuecomment-71151522.

Thanks Martin! Looks like this is out of my hands right now. But I'll keep
my eyes peeled if there is something on this patch that I'll need to fix.
Thanks again.
-pete
On Thu, Jan 22, 2015 at 11:30 PM, Martin Angers<EMAIL_ADDRESS>wrote:

Oh sure , that revert is absolutely not a blame of the pr, it's just that
I
can't take care of it right now so until then I'd rather not break
people's
stuff until we understand what causes the change of behaviour.
On Thu, Jan 22, 2015 at 22:04 Anthony Fok<EMAIL_ADDRESS>wrote:

Hi @PuerkitoBio https://github.com/PuerkitoBio,
Wow, I didn't expect such a quick response! Actually, perhaps this
"patch
to allows FlagRemoveDotSegments to work with relative links" is good,
and
perhaps it is Hugo that needs to modified for the potentially improved
behaviour in purell. :-) Anyhow, I think I will leave this to people
more
capable than me like you, @bjornerik https://github.com/bjornerik,
@tatsushid https://github.com/tatsushid and @owenwaller
https://github.com/owenwaller to look into. :-)
Thanks again!
—
Reply to this email directly or view it on GitHub
https://github.com/PuerkitoBio/purell/pull/5#issuecomment-71151522.

—
Reply to this email directly or view it on GitHub
https://github.com/PuerkitoBio/purell/pull/5#issuecomment-71157008.

Hi guys, any word on this? Since this was reverted should the pull request be reissued?

Sorry, I didn't have much time to go back to this yet. No need to resubmit, I can revert the revert if required.

The PR is ready in the fix-relative-paths branch, I opened an issue in the hugo repository to give them a chance to update their code first so that we don't break things. Hopefully it's a simple fix on their side and we can synchronize a merge to master soon. I do believe the PR is correct and does fix a buggy behavior of purell with relative paths.


Allow other Encoder implementations?

Hey, this is a cool lib! wondering if it makes sense to allow encoder implementations as arguments to the use macro, keeping Jason.Encoder as default? Thinking binary protocols etc

Hey, Thanks for taking a look!
I'm open to this. Feel free to submit a PR!
Otherwise I'm not sure when I'll get to it


Some bug in read_buffer, dump_buffer

When examples are run with these lines on:
    dump_buffers(pos_buf, vel_buf, enr_buf, hit_buf, "tmp")
    pos_buf, vel_buf, enr_buf, hit_buf = read_buffers("tmp")

The output plot looks as follows:

The same simulation run without these lines:

Related: https://github.com/krzysztofarendt/building3d/issues/81

I don't see this issue anymore:

Results from: https://github.com/krzysztofarendt/building3d/pull/88

Seems to be fixed.


better style event:list

This help to better style event:list command

Please can you include a screenshot of the difference here.

if instead of INFO can write Event or Listener i think is very nice :)
Before

After


Webhooks guide 1: Discussion bot powered by BLOOM

(i'll move the assets before merging)

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint.


Uncaught ReferenceError: global is not defined

What is the location of your example repository?
https://github.com/Nosto/shopify-hydrogen-demo, branch: ads-4544-upgrade-major-version-demo-2
Which package or tool is having this issue?
Oxygen
What version of that package or tool are you using?
@shopify/hydrogen: 2024.4.1, @shopify/remix-oxygen: 2.0.4, @shopify/mini-oxygen: 3.0.1
What version of Remix are you using?
2.9.1
Steps to Reproduce
I'm trying to upgrade my Remix and Hydrogen version to the latest (2.9.1 and 2024.4.1, respectively). And so also need to upgrade to latest versions of dependent libraries, one of those was mini-oxygen, which is now a requirement.
When I try to run locally with shopify hydrogen dev --codegen, I get an error with Miniflare:
Environment variables injected into MiniOxygen:

SESSION_SECRET                from local .env
PUBLIC_STOREFRONT_API_TOKEN   from local .env
PUBLIC_STORE_DOMAIN           from local .env
╭─ error ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
│                                                                                │
│  The Workers runtime failed to start. There is likely additional logging output above.                                                                          │
│                                                                                │
│  To investigate the issue, examine this stack trace:                                                                                │
│    at #assembleAndUpdateConfig (miniflare/dist/src/index.js:8868)                                                                                │
│      throw new MiniflareCoreError(                                                                                │
│    at processTicksAndRejections (node:internal/process/task_queues:95)                                                                                │
│    at runWith (miniflare/dist/src/index.js:3420)                                                                                │
│      return await awaitable;                                                                                │
│    at async #waitForReady (miniflare/dist/src/index.js:8925)                                                                                │
│      await this.#initPromise;                                                                                │
│    at async startWorkerdServer (@shopify/cli-hydrogen/dist/lib/mini-oxygen/workerd.js:89)                                                                       │
│      const { workerUrl, inspectorUrl } = await miniOxygen.ready;                                                                                │
│    at async runPreview (@shopify/cli-hydrogen/dist/commands/hydrogen/preview.js:69)                                                                             │
│      const miniOxygen = await startMiniOxygen(                                                                                │
│    at run (@shopify/cli-hydrogen/dist/commands/hydrogen/preview.js:28)                                                                                │
│      await runPreview({                                                                                │
│    at runCommand (@shopify/cli/dist/chunk-3MMF34G4.js:210764)                                                                                │
│      const result = await command.run(argv, this);                                                                                │
│    at run (@shopify/cli/dist/chunk-3MMF34G4.js:212426)                                                                                │
│      return await config.runCommand(id, argvSlice, cmd);                                                                                │
│                                                                                │
╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯

    MiniflareCoreError [ERR_RUNTIME_FAILURE]: The Workers runtime failed to start. There is likely additional logging output above.
    Code: ERR_RUNTIME_FAILURE

But this isn't all, I thought perhaps I could do an Oxygen deployment and my environment (for e.g. workerd) would not be a hinderance. While doing this I get the following, even more obscure error:
Deploying to Oxygen..
Deployment failed, error: Uncaught ReferenceError: global is not defined
  at worker.mjs:19:32086
  at worker.mjs:19:36089
  at worker.mjs:1:246
  at worker.mjs:81:30979
╭─ error ──────────────────────────────────────────────────────────────────────╮
│                                                                              │
│  Uncaught ReferenceError: global is not defined                              │
│    at worker.mjs:19:32086                                                    │
│    at worker.mjs:19:36089                                                    │
│    at worker.mjs:1:2[46](https://github.com/Nosto/shopify-hydrogen-demo/actions/runs/8938871131/job/24553842268#step:6:47)                                                       │
│    at worker.mjs:81:30979                                                    │
│                                                                              │
│                                                                              │
│  Retrying the deployment may succeed.                                        │
│                                                                              │
│  Next steps                                                                  │
│    • If the issue persists, please check the Shopify status page [1] for     │
│      any known issues.                                                       │
│                                                                              │
╰──────────────────────────────────────────────────────────────────────────────╯
[1] https://status.shopify.com/
▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀
Verifying deployment has been completed ...
Error: Process completed with exit code 1.

This doesn't really point me to anything at all, I've played around with a few different versions but always come back to this error. Could someone please enlighten me around what could be happening here?
My Github action that has this error (as well as my Shopify Hydrogen demo store) is here: https://github.com/Nosto/shopify-hydrogen-demo/actions/runs/8938871131/job/24553842268. The branch in which I'm upgrading to the latest version is here: https://github.com/Nosto/shopify-hydrogen-demo/pull/41 (PR for diff of changes from a fully functional main branch).
Expected Behavior
Oxygen deployment should work.
Actual Behavior
Using mini-oxygen, version 3.0.1, results in an unexpected error (described above)

at async runPreview (@shopify/cli-hydrogen/dist/commands/hydrogen/preview.js:69)

Given the stack, it seems you're running the preview command instead of dev?
Try running the dev or preview commands with the --verbose flag, it should show you more Miniflare logs that might help find the issue.

About the h2 upgrade command, could you please point me to some docs around this? Would you mean the H2 db in this case? Just want to make sure I set it up right.

I mean the CLI command, shopify hydrogen upgrade or h2 upgrade if you have the shortcut installed. Docs: https://shopify.dev/docs/api/shopify-cli/hydrogen/hydrogen-upgrade
This command ensures you get valid versions of each package when updating.

With the --verbose flag I get the below in my local. This seems to be coming from the js-sha256 module, but I'm not sure it's linked to my .env, as it's the same error in my local but also via Oxygen deployment when I push to the branch.

I don't think this has to do anything with .env. One of your dependencies is probably accessing global.something and that breaks in workers environments because global is a Node.js-only API. The standard would be globalThis.something.
You could try polyfilling it with something like globalThis.global = globalThis or similar but you need to ensure it runs before the dependency accesses it.
That said, I see in your repo that you are bumping a Hydrogen and Remix many versions without changing the app code. A lot of things have been updated that need changes in-app. You can see this in the changelogs of each package or, if you use h2 upgrade command, a markdown file will be generated with the required changes to your code.
However, it might be simpler for you to start from scratch with a new Hydrogen template and apply your changes, if it's not a lot. You can start again with npm create @shopify/hydrogen@latest with the skeleton template, or add --template demo-store to get Hydrogen's demo store.


Crash at mount time

On MacOS X 10.11.3, SSHFS GUI crashes at mount time. If launched from Finder, the mount dies with the app. If started from the Terminal, the mount remain after the app crashes.
The details produced by OSX  are attached. It seems there is a strlen() used on a NULL pointer:
crash.txt

Should be fixed!


[develop] Feature/fave 17276 progress

Description
Adds progress component
Type of change
Please delete options that are not relevant.

[x] Bug fix (non-breaking change which fixes an issue)

Checklist:

[x] My code follows the style guidelines of this project
[x] I have performed a self-review of my own code
[x] I have commented my code, particularly in hard-to-understand areas
[x] My changes generate no new warnings

QA and UI approved in https://github.com/kfit-dev/ui-components/pull/31


Generic Input/Output of DataFrames

This PR includes:

Extended former InputLocation classes to support both read and write functions, also renamed it to DataLocation to reflect this change.
Added a GenericLocation which supports all Spark confs, modes, and options, so it can be used to operate virtually any connectors supported by Spark
In GenericLocation, I added a format-specific patching mechanism to workaround quirks come with different connectors, e.g. CosmosDb requires rows to have an id column with unique values.
Update FeathrGenJob and FeathrJoinJob, enabling them to use a JSON-encoded string of DataLocation instead of plain path as the input and output target.

Theoretically, Feathr core can support all Spark connectors with this patch, but we still run a series of compatibility tests to confirm the final list.
NOTE: This PR only involves Feathr core, the corresponding Feathr Client changes will be in upcoming PRs.

This PR looks good to me and is a good way of extending to other sources in the future. Thanks @windoze for the work!


How to migrate system node pool to another vm size without any downtime

Hi,
What is the easiest way to change the system node pool vm size without any downtime?
I have a running cluster with a single system node pool of 3 nodes, vm size Standard_DS2_v2. I want to have instead a single system node pool of 2 nodes of vm size Standard_E2as_v4.
I see that I can create a new system node pool of the new vm size, and then delete the old system node pool, but how can I tell K8s to move all pods from old to new node pool, before I delete the old one, so that there is zero downtime for the clients? Or does this happen automatically and I should not worry at all?
Best regards,
Deyan

Document Details
⚠ Do not edit this section. It is required for docs.microsoft.com ➟ GitHub issue linking.

ID: 225b99ee-4c4d-54c2-cafa-ddcd286438da
Version Independent ID: ca9410a8-8ddb-5528-a0ac-0963458f190e
Content: Use system node pools in Azure Kubernetes Service (AKS) - Azure Kubernetes Service
Content Source: articles/aks/use-system-pools.md
Service: container-service
GitHub Login: @mlearned
Microsoft Alias: mlearned

@deyanp
Thanks for your Question! We are currently investigating and will get back on this.

@MicrosoftDocs/aks-pm Can you please check and add your comments.

@deyanp
You can try creating a new pool with the other size as per your requirement,and ensure you have more than one replica for your application.
Then apply the PodDisruptionBudgets to ensure at least 1 or more replica are kept running. The go ahead and Drain old nodepool to new and optionally delete old nodepool.
Recommending to try this in non-production environment.

@deyanp
Hope the provided information is helpful.
We will now close this issue. If there are further questions regarding this, please tag me in a comment. I will reopen it and we will continue the discussion.


Actions cluster test support

Problem
Support for testing actions clusters is missing.
Change overview
Support for testing action clusters was added to the bridge-app.
Testing
The updates to bridge-app were tested using chip-tool
The following command can be used to get the action cluster endpoint list
chip-tool bridgedactions read endpoint-list 12344321 1

Start bridge app and wait for initialization to complete
Run chip-tool pairing
chip-tool pairing ethernet 12344321 20202021 3840 localhost 5540
Run chip-tool to verify endpoint lists have 2 devices in "Room 1" and 2 devices in "Room 2"
Press r in the bridge-app to rename "Room 1" to "Room 1 renamed" (TC-ACT-2.2 step 2c)
press f in the bridge-app to move "Action Light 3" from "Room 2" to "Room 1" (TC-ACT-2.2 step 2f)
press i in the bridge-app to remove "Room 2" from exposed endpoint lists (TC-ACT-2.2 step 2i)
press l in the bridge-app to add "Zone 1" and include "Action Light 2" in the new zone (TC-ACT-2.2 step 2l)

Fast tracking, given this has had enough time for review > 3 days

/rebase


Error after clicking title screen

Got the data dump from my smartphone and decrypt it.
Run UMO and title screen appear, after clicking it got black screen, cursor and BGM music are still working.
error log:
TODO
UnityEngine.Debug:LogError(Object)
TodoLogger:LogError(Int32, String) (at Assets/Scripts/TodoLogger.cs:75)
<LGFPCADOCAA_ShowError>d__33:MoveNext() (at Assets/UMAssets/Scripts/IMMAOANGPNK.cs:337)
UnityEngine.MonoBehaviour:StartCoroutine(IEnumerator)
MonoBehaviourExtensions:StartCoroutineWatched(MonoBehaviour, IEnumerator) (at Assets/Scripts/CoroutineWatcher.cs:13)
<MHEKMICKGDM_LoadFromStorage>d__32:MoveNext() (at Assets/UMAssets

Hello,
From what I unterstand, yuo try to run the game on PC. Where did you get the data, when you say from your phone, is it from the old game ?

Its from the game just before the server shutdown (around June 2022 I think).
I copied the android and db folders from my smartphone to my PC just as the instruction says (took almost 1 full day to copy all the files).
However I noticed that the directory structure from my game is rather different from the one you posted on the wiki, noticeably is the ar folder.
Is there something wrong with my game data?
It works fine in my smartphone before the server shutdown.

The original game was downloading data on demand and deleting unused one to not take too much place on the phone. But you need a full dump now for it to works. This can either be done :

With the tools here if the server still works : https://github.com/Xele02/UMO/tree/main/Tools
By downloading one of the dump that was done. I'll see if we still have something uploaded ready somewhere.

You can dld the dump on this google drive. Tell me when you got the link, I'll delete this message : https://drive.google.com/file/d/16RsoJ7uOMcwp_sNQijNwz8DXAyVJpryD/view?usp=drivesdk

Ok, I got the link.
You can delete the link now.
I'll update as soon as I finished with the data dump.

Did a clean install of UMO 5.0 and use th dl game data.
Still stuck on blackscreen after title.
These are the error logs:
TODO
UnityEngine.Debug:LogError(Object)
TodoLogger:LogError(Int32, String) (at Assets/Scripts/TodoLogger.cs:76)
JHHBAFKMBDL:GKMAHMLNMEK(IMCBBOAFION, String) (at Assets/UMAssets/Scripts/JHHBAFKMBDL.cs:239)
<MHEKMICKGDM_LoadFromStorage>d__33:MoveNext() (at Assets/UMAssets/Scripts/IMMAOANGPNK.cs:328)
UnityEngine.SetupCoroutine:InvokeMoveNext(IEnumerator, IntPtr) (at C:/buildslave/unity/build/Runtime/Export/Coroutines.cs:17)
Something wrong with my Unity setup?

Yeah, but I don't think it's from the data. Can you link the full log which can be found here :

Operating system
Log files

Linux
~/.config/unity3d/Editor.log

macOS
~/Library/Logs/Unity/Editor.log

Windows
C:\Users\username\AppData\Local\Unity\Editor\Editor.log

https://github.com/Akhmadi14/Others/blob/main/Editor.log

Thx, I'll try to see if I understand what's wrong. In the mean time, you can try  to delete the directory C:/Users/username/AppData/LocalLow/UtaMacross/UtaMacross/ . This is where are saved some local file and perhaps the first run with your phone data wrote something that the current version don't like.

And to be sure, the data of the full dump is the one in "F:\UMO\Tools\Data/" (this is where the editor search in the log) , and you removed the previous one if it was in the same directory.

Ok, I'll try to do test on fresh account to see if I have the problem cause for now I don't really why this is happening

Hello, I try to do some test, but didn't had your problem :

Removed the profile directory
Dld and extract 0.5 zip
Dld the data from the google drive and extracted.
Opened unity hub and loaded theproject in the Unity Directory.
In the editor, Menu UMO > Start
Selected the directory with the data from the gg drive.
Game started normally.

So for another test, can you open the option (UMO / Options) and change the log number to 999 :

And resend the unity log after running again the game with those options. I will see if I can find more info with more log.

This are the newest editor log.
https://github.com/Akhmadi14/Others/blob/main/Editor.log
I did the same step as you above, and still doesn't work. I suspect something in my Unity asset might be corrupt or something.
I'm using Unity editor 2018.4.23f1.

Looks like the game don't find those files which should be in the data zip :
F:\UMO\Tools\Data/db/ar_event!sa4130fcbz!.dat
F:/UMO/Tools/Data/db/md-20220626-000000_v00007790_s1_h00000000!s602bc890z!.dat
F:/UMO/Tools/Data/db/sd!s6b10b98cz!.dat

Ok, seems like I got some error on the external hard-drive where the main data is stored (some files got corrupted including the one you mention above) which causing those errors.
I'm gonna sort the problems then check it out again.

Any news ? I'll close the bug in 1 week considering it's fine now


Installation should add a link in pfSense to UniFi controller

I can look into doing this if a PR is welcome

Have at it. It was among the original goals of the project.

I got mine working by adding a line to head.inc
$services_menu[] = array(gettext("UniFi Controller"), "/services_unifi.php");
Then creating a services_unifi.php file that contains:
<?php require("guiconfig.inc"); header("location: https://<IP_ADDRESS>:8443"); ?>
Only problem is, i had to manually insert my ip of the controller.  Anyone know of a better way?

Wouldn't $_SERVER['SERVER_ADDR'] work for that?
Do you have this in a branch on github we can look at?

I suppose the install.sh script could check whether services_unifi.php exists and if not, then add it in the way we do the .rc file, and then check for the line in head.inc and insert it if necessary…

Thanks for your reply John.  I'm not a PHP dev by day lol.  I don't have anything on github at the moment.

Well thanks for the suggestion and example. It’s a simple solution. The next step is to automate.

You're welcome.  I like the idea behind this.  If I can help in testing in any way let me know.

Thanks inetsec!  Worked great.

any updates on this feature?

Have a nice day,
is there any progress in this thought? It is almost unknown in Linux, but I am no stranger to PHP, HTML and writing C C code for x86 and PICs. So, if you help me stim. So in the evening I can add this to the original installation script. I would love to have a UniFi driver in the PfSense firewall. So far, I've installed the UniFi driver from the console. But I can't find out if the driver works, I don't know how and I don't know how to get to the UniFi web interface of the driver.
If you tell me exactly where to find certain files, I could give it in PHP.
Have a nice evening Pavel Prochazka.zde.cz

Wouldn't $_SERVER['SERVER_ADDR'] work for that?
Do you have this in a branch on github we can look at?

require("guiconfig.inc");
$ip_host = "https://" . ($_SERVER['SERVER_ADDR']) . ':8443';
header("location: $ip_host");


MATLAB catastrophic crashes

Debugging protocols is difficult as MATLAB often suffers a fatal error with no useful information to help solve the issue.  I've noticed that there are a number of TODOs in the signals C code, all pertaining to better error handling.  Hopefully fixing these will help reduce the fatal MATLAB errors.

[ ] handle allocation fails etc in sqAddNode
[ ] trap and report errors when notifying event targets in sqApply
[ ] ensure idx >= 0 && id < net.nNodes && net.nodes[id].inUse in nodePtrs
[ ] check size and data type of src and dst arrays match in setNodeCurrValue
[ ] recompile MEX files with debug symbols(?)

fatal crash when trying this assignment to the visStim subscriptable signal handler in an exp def:
startTime = t.at(events.expStart.map(true));
curExpTime = t - startTime;
circle = vis.patch(t, 'circle');
circle.azimuth = 3 * curExpTime; % this assignment causes the crash
visStim.circle = circle;

maybe because trying to render the azimuth too quickly?

This is occurring because the 'azimuth' property for the 'vis.patch' signal is trying to be set to an empty value. This occurs in the above code since 't' does not get posted to until after 'events.expStart' gets posted to. This should be documented and a fix should be implemented.

@jaib1 What you're talking about may be related to cortex-lab/Rigbox#44.  Are you running in a rig with a pre-delay > 0?

yes, I think they may be related, though I was not using a pre-delay. I also believe I've had similar crashes when trying to update a visual stimuli via a signal that does not take a value - I'll try to replicate this.
Regarding 't' and 'events.expStart', do you know why 'startTime' in this assignment
startTime = t.at(events.expStart.map(true));
does not take a value?

Also note that the main loop begins (and therefore t updates) before expStart updates.

I don't think this is true without a preDelay.

also, assigning an origin signal to a visStim handler element causes a MATLAB fatal crash. e.g:
os = t.Node.Net.origin('os');
os.post(0);
rect = vis.patch(t, 'rectangle')
rect.azimuth = os;
visStim.rect = rect;

Regarding 't' and 'events.expStart', do you know why 'startTime' in this assignment
startTime = t.at(events.expStart.map(true));
does not take a value?

https://github.com/dendritic/signals/blob/c2cf2854491525905ed9f21e0d708b451bfce988/%2Bsig/%2Btransfer/at.m#L17-L28
It's exactly as you say: at the time events.expStart updates t has no value, so how could it update?  startTime would update if you had non-zero pre-delay.

also, assigning an origin signal to a visStim handler element causes a MATLAB fatal crash. e.g:
os = t.Node.Net.origin('os');
os.post(0);
rect = vis.patch(t, 'rectangle')
rect.azimuth = os;
visStim.rect = rect;

Yes I can reproduce it with the following code:
clear all
[t, setElemsFun] = sig.playgroundPTB;
sigbox = t.Node.Net;
rect = vis.patch(t,'rect');
os = sigbox.origin('os');
rect.azimuth = os;
os.post(0.0)
setElemsFun(struct('rect', rect));

rect.show=true;
os.post(1)

Caught "std::exception" Exception message is:
FatalException

Posting to an origin signal before creating dependent signals is pointless: that value won't be propagated.  At first I thought that was the cause of your error.  For me MATLAB only crashes when posting to the origin signal after show is set to true.

Posting to an origin signal before creating dependent signals is pointless: that value won't be propagated. At first I thought that was the cause of your error

Right, which is why in my code snippet I didn't use a dependent signal based on 'os', I used 'os' directly. Just to be sure, this makes sure this wouldn't be the cause of the error, right?

For me MATLAB only crashes when posting to the origin signal after show is set to true.

Hm, are you sure it's only after? That's a bit surprising. Any thoughts on why this would be different for us? What version of MATLAB are you running on your laptop?

Are you using your test version of SignalsExp, if so maybe that's the difference?  I'm using 2018b.  If I run the following I don't have any problems:
clear all
[t, setElemsFun] = sig.playgroundPTB;
sigbox = t.Node.Net;
rect = vis.patch(t,'rect');
os = sigbox.origin('os');
rect.azimuth = os;
os.post(0.0)
setElemsFun(struct('rect', rect));
% Start timer %
os.post(0.0)
os.post(1)
os.post('h')
os.post(3)

But in that code snippet, you are not setting:
rect.show = true
There are a few issues that make it impossible for me to run 'sig.playgroundPTB' in it's current form (incompatibility with my version of GUI layout toolbox, incompatibility with how the screens are defined, etc...), but if you run this exact expdef
function visTest(t, events, params, visStim, inputs, outputs, audio)
%VISTEST For testing visual stimuli

  os = t.Node.Net.origin('os');
  os.post(1);
  rect = vis.patch(t, 'rectangle');
  rect.azimuth = os;
  rect.show = true;
  visStim.rect = rect;
  events.endTrial = events.newTrial.delay(5);

end

and it doesn't crash, then something very strange indeed is happening. For me, the crash occurs regardless of where or whether or not I post to 'os' in the expdef, simply as long as 'os' is defined as an origin signal and fed into one of the elements of 'rect', and 'rect.show' is set to 'true'.

RE sig.playgroundPTB: Could you create an issue and address these issues as a matter of urgency (create a pull request; don't hotfix)?  We need playgroundPTB to work smoothly on most computers as it's used in the Ringach example and should be used in other GUI-based examples.

But in that code snippet, you are not setting:
rect.show = true

Yes, as I said earlier MATLAB only crashed when posting to the signal after the stimulus is shown.  If I run the above code I don't get a crash.  The only difference if that show is set to false.

When I run the code posted here MATLAB doesn't crash.  If I then type rect.show = true;, the next time I update the origin signal MATLAB crashes.

and does it crash if you run this:
clear all
[t, setElemsFun] = sig.playgroundPTB;
sigbox = t.Node.Net;
rect = vis.patch(t,'rect');
os = sigbox.origin('os');
os.post(0.0)
rect.azimuth = os;
rect.show = true;
setElemsFun(struct('rect', rect));

and this:
clear all
[t, setElemsFun] = sig.playgroundPTB;
sigbox = t.Node.Net;
rect = vis.patch(t,'rect');
os = sigbox.origin('os');
rect.azimuth = os;
rect.show = true;
setElemsFun(struct('rect', rect));

?

Yes

also occasionally get crashes when running clear all in the command window where a network and signals have been defined.

'onValue' causes MATLAB crashes occasionally:
e.g.
net = sig.Net;
os = net.origin('os');
l = feval(@onValue, os, @(x) disp(x));
os1.post(1);

(I realize this is not typically the syntax used, but this syntax is what caused the crash)


Test

[Enter feedback here]

Document Details
⚠ Do not edit this section. It is required for docs.microsoft.com ➟ GitHub issue linking.

ID: 996199f0-22c1-253c-30c3-500c7a814582
Version Independent ID: 382f62c2-5143-6795-6358-28916f0d11de
Content: What is the Anomaly Detector API? - Azure Cognitive Services
Content Source: articles/cognitive-services/Anomaly-Detector/overview.md
Service: cognitive-services
Sub-service: anomaly-detector
GitHub Login: @aahill
Microsoft Alias: aahi

Testing closing this issue with please close. #please-close


Using stretchr/testify

Do we want to use https://github.com/stretchr/testify to simplify tests?

No objection from me, we use it in other Prometheus repos.
What do you think @discordianfish?

Sure, assuming it simplifies the tests


Fixed readPROCAR() to use updated FatBands constructor

This should in principle fix #57 but I'll have @xamberl review this change (I don't have any PROCAR files to test on yet)

I realized I left an old variable from a different fix attempt, but I removed that.


decrypt virtual machines with sesparse.vmdk format

Hello.
Got an infection of a server with several virtual machines. Using your script managed to restore 3 out of 4 machines, windows server 2016 booted without any problems. There was a problem with the machine on which was snapshot. This is the list of files of the virtual machine:

When specify AstRun_srv.vmx script, it passes successfully however after registering the machine does not start:
Failed to power on virtual machine AstRun_srv. File AstRun_srv_1-000001.vmdk was not found
Here is the content of the .vmx file:
.encoding = "UTF-8"
config.version = "8"
virtualHW.version = "14"
vmci0.present = "TRUE"
floppy0.present = "FALSE"
numvcpus = "12"
memSize = "32768"
bios.bootRetry.delay = "10"
firmware = "efi"
powerType.suspend = "soft"
tools.upgrade.policy = "manual"
sched.cpu.units = "mhz"
sched.cpu.affinity = "all"
vm.createDate = "1593152686125618"
scsi0.virtualDev = "lsisas1068"
scsi0.present = "TRUE"
sata0.present = "TRUE"
usb_xhci.present = "TRUE"
scsi0:0.deviceType = "scsi-hardDisk"
scsi0:0.fileName = "AstRun_srv-000001.vmdk"
sched.scsi0:0.shares = "normal"
sched.scsi0:0.throughputCap = "off"
scsi0:0.present = "TRUE"
scsi0:1.deviceType = "scsi-hardDisk"
scsi0:1.fileName = "AstRun_srv_1-000001.vmdk"
sched.scsi0:1.shares = "normal"
sched.scsi0:1.throughputCap = "off"
scsi0:1.present = "TRUE"
ethernet0.virtualDev = "e1000e"
ethernet0.networkName = "LAN3_Grp_AstRus"
ethernet0.addressType = "generated"
ethernet0.present = "TRUE"
displayName = "AstRun_srv"
guestOS = "windows9-64"
uefi.secureBoot.enabled = "TRUE"
toolScripts.afterPowerOn = "TRUE"
toolScripts.afterResume = "TRUE"
toolScripts.beforeSuspend = "TRUE"
toolScripts.beforePowerOff = "TRUE"
tools.syncTime = "FALSE"
uuid.bios = "56 4d 8e 2c 82 0a 68 1c-da 6d ba d0 fb 06 c3 97"
uuid.location = "56 4d 8e 2c 82 0a 68 1c-da 6d ba d0 fb 06 c3 97"
vc.uuid = "52 b7 2d 1d 64 9b 2d 63-2d 89 6a a7 58 68 b5 ca"
sched.cpu.min = "0"
sched.cpu.shares = "normal"
sched.mem.min = "0"
sched.mem.minSize = "0"
sched.mem.shares = "normal"
ethernet0.generatedAddress = "00:0c:29:06:c3:97"
vmci0.id = "-83442793"
cleanShutdown = "FALSE"
extendedConfigFile = "AstRun_srv.vmxf"
mks.enable3d = "TRUE"
tools.guest.desktop.autolock = "FALSE"
nvram = "AstRun_srv.nvram"
pciBridge0.present = "TRUE"
svga.present = "TRUE"
pciBridge4.present = "TRUE"
pciBridge4.virtualDev = "pcieRootPort"
pciBridge4.functions = "8"
pciBridge5.present = "TRUE"
pciBridge5.virtualDev = "pcieRootPort"
pciBridge5.functions = "8"
pciBridge6.present = "TRUE"
pciBridge6.virtualDev = "pcieRootPort"
pciBridge6.functions = "8"
pciBridge7.present = "TRUE"
pciBridge7.virtualDev = "pcieRootPort"
pciBridge7.functions = "8"
hpet0.present = "TRUE"
RemoteDisplay.maxConnections = "-1"
sched.cpu.latencySensitivity = "normal"
svga.autodetect = "FALSE"
disk.EnableUUID = "TRUE"
numa.autosize.cookie = "120001"
numa.autosize.vcpu.maxPerVirtualNode = "12"
sched.swap.derivedName = "/vmfs/volumes/5ebac8f4-ba43223e-d524-309c239ced34/AstRun_srv/AstRun_srv-625b630d.vswp"
pciBridge0.pciSlotNumber = "17"
pciBridge4.pciSlotNumber = "21"
pciBridge5.pciSlotNumber = "22"
pciBridge6.pciSlotNumber = "23"
pciBridge7.pciSlotNumber = "24"
scsi0.pciSlotNumber = "160"
ethernet0.pciSlotNumber = "192"
usb_xhci.pciSlotNumber = "224"
vmci0.pciSlotNumber = "32"
sata0.pciSlotNumber = "33"
scsi0.sasWWID = "50 05 05 6c 82 0a 68 10"
ethernet0.generatedAddressOffset = "0"
vm.genid = "1025359355794109798"
vm.genidX = "5812918138057738809"
monitor.phys_bits_used = "43"
vmotion.checkpointFBSize = "4194304"
vmotion.checkpointSVGAPrimarySize = "67108864"
softPowerOff = "FALSE"
toolsInstallManager.lastInstallError = "0"
svga.guestBackedPrimaryAware = "TRUE"
tools.remindInstall = "FALSE"
toolsInstallManager.updateCounter = "2"
migrate.hostLog = "./AstRun_srv-625b630d.hlog"
svga.vramSize = "67108864"
sata0:0.startConnected = "FALSE"
scsi0:0.redo = ""
scsi0:1.redo = ""

When I try to feed the AstRun_srv-000001 file to the script, it gives the following errors:
[root@static:/vmfs/volumes/5ebac8f4-ba43223e-d524-309c239ced34/AstRun_srv] /tmp/recover.sh AstRun_srv-000001
mkdir: can't create directory 'encrypted_files': File exists
Moving encrypted AstRun_srv-000001.vmdk to encrypted_files
mv: can't rename 'AstRun_srv-000001.vmdk': No such file or directory
ls: AstRun_srv-000001-flat.vmdk: No such file or directory

Creating copy of AstRun_srv-000001-flat.vmdk
Invalid file length specifier: -d
rm: can't remove 'temp-flat.vmdk': No such file or directory

Adding AstRun_srv-000001.vmdk
sed: temp.vmdk: No such file or directory
sed: temp.vmdk: No such file or directory
mv: can't rename 'temp.vmdk': No such file or directory

Copying AstRun_srv-000001.vmx
mv: can't rename 'AstRun_srv-000001.vmx': No such file or directory
cp: can't stat 'AstRun_srv-000001.vmx~': No such file or directory
Error: unable to find vmx backup. You may be unable to re-register the virtual machine.

Moving encrypted AstRun_srv-000001.vmsd to encrypted_files
mv: can't rename 'AstRun_srv-000001.vmsd': No such file or directory

Moving encrypted AstRun_srv-000001.nvram to encrypted_files
mv: can't rename 'AstRun_srv-000001.nvram': No such file or directory

Validating...
Failed to open disk link /vmfs/volumes/5ebac8f4-ba43223e-d524-309c239ced34/AstRun_srv/AstRun_srv-000001.vmdk :The system cannot find the file specified (25)Disk chain is not consistent : The system cannot find the file specified (25)

Error. Trying to update the file size.
sed: AstRun_srv-000001.vmdk: No such file or directory
Failed to open disk link /vmfs/volumes/5ebac8f4-ba43223e-d524-309c239ced34/AstRun_srv/AstRun_srv-000001.vmdk :The system cannot find the file specified (25)Disk chain is not consistent : The system cannot find the file specified (25)

Error. Could not recover. Please consult CISA's guidance for further information: https://www.cisa.gov/uscert/ncas/alerts/aa23-039a

I guess because of the snapshot I have a disk divided into files AstRun_srv_1-000001-sesparse.vmdk but script is looking for *flat.vmdk. Maybe someone has a solution how to run a virtual machine in this case or  just pull the files from the drive? Maybe redo the script so that it would work on the files AstRun_srv_1-000001-sesparse.vmdk; AstRun_srv-000001-sesparse.vmdk; AstRun_srv-Snapshot1.vmem ?
If connect the disks AstRun_srv.vmdk and AstRun_srv_1.vmdk in the new created virtual machine - it starts and works correctly but there is old data.

may be its complete insanity, i change word in script flat ---> sesparse and start it with:
/tmp/recover.sh AstRun_srv_1-000001 thin
i got:
...
Validating...
Disk chain is consistent.

Success! Unregister the virtual machine and re-register it and you should be good to go.

after i try add disk to another virtual machine, i see correct size disk before encrypting, but inside didn't detect any partition or data, R-STUDIO after scaning show all unrecognized sectro =(

i go another way, i rename *-sesparse.vmdk to *-flat.vmdk in virtual machine folder, run script - success, reaname back *-flat.vmdk to *-sesparse.vmdk, after add disk in another virtual machine and scann R-STUDIO and i got lost files! But various dates.

update, i see my paper tree, my arhives and R-STUDIO show right size, but when i try restore *.7z arhives they give me error:
Closing attribute: parsed allocated size (92325376) differ from stored one (738603008)
I will try another programm for restore

sesparse f**me also up :D ive to do it manually without the script

Hi @kpma1985, are you able to share the commands that you ran to successfully recover with sesparse?

@kpma1985 Hello, can you give some tips pls?

sesparse f**me also up :D i solved it manually without the script

how do you solved it? please share the information. it would be great help

@kpma1985 Hello, can you please share your finding and solution..?
Thx a lot in advance..
Ciao..Mario.

.encoding = "UTF-8"
config.version = "8"
virtualHW.version = "14"
vmci0.present = "TRUE"
floppy0.present = "FALSE"
numvcpus = "12"
memSize = "32768"
bios.bootRetry.delay = "10"
firmware = "efi"
powerType.suspend = "soft"
tools.upgrade.policy = "manual"
sched.cpu.units = "mhz"
sched.cpu.affinity = "all"
vm.createDate = "1593152686125618"
scsi0.virtualDev = "lsisas1068"
scsi0.present = "TRUE"
sata0.present = "TRUE"
usb_xhci.present = "TRUE"
scsi0:0.deviceType = "scsi-hardDisk"
scsi0:0.fileName = "AstRun_srv-000001.vmdk"
sched.scsi0:0.shares = "normal"
sched.scsi0:0.throughputCap = "off"
scsi0:0.present = "TRUE"
scsi0:1.deviceType = "scsi-hardDisk"
scsi0:1.fileName = "AstRun_srv_1-000001.vmdk"
sched.scsi0:1.shares = "normal"
sched.scsi0:1.throughputCap = "off"
scsi0:1.present = "TRUE"
ethernet0.virtualDev = "e1000e"
ethernet0.networkName = "LAN3_Grp_AstRus"
ethernet0.addressType = "generated"
ethernet0.present = "TRUE"
displayName = "AstRun_srv"
guestOS = "windows9-64"
uefi.secureBoot.enabled = "TRUE"
toolScripts.afterPowerOn = "TRUE"
toolScripts.afterResume = "TRUE"
toolScripts.beforeSuspend = "TRUE"
toolScripts.beforePowerOff = "TRUE"
tools.syncTime = "FALSE"
uuid.bios = "56 4d 8e 2c 82 0a 68 1c-da 6d ba d0 fb 06 c3 97"
uuid.location = "56 4d 8e 2c 82 0a 68 1c-da 6d ba d0 fb 06 c3 97"
vc.uuid = "52 b7 2d 1d 64 9b 2d 63-2d 89 6a a7 58 68 b5 ca"
sched.cpu.min = "0"
sched.cpu.shares = "normal"
sched.mem.min = "0"
sched.mem.minSize = "0"
sched.mem.shares = "normal"
ethernet0.generatedAddress = "00:0c:29:06:c3:97"
vmci0.id = "-83442793"
cleanShutdown = "FALSE"
extendedConfigFile = "AstRun_srv.vmxf"
mks.enable3d = "TRUE"
tools.guest.desktop.autolock = "FALSE"
nvram = "AstRun_srv.nvram"
pciBridge0.present = "TRUE"
svga.present = "TRUE"
pciBridge4.present = "TRUE"
pciBridge4.virtualDev = "pcieRootPort"
pciBridge4.functions = "8"
pciBridge5.present = "TRUE"
pciBridge5.virtualDev = "pcieRootPort"
pciBridge5.functions = "8"
pciBridge6.present = "TRUE"
pciBridge6.virtualDev = "pcieRootPort"
pciBridge6.functions = "8"
pciBridge7.present = "TRUE"
pciBridge7.virtualDev = "pcieRootPort"
pciBridge7.functions = "8"
hpet0.present = "TRUE"
RemoteDisplay.maxConnections = "-1"
sched.cpu.latencySensitivity = "normal"
svga.autodetect = "FALSE"
disk.EnableUUID = "TRUE"
numa.autosize.cookie = "120001"
numa.autosize.vcpu.maxPerVirtualNode = "12"
sched.swap.derivedName = "/vmfs/volumes/5ebac8f4-ba43223e-d524-309c239ced34/AstRun_srv/AstRun_srv-625b630d.vswp"
pciBridge0.pciSlotNumber = "17"
pciBridge4.pciSlotNumber = "21"
pciBridge5.pciSlotNumber = "22"
pciBridge6.pciSlotNumber = "23"
pciBridge7.pciSlotNumber = "24"
scsi0.pciSlotNumber = "160"
ethernet0.pciSlotNumber = "192"
usb_xhci.pciSlotNumber = "224"
vmci0.pciSlotNumber = "32"
sata0.pciSlotNumber = "33"
scsi0.sasWWID = "50 05 05 6c 82 0a 68 10"
ethernet0.generatedAddressOffset = "0"
vm.genid = "1025359355794109798"
vm.genidX = "5812918138057738809"
monitor.phys_bits_used = "43"
vmotion.checkpointFBSize = "4194304"
vmotion.checkpointSVGAPrimarySize = "67108864"
softPowerOff = "FALSE"
toolsInstallManager.lastInstallError = "0"
svga.guestBackedPrimaryAware = "TRUE"
tools.remindInstall = "FALSE"
toolsInstallManager.updateCounter = "2"
migrate.hostLog = "./AstRun_srv-625b630d.hlog"
svga.vramSize = "67108864"
sata0:0.startConnected = "FALSE"
scsi0:0.redo = ""
scsi0:1.redo = ""