Eliminate pmsg_lock in TMSG

The lock was needed to serialize writing to log file.
However, due to data races the lock can cause deadlock as well:
#0  0x00007f6f86cf2f25 in spinlock_lock (l=0x7f6f86f2bf08 <pmsg_lock>) at /home/la5/git/hpctoolkit/BUILD/../src/lib/prof-lean/spinlock.h:115
#1  0x00007f6f86cf34f7 in hpcrun_write_msg_to_log (echo_stderr=false, add_thread_id=true, tag=0x7f6f86d0efad "LINUX_PERF", fmt=0x7f6f86d0efe0 "buffer header t: %d, s: %d, head: %d, tail: %d",
box=0x7ffe9514a5e0) at ../../../../src/tool/hpcrun/messages/messages-async.c:238
#2  0x00007f6f86cf30d5 in hpcrun_pmsg (tag=0x7f6f86d0efad "LINUX_PERF", fmt=0x7f6f86d0efe0 "buffer header t: %d, s: %d, head: %d, tail: %d")
at ../../../../src/tool/hpcrun/messages/messages-async.c:154
#3  0x00007f6f86ce949e in read_perf_buffer (current=0x7f6f845d1050, mmap_info=0x7ffe9514a760) at ../../../../src/tool/hpcrun/sample-sources/perf/perf_mmap.c:298
#4  0x00007f6f86ce8c94 in perf_event_handler (sig=29, siginfo=0x7ffe9514aab0, context=0x7ffe9514a980) at ../../../../src/tool/hpcrun/sample-sources/perf/linux_perf.c:1269
#5  0x00007f6f86ab23dd in monitor_signal_handler (sig=29, info=0x7ffe9514aab0, context=0x7ffe9514a980) at ../../libmonitor/src/signal.c:213
#6
#7  0x00007f6f8636681d in nanosleep () from /lib64/libc.so.6
#8  0x00007f6f86398534 in usleep () from /lib64/libc.so.6
#9  0x00007f6f86ab9a8a in monitor_thread_shootdown () at ../../libmonitor/src/pthread.c:603
#10 0x00007f6f86ab9079 in monitor_end_process_fcn (how=1) at ../../libmonitor/src/main.c:298
#11 0x00007f6f86ab930d in monitor_main (argc=, argv=0x7ffe9514b2b8, envp=0x7ffe9514b2c8) at ../../libmonitor/src/main.c:503
#12 0x00007f6f862ba401 in __libc_start_main () from /lib64/libc.so.6
#13 0x00007f6f86ab899e in __libc_start_main (main=, argc=1, argv=0x7ffe9514b2b8, init=0x400960 <__libc_csu_init>, fini=0x4009d0 <__libc_csu_fini>,
rtld_fini=0x7f6f86f48e50 <_dl_fini>, stack_end=0x7ffe9514b2a8) at ../../libmonitor/src/main.c:539
#14 0x000000000040074a in _start ()
There is no need to have lock in writing a file. If the writing is not in ordered, that's fine. We should add loop around async write of log messages instead:
spinlock_lock(&pmsg_lock);
// use write to logfile file descriptor, instead of fprintf stuff
//
write(messages_logfile_fd(), buf, strlen(buf));
msgs_out++;
spinlock_unlock(&pmsg_lock);

Still deadlock on ori while profiling AMG 2006
(gdb) bt
#0  0x000015158527c12e in spinlock_lock (l=0x1515854bbd48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
#1  0x000015158527c700 in hpcrun_write_msg_to_log (echo_stderr=false, add_thread_id=true, tag=0x1515852a289d "LINUX_PERF", fmt=0x1515852a28d0 "%d context switch %d, time: %u", box=0x7ffd1f96edf0)
    at ../../../../src/tool/hpcrun/messages/messages-async.c:238
#2  0x000015158527c2de in hpcrun_pmsg (tag=0x1515852a289d "LINUX_PERF", fmt=0x1515852a28d0 "%d context switch %d, time: %u") at ../../../../src/tool/hpcrun/messages/messages-async.c:154
#3  0x00001515852820da in read_perf_buffer (current_perf_mmap=0x15158563a000, attr=0x15158170b7c8, mmap_info=0x7ffd1f96ef40) at ../../../../src/tool/hpcrun/sample-sources/perf/perf_mmap.c:468
#4  0x0000151585280ec0 in perf_event_handler (sig=38, siginfo=0x7ffd1f96f370, context=0x7ffd1f96f240) at ../../../../src/tool/hpcrun/sample-sources/perf/linux_perf.c:1153
#5  0x000015158503443d in monitor_signal_handler (sig=38, info=0x7ffd1f96f370, context=0x7ffd1f96f240) at ../../libmonitor/src/signal.c:217
#6  <signal handler called>
#7  0x000015158415084d in nanosleep () from /lib64/libc.so.6
#8  0x0000151584182454 in usleep () from /lib64/libc.so.6
#9  0x000015158503ba3a in monitor_thread_shootdown () at ../../libmonitor/src/pthread.c:603
#10 0x000015158503b039 in monitor_end_process_fcn (how=1) at ../../libmonitor/src/main.c:298
#11 0x000015158503b2cd in monitor_main (argc=<optimized out>, argv=0x7ffd1f96fb78, envp=0x7ffd1f96fbc8) at ../../libmonitor/src/main.c:503
#12 0x00001515840a4431 in __libc_start_main () from /lib64/libc.so.6
#13 0x000015158503a95e in __libc_start_main (main=<optimized out>, argc=9, argv=0x7ffd1f96fb78, init=0x4ae7b0 <__libc_csu_init>, fini=0x4ae820 <__libc_csu_fini>, rtld_fini=0x1515854da1b0 <_dl_fini>, stack_end=0x7ffd1f96fb68)
    at ../../libmonitor/src/main.c:539
#14 0x000000000040460a in _start ()

To reproduce this problem on ori, one needs to profile AMG2006 with blocktime event AND debug output flag enabled. If we remove one of them, hpcrun works just fine.
mpirun -np 2 /home/la5/pkgs/hpctoolkit-fix//bin/hpcrun -dd LINUX_PERF -e cycles -e blocktime -e MEM_TRANS_RETIRED:LATENCY_ABOVE_THRESHOLD -e MEM_TRANS_RETIRED:PRECISE_STORE -t ./amg2006 -P 2 1 1 -r 18 18 18

At the end of the execution, the application will stall because all the value of pmsg_lock is 1 for all threads. This doesn't make any sense.
(gdb) p pmsg_lock
$1 = {thelock = {__val = 1}}
(gdb) info threads
  Id   Target Id         Frame
* 1    Thread 0x14fe39e88580 (LWP 5575) "amg2006" 0x000014fe39a43700 in hpcrun_write_msg_to_log (echo_stderr=false, add_thread_id=true, tag=0x14fe39a6989d "LINUX_PERF", fmt=0x14fe39a698d0 "%d context switch %d, time: %u",
    box=0x7ffc72a840f0) at ../../../../src/tool/hpcrun/messages/messages-async.c:238
  2    Thread 0x14fe34ed0700 (LWP 5608) "amg2006" 0x000014fe39a4312e in spinlock_lock (l=0x14fe39c82d48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
  3    Thread 0x14fe34bcf700 (LWP 5610) "amg2006" 0x000014fe39a4312e in spinlock_lock (l=0x14fe39c82d48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
  4    Thread 0x14fe348ce700 (LWP 5612) "amg2006" 0x000014fe39a4312e in spinlock_lock (l=0x14fe39c82d48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
(gdb) thread 2
[Switching to thread 2 (Thread 0x14fe34ed0700 (LWP 5608))]
#0  0x000014fe39a4312e in spinlock_lock (l=0x14fe39c82d48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
115       while (!atomic_compare_exchange_weak_explicit(&l->thelock, &old_lockval, SPINLOCK_LOCKED_VALUE,
(gdb) p *l
$2 = {thelock = {__val = 1}}
(gdb) thread 3
[Switching to thread 3 (Thread 0x14fe34bcf700 (LWP 5610))]
#0  0x000014fe39a4312e in spinlock_lock (l=0x14fe39c82d48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
115       while (!atomic_compare_exchange_weak_explicit(&l->thelock, &old_lockval, SPINLOCK_LOCKED_VALUE,
(gdb) p *l
$3 = {thelock = {__val = 1}}
(gdb) thread 4
[Switching to thread 4 (Thread 0x14fe348ce700 (LWP 5612))]
#0  0x000014fe39a4312e in spinlock_lock (l=0x14fe39c82d48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
115       while (!atomic_compare_exchange_weak_explicit(&l->thelock, &old_lockval, SPINLOCK_LOCKED_VALUE,
(gdb) p *l
$4 = {thelock = {__val = 1}}

@jmellorcrummey : I think this is not a priority, but can cause false alarm for developers and testers.

Deadlock occurs when perf_events interrupting sleep() within monitor_thread_shotdown and the handler write a log message:
115       while (!atomic_compare_exchange_weak_explicit(&l->thelock, &old_lockval, SPINLOCK_LOCKED_VALUE,
Missing separate debuginfos, use: dnf debuginfo-install glibc-2.24-10.fc25.x86_64 libgcc-6.4.1-1.fc25.x86_64 libgfortran-6.4.1-1.fc25.x86_64 libgomp-6.4.1-1.fc25.x86_64 libquadmath-6.4.1-1.fc25.x86_64 nss-mdns-0.10-17.fc24.x86_64
(gdb) bt
#0  0x00007fc43de5a12e in spinlock_lock (l=0x7fc43e099d48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
#1  0x00007fc43de5a700 in hpcrun_write_msg_to_log (echo_stderr=false, add_thread_id=true, tag=0x7fc43de8089d "LINUX_PERF", fmt=0x7fc43de808d0 "%d context switch %d, time: %u", box=0x7ffce94224f0)
    at ../../../../src/tool/hpcrun/messages/messages-async.c:238
#2  0x00007fc43de5a2de in hpcrun_pmsg (tag=0x7fc43de8089d "LINUX_PERF", fmt=0x7fc43de808d0 "%d context switch %d, time: %u") at ../../../../src/tool/hpcrun/messages/messages-async.c:154
#3  0x00007fc43de600da in read_perf_buffer (current_perf_mmap=0x7fc43e218000, attr=0x7fc43a2e9890, mmap_info=0x7ffce9422640) at ../../../../src/tool/hpcrun/sample-sources/perf/perf_mmap.c:468
#4  0x00007fc43de5eec0 in perf_event_handler (sig=38, siginfo=0x7ffce9422a70, context=0x7ffce9422940) at ../../../../src/tool/hpcrun/sample-sources/perf/linux_perf.c:1153
#5  0x00007fc43dc1243d in monitor_signal_handler (sig=38, info=0x7ffce9422a70, context=0x7ffce9422940) at ../../libmonitor/src/signal.c:217
#6  <signal handler called>
#7  0x00007fc43cd2e84d in nanosleep () from /lib64/libc.so.6
#8  0x00007fc43cd60454 in usleep () from /lib64/libc.so.6
#9  0x00007fc43dc19a3a in monitor_thread_shootdown () at ../../libmonitor/src/pthread.c:603
#10 0x00007fc43dc19039 in monitor_end_process_fcn (how=1) at ../../libmonitor/src/main.c:298
#11 0x00007fc43dc192cd in monitor_main (argc=<optimized out>, argv=0x7ffce9423278, envp=0x7ffce94232c8) at ../../libmonitor/src/main.c:503
#12 0x00007fc43cc82431 in __libc_start_main () from /lib64/libc.so.6
#13 0x00007fc43dc1895e in __libc_start_main (main=<optimized out>, argc=9, argv=0x7ffce9423278, init=0x4ae7b0 <__libc_csu_init>, fini=0x4ae820 <__libc_csu_fini>, rtld_fini=0x7fc43e0b81b0 <_dl_fini>,
    stack_end=0x7ffce9423268) at ../../libmonitor/src/main.c:539
#14 0x000000000040460a in _start ()
(gdb) detach
Detaching from program: /home/la5/git/hpctoolkit-tests/applications/AMG2006/test/amg2006, process 19064
(gdb) attach 19065
Attaching to program: /home/la5/git/hpctoolkit-tests/applications/AMG2006/test/amg2006, process 19065
[New LWP 19097]
[New LWP 19100]
[New LWP 19101]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
0x00007fba65a8a12e in spinlock_lock (l=0x7fba65cc9d48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
115       while (!atomic_compare_exchange_weak_explicit(&l->thelock, &old_lockval, SPINLOCK_LOCKED_VALUE,
(gdb) bt
#0  0x00007fba65a8a12e in spinlock_lock (l=0x7fba65cc9d48 <pmsg_lock>) at /home/la5/git/photon/hpctoolkit/BUILD-fix/../src/lib/prof-lean/spinlock.h:115
#1  0x00007fba65a8a700 in hpcrun_write_msg_to_log (echo_stderr=false, add_thread_id=true, tag=0x7fba65ab089d "LINUX_PERF", fmt=0x7fba65ab08d0 "%d context switch %d, time: %u", box=0x7fff76ea93f0)
    at ../../../../src/tool/hpcrun/messages/messages-async.c:238
#2  0x00007fba65a8a2de in hpcrun_pmsg (tag=0x7fba65ab089d "LINUX_PERF", fmt=0x7fba65ab08d0 "%d context switch %d, time: %u") at ../../../../src/tool/hpcrun/messages/messages-async.c:154
#3  0x00007fba65a900da in read_perf_buffer (current_perf_mmap=0x7fba65e48000, attr=0x7fba61f19878, mmap_info=0x7fff76ea9540) at ../../../../src/tool/hpcrun/sample-sources/perf/perf_mmap.c:468
#4  0x00007fba65a8eec0 in perf_event_handler (sig=38, siginfo=0x7fff76ea9970, context=0x7fff76ea9840) at ../../../../src/tool/hpcrun/sample-sources/perf/linux_perf.c:1153
#5  0x00007fba6584243d in monitor_signal_handler (sig=38, info=0x7fff76ea9970, context=0x7fff76ea9840) at ../../libmonitor/src/signal.c:217
#6  <signal handler called>
#7  0x00007fba6495e84d in nanosleep () from /lib64/libc.so.6
#8  0x00007fba64990454 in usleep () from /lib64/libc.so.6
#9  0x00007fba65849a3a in monitor_thread_shootdown () at ../../libmonitor/src/pthread.c:603
#10 0x00007fba65849039 in monitor_end_process_fcn (how=1) at ../../libmonitor/src/main.c:298
#11 0x00007fba658492cd in monitor_main (argc=<optimized out>, argv=0x7fff76eaa158, envp=0x7fff76eaa1a8) at ../../libmonitor/src/main.c:503
#12 0x00007fba648b2431 in __libc_start_main () from /lib64/libc.so.6
#13 0x00007fba6584895e in __libc_start_main (main=<optimized out>, argc=9, argv=0x7fff76eaa158, init=0x4ae7b0 <__libc_csu_init>, fini=0x4ae820 <__libc_csu_fini>, rtld_fini=0x7fba65ce81b0 <_dl_fini>,
    stack_end=0x7fff76eaa148) at ../../libmonitor/src/main.c:539
#14 0x000000000040460a in _start ()

This repository has moved to GitLab, please continue this issue there.


Enhancement: All prefix tokenizer.

Powerup auto-completion or suggest with all-prefix tokenization.
Example:

Document containing: "Simple Line"
Gets tokenized into: [S, Si, Sim, Simp, Simpl, Simple, L, Li, Lin, Line]
Matches lookup: Simple one

Usage:
var idx = lunr(function(){
    this.ref('refid');
    this.field('field1');
    // ...

    // 1. Remove stemmer from pipeline.
    // 2. Activate tokenzier.
    this.pipeline.remove(lunr.stemmer);
    this.tokenizer = lunr.allprefixtokenizer;

    // ...add doc...
});

The way that tokens are stored in lunr.TokenSet should allow good performance for prefix searches already. Do you have an example where this approach works better than just using a trailing wildcard suffix?
You could certainly implement this kind of approach as a pipeline function without having to replace the tokeniser.
Because of that I don't think this change should be part of lunr, though I encourage you to package it as a separate module/plugin.


jruby-<IP_ADDRESS> failing to install

With the following error, on archlinux, with rbenv and ruby-build updated to today's master:
:✘1 ~/.rbenv/plugins/ruby-build [master|✔]$ rbenv install jruby-<IP_ADDRESS>
Downloading jruby-bin-<IP_ADDRESS>.tar.gz...
-> https://dqw8nmjcqpjn7.cloudfront.net/ddb23c95f4b3cc3fc1cc57b81cb4ceee776496ede402b9a6eb0622cf15e1a597
Installing jruby-<IP_ADDRESS>...

BUILD FAILED (Arch Linux using ruby-build 20170523-25-g476d09b)

Inspect or clean up the working tree at /tmp/ruby-build.20170617090627.22925
Results logged to /tmp/ruby-build.20170617090627.22925.log

Last 10 log lines:
Download Results:
gid   |stat|avg speed  |path/URI
======+====+===========+=======================================================
4016f3|OK  |   215KiB/s|/tmp/ruby-build.20170617090627.22925/jruby-<IP_ADDRESS>.tar.gz

Status Legend:
(OK):download completed.
/tmp/ruby-build.20170617090627.22925/jruby-<IP_ADDRESS> /tmp/ruby-build.20170617090627.22925 ~/.rbenv/plugins/ruby-build
ERROR:  While executing gem ... (IPAddr::InvalidAddressError)
    invalid address

Would it have something to do with #1088?

Same error for jruby-<IP_ADDRESS>.

I couldn't reproduce it.
[hsbt@arch ~]$ uname -a
Linux arch 4.10.14-1-ec2 #1 SMP Thu May 4 09:15:16 UTC 2017 x86_64 GNU/Linux

[hsbt@arch ~]$ rbenv install jruby-<IP_ADDRESS>
Downloading jruby-bin-<IP_ADDRESS>.tar.gz...
-> https://dqw8nmjcqpjn7.cloudfront.net/ddb23c95f4b3cc3fc1cc57b81cb4ceee776496ede402b9a6eb0622cf15e1a597
Installing jruby-<IP_ADDRESS>...
/home/hsbt/.rbenv/plugins/ruby-build/bin/ruby-build: line 717: warning: command substitution: ignored null byte in input
/home/hsbt/.rbenv/plugins/ruby-build/bin/ruby-build: line 717: warning: command substitution: ignored null byte in input
Installed jruby-<IP_ADDRESS> to /home/hsbt/.rbenv/versions/jruby-<IP_ADDRESS>

@hsbt You are right, it is now working for me as well. I guess it was only a server which was down.
I had not had the time to look which IP address it was.
Thanks!


Can I convert an HTML into an epub and automatically generate a table of contents with the H2 headings?

Feature description
Can I convert an HTML into an epub and automatically generate a table of contents with the H2 headings?

We could probably add a --toc-level option to control how deep the table of contents of the generated document should be.

Released --toc-level as part of<EMAIL_ADDRESS>


portfetch: require MacPorts svn on >= Catalina

there is a /usr/bin/svn on Catalina, but it is
only a small forwarding binary and is non-functional
 % /usr/bin/svn
svn: error: Failed to locate 'svn'.
svn: error: The subversion command line tools are no longer provided by Xcode.

closes: https://trac.macports.org/ticket/61981
(Note: personally I would probably rather just require macports' subversion on all systems for simplicity)

Whether svn works on Catalina isn't based on the OS version; it's based on the Xcode version.

Whether svn works on Catalina isn't based on the OS version; it's based on the Xcode version.

We can’t use xcodeversion for any tests as it doesn’t work if have only the CLT’s installed.

We can’t use xcodeversion for any tests as it doesn’t work if have only the CLT’s installed.

so we don't like this? I can close it -- almost nothing uses svn any more anyway.

so we don't like this? I can close it -- almost nothing uses svn any more anyway.

I still think @ryandesign or @jmroot will have to do something like this to fix base, otherwise svn fetching will remain broken, and what's the good in that?
But they will come up with their own fix for it, in their own good time.

I still think @ryandesign or @jmroot will have to do something like this to fix base, otherwise svn fetching will remain broken, and what's the good in that?
But they will come up with their own fix for it, in their own good time.


feat: Update repos readmes

Codecov Report

Merging #1443 into master will decrease coverage by 6.73%.
The diff coverage is n/a.

@@            Coverage Diff             @@
##           master    #1443      +/-   ##
==========================================
- Coverage   82.37%   75.64%   -6.74%
==========================================
  Files          48       38      -10
  Lines        1901     1084     -817
  Branches      441      204     -237
==========================================
- Hits         1566      820     -746
+ Misses        324      253      -71
  Partials       11       11

Impacted Files
Coverage Δ

packages/raven-node/vendor/node-lsmod.js

...ges/raven-node/lib/instrumentation/instrumentor.js

packages/raven-node/lib/transports.js

packages/raven-node/lib/instrumentation/http.js

packages/raven-node/lib/client.js

packages/raven-node/lib/utils.js

packages/raven-node/lib/parsers.js

packages/raven-node/vendor/json-stringify-safe.js

packages/raven-node/lib/instrumentation/console.js

packages/raven-node/index.js

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update b38bed0...26c5c36. Read the comment docs.


Update

fix: improve overview card style
fix: improve styling of changelog view
refactor: replace educorvi logo with svg

:tada: This PR is included in version timeclicker_client-v1.6.2 :tada:
The release is available on GitHub release
Your semantic-release bot :package::rocket:


Update description of overview table

Clarify how to read (the very helpful!) the overview table.

Thanks for the suggestion but I'm not sure it's a good idea, cause if we make the column names any longer it probably becomes necessary to scroll to read the full table?

Can you try it locally how it looks? Push preview is not available for PRs from forks

Codecov Report
All modified and coverable lines are covered by tests :white_check_mark:

Project coverage is 92.53%. Comparing base (e14bdfc) to head (f9ff0e7).

Additional details and impacted files
@@             Coverage Diff             @@
##             main     #240       +/-   ##
===========================================
+ Coverage   69.30%   92.53%   +23.22%
===========================================
  Files          74       16       -58
  Lines        3747     1782     -1965
===========================================
- Hits         2597     1649      -948
+ Misses       1150      133     -1017

:umbrella: View full report in Codecov by Sentry.
:loudspeaker: Have feedback on the report? Share it here.


Url templating

Managing URLs was getting hacky.
This PR moves the responsibility to each service endpoint (which is logically where it should be defined) to determine the correct template to build the eventual path of the request.

Coverage decreased (-0.7%) to 96.324% when pulling c5ae49359118e8c1de82b01b52588383b4ad755e on url-templating into ab922d3c9bfd1f8cae82675c08502c8680e34a65 on master.

Coverage decreased (-0.7%) to 96.324% when pulling e67924ae3d8d29cfc3ff5c068cd6b0310478adb0 on url-templating into ab922d3c9bfd1f8cae82675c08502c8680e34a65 on master.

Coverage remained the same at 97.059% when pulling 26ea668a6f73501e0bf711001462055c1cde7d60 on url-templating into ab922d3c9bfd1f8cae82675c08502c8680e34a65 on master.


Implement external file references to the Workflow parser

What would you like to be added:
In the specification, we have the possibility to add external files references in the workflow definition, for instance:
{
  "id": "VetAppointmentWorkflow",
  "name": "Vet Appointment Workflow",
  "description": "Vet service call via events",
  "version": "1.0",
  "events": "features/vetappointmenteventrefs.json",
  "retries": "features/vetappointmentretries.json",
  "states": [
    {
      "name": "MakeVetAppointmentState",
      "type": "operation",
      "start": {
        "kind": "default"
      },
      "actions": [
        {
          "name": "MakeAppointmentAction",
          "eventRef": {
            "triggerEventRef": "MakeVetAppointment",
            "data": "{{ $.patientInfo }}",
            "resultEventRef":  "VetAppointmentInfo"
          },
          "actionDataFilter": {
            "dataResultsPath": "{{ $.appointmentInfo }}"
          },
          "timeout": "PT15M"
        }
      ],
      "onErrors": [
        {
          "error": "TimeoutError",
          "code": "500",
          "retryRef": "TimeoutRetryStrategy",
          "end": {
            "kind": "default"
          }
        }
      ],
      "end": {
        "kind": "default"
      }
    }
  ]
}

In this example, retries and events are defined externally to the main file. The SDK supposed to incorporate those while parsing the main definition.
Why is this needed:
To conform with the specification.

Hello,
So I suppose there should be implemented the following:

Recognize directory/file strings using some sort of regexp
Parse the encapsulated file (using recursion in case of additional nested files)
Replace them in the parsed workflow

Am I right?

Hi!

Recognize directory/file strings using some kind of regexp

You can unmarshal the workflow into RawJson first and use a map to access functions, events, or retries, which is the types that can have an external reference

Parse the encapsulated file (using recursion in case of additional nested files)

I don't think we will have nested files for functions, events, and retries. Only for Subflows. But for this state type, we should handle it differently, maybe in another issue. See for example https://github.com/serverlessworkflow/specification/blob/0.5.x/schema/functions.json

Replace them in the parsed workflow

:+1:
One remark only is that the nested references should be read from the relative path unless it starts with file://. @tsurdilo do we support URI on functions, events, and retries fields? All I see is that they are references for the object itself.

@ricardozanini if they are string we currently define as "URI pointing to a resource containing json/yaml function/events/retries definitions".
We just use classpath atm in the java sdk for lookup but should prob update to use uri :) I think it's fine to define it as you guys wish for now and then enhance later if needed

@tsurdilo thanks for the clarification. For this use case specifically, I believe we might need some tweaks in the unmarshalling process. The generated code attributes an array of events to the events field in the workflow. I believe the "RawJson" in our case should do.
@iwita We might need to have a new structure to parse the workflow first into functions, events and retries into RawJson. Then verify if it's a URI to parse these objects first.

Hi @ricardozanini and @tsurdilo,
Thank you for your suggestions and help.
I checked the code and I am back with some suggestions:

In order to be able to identify any errors from the full workflow unmarshalling, I would propose to change the WorkflowCommon altering the json tags of the needed fields.
	 Events           []Eventdef `json:"-"`
This way, those fields will be ignored during the initial workflow parsing.

Furthermore, by parsing the json file into json.RawMessage and then unmarshalling the desired fields (using a map) into a string we will be able to identify if we have a string or other interface. In the first case, we will check the cases of URI, relative or absolute path, and then will unmarshal the required fields (functions, events, retries) from the respective files. Else, we will unmarshal those fields explicitly from the map[string]RawMessage.

P.S.1  I will submit a new PR today or tomorrow if you agree with the above.
P.S.2  There is no current definition of Retries type.
P.S.3  Should I implement unit testing or there would be some other issue opened?

Hi @iwita ! Many thanks for your suggestions!

In order to be able to identify any errors from the full workflow unmarshalling, I would propose to change the WorkflowCommon altering the json tags of the needed fields.

I believe it's better to implement a UnmarshalJSON function to EventDef, FunctionDef, and RetryDef, this way we can fetch for the URI and unmarshal it recursively. The problem is that the tool we are using to generate code is already generating the functions for these types. We will have to configure the tool to not generate it for these types somehow.

Furthermore, by parsing the json file into json.RawMessage and then unmarshalling the desired fields (using a map) into a string we will be able to identify if we have a string or other interface. In the first case, we will check the cases of URI, relative or absolute path, and then will unmarshal the required fields (functions, events, retries) from the respective files. Else, we will unmarshal those fields explicitly from the map[string]RawMessage.

This can be done on the respective UnmarshalJSON functions

I will submit a new PR today or tomorrow if you agree with the above.

That's great! :+1:

There is no current definition of Retries type.

Yeah, I just noticed that indeed. This is weird, I need to take a careful look into the tooling that generated the code. I just quickly tried generating it again, and the Retries definition is not there.

Should I implement unit testing or there would be some other issue opened?

Oh please, yes, implement them :)

Hello,

I believe it's better to implement a UnmarshalJSON function to EventDef, FunctionDef, and RetryDef, this way we can fetch for the URI and unmarshal it recursively. The problem is that the tool we are using to generate code is already generating the functions for these types. We will have to configure the tool to not generate it for these types somehow.

Since UnmarshalJSON implements the json.Unmarshaler for each respective field (Eventdef, Functiondef ), we needed a wrapper type in order to handle slices of (Eventdef, Functiondef ). Otherwise, by modifying the Eventdef handler, json.Unmarshal() (for the workflow type), didn't even reach Eventdef type, because it returned an error when detecting a string instead of a []Eventdef.
While I tried to figure out how to do it this way, I ended up with an ugly solution.
func FromJSONSourceHelper(source []byte) (workflow *model.Workflow, err error) {
	var m map[string]json.RawMessage
        // unmarshal the source into a map in order to get discrete fields
	if err := json.Unmarshal(source, &m); err != nil {
		return nil, err
	}
	nestedCandidates := []string{"events", "retries", "functions"}
	var s string
	for _, v := range nestedCandidates {
		if err := json.Unmarshal(m[v], &s); err != nil {
			if _, ok := m[v]; !ok {
				continue
			}
			// It's not a string
			return FromJSONSource(source)
		}
		// It's a string
		var fileBytes []byte
		if fileBytes, err = getBytesFromFile(s); err != nil {
			return nil, err
		}
		if err = json.Unmarshal(fileBytes, &m); err != nil {
			return nil, err
		}
	}
	bytes, err := json.Marshal(m)
	return FromJSONSourceHelper(bytes)
}

Hey @iwita !
yes, since the structure is an array of attributes and not a string, the unmarshal will complain. That's why I said that we might need to change the UnmarshalJSON function and unmarshal to RawJson first, check if it's a string first. More or less what you did in the example.
This is the unmarshal function I'm referring to:
https://github.com/serverlessworkflow/sdk-go/blob/master/model/zz_generated.types_events.go#L104-L119
The biggest problem is that this function is generated, that's why we will need to configure the tooling that generates it to not do it.

Hi all,
I searched for the model/zz_* files generation process. The https://github.com/atombender/go-jsonschema repository is used in order to create the schema for each type. It does not seem to produce any other files.
When running the hack/generate-spec-types.sh script there are several warnings occurring.

The Events, Functions types are created as map[string]interface{}, since in the json definitions (https://github.com/serverlessworkflow/specification/blob/master/schema/workflow.json) those types are declared either as strings either as an array of objects.
When I tried to add the retries.json file, there are errors occurring related to a parameter that is float instead of integer. More specifically, while there is a float value in the field Type.Parameters.Minimum, which can be found here: https://github.com/atombender/go-jsonschema/blob/master/pkg/schemas/model.go#L78, the script errors for not being an integer.

Regarding this issue, I changed the types.go file, in order to isolate []Eventdef and []Function (by removing it from WorkflowCommon type) similar to []State. This way, we are able to parse those types independently from here: https://github.com/serverlessworkflow/sdk-go/blob/master/model/unmarshal_json.go#L23-L53.

Great additions, @iwita
Regarding this:

When I tried to add the retries.json file, there are errors occurring related to a parameter that is float instead of integer. More specifically, while there is a float value in the field Type.Parameters.Minimum, which can be found here: https://github.com/atombender/go-jsonschema/blob/master/pkg/schemas/model.go#L78, the script errors for not being an integer.

Reading this part of the code, I believe they follow the spec and we might need to change our field to float instead. @tsurdilo wdty?
https://github.com/serverlessworkflow/specification/blob/master/schema/retries.json#L37-L38

Thank you @ricardozanini

Reading this part of the code, I believe they follow the spec and we might need to change our field to float instead. @tsurdilo wdty?

The script complains about the jitter minimum field.
https://github.com/serverlessworkflow/specification/blob/master/schema/retries.json#L43-L44

I think in jsonschema "number" represents both integers and floats. Maybe best to just use float on the sdk side as thats superset?

@tsurdilo @iwita I believe I covered this problem in this PR: https://github.com/serverlessworkflow/specification/pull/193

Thank you @ricardozanini

Reading this part of the code, I believe they follow the spec and we might need to change our field to float instead. @tsurdilo wdty?

The script complains about the jitter minimum field.
https://github.com/serverlessworkflow/specification/blob/master/schema/retries.json#L43-L44

jitter or maxAttempts? Since the last one was in integer format and float was expected.

@tsurdilo @iwita I believe I covered this problem in this PR: serverlessworkflow/specification#193

Now there is an additional error occurring in hack/generate-spec-types.sh. This time for the workflow.json because it does not accept once again the minimum field.
--> Generating specification types
gojsonschema: Loading /tmp/serverlessworkflow/schema/common.json
gojsonschema: Loading /tmp/serverlessworkflow/schema/events.json
 gojsonschema: Warning: Multiple types map to the name "Metadata"; declaring duplicate as "Metadata_1" instead
 gojsonschema: Warning: Object type with no properties has required fields; skipping validation code for them since we don't know their types
 gojsonschema: Loading /tmp/serverlessworkflow/schema/functions.json
 gojsonschema: Warning: Object type with no properties has required fields; skipping validation code for them since we don't know their types
 gojsonschema: Loading /tmp/serverlessworkflow/schema/workflow.json
gojsonschema: Failed: json: cannot unmarshal number 0.0 into Go struct field Type.definitions.properties.minimum of type int
Do you think there is any bug in the generator?

I'll revert them back to 0 instead. 0.0 and 0 seems the same to me. Might be a bug in their side, but I'd rather keep 0. @tsurdilo would you be angry if we revert from 0.0 to 0 in the spec?

@ricardozanini that's fine. let's go for it.

Then I hope it does not complain for the maximum value as well (which is 1.0) in the jitter case.

@iwita can you test with this PR instead? https://github.com/serverlessworkflow/specification/pull/195


GRADLE_ARGS partially ignored in recursive mode

Having a multi-module gradle project with GRALDE_ARGS set, these are not applied to every gradle call executed:
# export GRADLE_ARGS="-PartifactoryUser=*ARTIFACTORY_USER* -PartifactoryPassword=*ARTIFACTORY_PASSWORD*"
# cdxgen ...

Executing /home/heubeck/w/sbom-test/branded-services-support/gradlew properties -q --console plain --build-cache in .
Executing /home/heubeck/w/sbom-test/branded-services-support/gradlew :ui-app:properties -q --console plain --build-cache in .
Executing /home/heubeck/w/sbom-test/branded-services-support/gradlew :ui-server:properties -q --console plain --build-cache in .
Executing /home/heubeck/w/sbom-test/branded-services-support/gradlew :ui-app:dependencies -q --console plain --build-cache -PartifactoryUser=*ARTIFACTORY_USER* -PartifactoryPassword=*ARTIFACTORY_PASSWORD* in .
Executing /home/heubeck/w/sbom-test/branded-services-support/gradlew :ui-server:dependencies -q --console plain --build-cache -PartifactoryUser=*ARTIFACTORY_USER* -PartifactoryPassword=*ARTIFACTORY_PASSWORD* in .

@heubeck, does the gradle properties command support GRADLE_ARGS?
@malice00, any thoughts on bringing back this functionality?
https://github.com/CycloneDX/cdxgen/pull/385/files

does the gradle properties command support GRADLE_ARGS?

I've no idea, just recognized that I cannot add options like -p to address modules within a project.

any thoughts on bringing back this functionality? Below is the PR that removed this variable.

🤯

does the gradle properties command support GRADLE_ARGS?

I've no idea, just recognized that I cannot add options like -p to address modules within a project.

any thoughts on bringing back this functionality? Below is the PR that removed this variable.

exploding_head
Put it the other way, is there an issue with the current behaviour like sbom having less components or dependency etc?

Put it the other way, is there an issue with the current behaviour like sbom having less components or dependency etc?

there's a multi-module gradle project, for which I have to start cdxgen in the project root folder, but have to address single modules using -p some-subodule which isn't propagated to the gradle properties command, but to gradle dependencies.
not quite sure, how much of a problem it is by now, just recognized the inconsistency.

ok, did some comparison.
the dependency components are equal: taken from the submodul I gave with GRADLE_ARGS="-p sumodule".
but the metadata.component information refers to the root project instead of the subproject (obvious because properties is called for that).
Not quite sure, if there are cases with project properites like -Psomething=need could be relevant for gralde properties - but having no option at all to configure it feels wrong.

@heubeck any chance you can contribute a test project to replicate this?

@heubeck any chance you can contribute a test project to replicate this?

will do.

Sorry for the late reply.
I had issues with the GRADLE_ARGS being added to both Gradle-calls in the code (properties and depdencies) where only the latter understood the added arguments. Even worse, the call to 'properties' failed because of unknown arguments!
I can understand you wanting to have GRADLE_ARGS back, but maybe we should have 2 versions of it, eg:

GRADLE_ARGS_PROPERTIES
GRADLE_ARGS_DEPENDENCIES
That way, it you can even set different arguments for either call!

Or mavbe even go the extra mile and make 3 EnvVars, where the documentation than clearly explains that 'GRADLE_ARGS' is used for both calls, but the above are for the specific calls?

@malice00 I am happy with the proposed 3 env variable approach. Would it be possible to send a PR?

It seems somebody already did! :-) Thanks @heubeck!


Optimizing carbon emission instead of power consumption

Hi! I'm currently adopting this testbed to optimize the carbon emission of the data center. I've been trying to modify 2ZoneDataCenterHVAC_wEconomizer_Temp_Fan.idf so that the observation sent by @ExtCtrlObs includes the carbon emission sensor data.
I added three sensors as follows:
EnergyManagementSystem:Sensor,
    CARBON_EQUIVALENT,    !- Name
    Site,          !- Output:Variable or Output:Meter Index Key Name
    Environmental Impact Total CO2 Emissions Carbon Equivalent Mass;  !- Output:Variable or Output:Meter Name

EnergyManagementSystem:Sensor,
    CO2_EMISSION,          !- Name
    Site,          !- Output:Variable or Output:Meter Index Key Name
    Environmental Impact Electricity CO2 Emissions Mass;  !- Output:Variable or Output:Meter Name

EnergyManagementSystem:Sensor,
    CO_EMISSION,        !- Name
    Site,          !- Output:Variable or Output:Meter Index Key Name
    Environmental Impact Electricity CO Emissions Mass;  !- Output:Variable or Output:Meter Name

with the corresponding output variables:
Output:Variable,
  *,                       !- Key Value
  Environmental Impact Total CO2 Emissions Carbon Equivalent Mass,  !- Variable Name
  timestep;

Output:Variable,
  *,                       !- Key Value
  Environmental Impact Electricity CO2 Emissions Mass,  !- Variable Name
  timestep;                 !- Reporting Frequency

Output:Variable,
  *,                       !- Key Value
  Environmental Impact Electricity CO Emissions Mass,  !- Variable Name
  timestep;

The EMS program for sending the observation is basically the same as the original code, with the power consumption EMS variables substituted with the carbon emission ones:
EnergyManagementSystem:ProgramCallingManager,
    ExtCtrl-Based Observation Collector,  !- Name
    EndOfZoneTimestepAfterZoneReporting,  !- EnergyPlus Model Calling Point
    ExtCtrlBasedObservationCollector;  !- Program Name 1

  EnergyManagementSystem:Program,
    ExtCtrlBasedObservationCollector,  !- Name
    IF WarmupFlag == 0.0,    !- Program Line 1
    SET tmp_val1 = @ExtCtrlObs 1 OutdoorTemp,  !- Program Line 2
    SET tmp_val1 = @ExtCtrlObs 2 WestZoneTemp,  !- <none>
    SET tmp_val1 = @ExtCtrlObs 3 EastZoneTemp,  !- <none>
    SET tmp_val1 = @ExtCtrlObs 4 PUE_Value,  !- <none>
    SET tmp_val1 = @ExtCtrlObs 5 CARBON_EQUIVALENT,  !- <none>
    SET tmp_val1 = @ExtCtrlObs 6 CO_EMISSION,  !- <none>
    SET tmp_val1 = @ExtCtrlObs 7 CO2_EMISSION,  !- <none>
    ELSE,                    !- <none>
    ENDIF;                   !- <none>

(The variable PUE_Value here is just a placeholder as this value is neither involved in the reward calculation process nor included in the observation space.)
I modified the weight in the reward function. The training went on well and the observed carbon emission values decreased significantly during the inference process. Yet I just found that these carbon emission values sent as observation are different from the generated output in eplusout.csv.
For example, here's a snippet of the logged values of CARBON_EQUIVALENT from the first few observations:
Carbon Equivalent
--
0.0
0.133487
0.337323
0.307257
0.2369
0.282264
0.391187
0.225665
0.235469
0.157378
0.238455

And here's a snippet of the same timesteps in eplusout.csv:
Site:Environmental Impact Total CO2 Emissions Carbon Equivalent Mass [kg](TimeStep)
--
2.00229877741957
5.05984938339263
3.37983056774111
3.31660334495416
3.10490202242562
2.73830923089479
3.38498004573822
3.53203668198742
2.36067179328403
3.57682565240907
5.63639703991733
3.47494246773142

As a result, though the observed values got significantly reduced during the model's inference, it just doesn't seem to reduce carbon emission judging from the EnergyPlus output. The same inconsistency doesn't exists in the original case where electricity demand values are sent as observations.
I saw some issues discussing the inconsistency between EMS and output variables (https://github.com/NREL/EnergyPlus/issues/7563, https://github.com/NREL/EnergyPlus/issues/8915), and the problem seems to be the calling program manager. Yet I've tried a bunch of different calling points including EndOfSystemTimestepBeforeHVACReporting and AfterPredictorBeforeHVACManagers but they didn't fix the inconsistency.
Is there any suggestion on how solve this issue? Anything about applying this testbed to optimize carbon emission would be very helpful. Thanks!

Hi @WynnCJF. A discrepancy is possible if you don't use the right calling point, but it usually expresses as a lag, not a completely different value. There's also a possible problem when using EnergyManagementSystem:OutputVariable with an incorrect update frequency or aggregation (averaged -> summed).
What you could try:

Add an Output:Meter for comparison. In some cases meters can help, sometimes they're completely off.
Add a EnergyManagementSystem:OutputVariable to check if it tells the same than Output:Variable and Output:Meter

I also recommend running your tests outside of this testbed since it's seems to be an EnergyPlus-related issue
Here is a test example (not tested):
EnergyManagementSystem:Sensor,
  CARBON_EQUIVALENT,                                                !- Name
  Site,                                                             !- Output:Variable or Output:Meter Index Key Name
  Environmental Impact Total CO2 Emissions Carbon Equivalent Mass;  !- Output:Variable or Output:Meter Name

EnergyManagementSystem:ProgramCallingManager,
  Test Prgm,                              !- Name
  EndOfZoneTimestepBeforeZoneReporting,   !- EnergyPlus Model Calling Point
  Test_Prgm;                              !- Program Name 1

EnergyManagementSystem:Program,
  Test_Prgm,                              !- Name
  SET carbon = CARBON_EQUIVALENT;         !- Program Line 1

EnergyManagementSystem:OutputVariable,
  EMS Carbon,                             !- Name
  carbon,                                 !- EMS Variable Name
  Summed,                                 !- Type of Data in Variable
  SystemTimestep,                         !- Update Frequency
  Test_Prgm,                              !- EMS Program or Subroutine Name
  J;                                      !- Units

!- Case 1: EMS output variable
Output:Variable,
  EMS,                                    !- Key Value
  EMS Carbon,                             !- Variable Name
  timestep;                               !- Reporting Frequency

!- Case 2: Output Variable
Output:Variable,
  *,                                                                !- Key Value
  Environmental Impact Total CO2 Emissions Carbon Equivalent Mass,  !- Variable Name
  timestep;                                                         !- Reporting Frequency

!- Case 3: Output Meter
Output:Meter,
  ElectricEmissions:CO2 ,  !- Key Name
  timestep;                !- Reporting Frequency

You may want to tweak the EnergyPlus Model Calling Point too, to check the impact.
See also EnergyPlus Input/Output Reference for other ways to express CO2 emissions as output variables and meters.


Fix SQL execution issue with endDelimiter '/' and splitStatements

Impact

[x] Bug fix (non-breaking change which fixes expected existing functionality)
[ ] Enhancement/New feature (adds functionality without impacting existing logic)
[ ] Breaking change (fix or feature that would cause existing functionality to change)

Description
Fixes #5359

Things to be aware of

Things to worry about

Additional Context

@MalloD12 I added 2 tests, and test ":moreSlashEndDelimiterTest" broke Oracle build. Would it be a valid test case?

@MalloD12 I added 2 tests, and test ":moreSlashEndDelimiterTest" broke Oracle build. Would it be a valid test case?

@filipelautert I think that is not a valid case, because after a / we are expecting the beginning of a new SQL statement.

@MalloD12 I added 2 tests, and test ":moreSlashEndDelimiterTest" broke Oracle build. Would it be a valid test case?

@filipelautert I think that is not a valid case, because after a / we are expecting the beginning of a new SQL statement.

I'm wondering now, whether we should tag this to be documented. @filipelautert do you know if we have things like this in confluence? Before we move ahead with this let me double-check it with Mario.

Is this fix going to get merge?

[DOCS] https://datical.atlassian.net/browse/PD-4819


Fix some potential ChainState issues

Motivation
The admin ID can't change, and we are locking the ChainState each time we access it.
We are locking and unlocking the ChainState multiple times in some cases to obtain the components of a chain's tip state (block hash, height and timestamp). If the state changed in between those instances, the components won't refer to the same state.
In some cases we return an error if the local node's chain is ahead of the ChainState. But in multi-owner chains, this can happen legitimately.
Proposal
Move admin_id out of ChainState. Fetch the tip state in one go. Only do the check for single-owner chains.
Also, don't unnecessarily collect the known keys' owners, and use contains_key instead.
Test Plan
CI should catch regressions.
Release Plan

Nothing to do / These changes follow the usual release cycle.

Links

reviewer checklist

Closing in favor of https://github.com/linera-io/linera-protocol/pull/2675.
Thanks for the comments, and thanks to @andresilva91 for fixing the underlying issue that caused the test to fail! #2675 should pass CI now and be less controversial.


[SPARK-49223][ML] Simplify the StringIndexer.countByValue with builtin functions

What changes were proposed in this pull request?
Simplify the StringIndexer.countByValue with builtin functions
Why are the changes needed?
the StringIndexerAggregator is not necessary here
Does this PR introduce any user-facing change?
No
How was this patch tested?
CI
Was this patch authored or co-authored using generative AI tooling?
No

I test four approaches with
import org.apache.spark.ml.feature.StringIndexer

val numCol = 300
val data = (0 to 10000).map { i => (i, 100 * i) }
var df = data.toDF("id", "label0")
(1 to numCol).foreach { idx =>
  df = df.withColumn(s"label$idx", col("label0") + 1)
}
val inputCols = (0 to numCol).map(i => s"label$i").toArray
val outputCols = (0 to numCol).map(i => s"labelIndex$i").toArray
val indexer = new StringIndexer().setInputCols(inputCols).setOutputCols(outputCols).setStringOrderType("frequencyAsc").fit(df)

1, master
2, the GroupedCount expression
3, regular aggregate I (similar to https://github.com/apache/spark/pull/47742#discussion_r1715912924)
    val selectedCols = getSelectedCols(dataset, inputCols.toImmutableArraySeq)
    val results = Array.fill(selectedCols.size)(new OpenHashMap[String, Long]())
    dataset.select(posexplode(array(selectedCols: _*)).as(Seq("index", "value")))
      .where(col("value").isNotNull)
      .groupBy("index", "value")
      .agg(count(lit(1)).as("count"))
      .collect()
      .foreach { row =>
        val index = row.getInt(0)
        val value = row.getString(1)
        val count = row.getLong(2)
        results(index).update(value, count)
      }
    results

4, regular aggregate II
    val selectedCols = getSelectedCols(dataset, inputCols.toImmutableArraySeq)
    val results = Array.fill(selectedCols.size)(new OpenHashMap[String, Long]())
    dataset.select(posexplode(array(selectedCols: _*)).as(Seq("index", "value")))
      .where(col("value").isNotNull)
      .groupBy("index", "value")
      .agg(count(lit(1)).as("count"))
      .groupBy("index")
      .agg(collect_list(struct("value", "count")))
      .collect()
      .foreach { row =>
        val index = row.getInt(0)
        val map = results(index)
        row.getSeq[Row](1).foreach {
          case Row(value: String, count: Long) => map.update(value, count)
        }
      }
    results

The duration is almost the same ~31 seconds.
But the shuffle size varies:
1, 22.0 MiB
2, 14.9 MiB
3, 26.0 MiB
4, 26.0 MiB + 28.7 MiB (it has two stages)
I cannot compare the memory usage, since 1 and 2 are using ObjectHashAggregate which seems not monitor the memory

So shall we still apply the GroupedCount expression, because:
1, it is just a replacement with existing StringIndexerAggregator
2, its shuffle size is the smallest, comparing to other approaches.
@hvanhovell @HyukjinKwon @WeichenXu123

merged to master


Example code in README doesn't work.

I've copy and pasted the example in the README and updated the print statements for python3 compatibility.  I get the following.
Traceback (most recent call last):
  File "first.py", line 1, in <module>
    from f5.bigip import ManagementRoot
  File "/Users/zleslie/.pyenv/versions/3.6.1/lib/python3.6/site-packages/f5/bigip/__init__.py", line 20, in <module>
    from icontrol.session import iControlRESTSession
ModuleNotFoundError: No module named 'icontrol'

Seems like a dependency issue with the pip package.

I just installed python 3.6.2, created a virtual environment with that version, installed the sdk via pip, and ran the example code just fine. The icontrol module is in the setup requirements. Can you try again?

@jasonrahm it used to not be there (a problem with our packaging that fixed when I did the 3.x work). It should be fixed now. If @zlesnr doesn't have any issue in the next couple hours, I think it's acceptable to close this by end-of-day

If the dependencies have been sorted out, sounds like this is done.  Thanks.


Load balancer for EKS cluster is publicly accessible

It seems like the load balancer for the EKS cluster is publicly accessible, which I believe is not desired?
You can test it by going to the load balancer address in a browser and it is publicly reachable.

I've noticed this too. @tobuck-aws any thoughts on this?


Bot crashes because user cannot be found

This is the same issue that pull request #24 is trying to work around. The bot crashes with the following:
Traceback (most recent call last):
  File "amibot.py", line 61, in <module>
    main()
  File "amibot.py", line 11, in main
    bot.run()
  File "/Library/Python/2.7/site-packages/slackbot/bot.py", line 36, in run
    self._dispatcher.loop()
  File "/Library/Python/2.7/site-packages/slackbot/dispatcher.py", line 94, in loop
    self._on_new_message(event)
  File "/Library/Python/2.7/site-packages/slackbot/dispatcher.py", line 52, in _on_new_message
    username = msguser['name']
TypeError: 'NoneType' object has no attribute '__getitem__

The problem is with this bit of code:
        botname = self._client.login_data['self']['name']
        try:
            msguser = self._client.users.get(msg['user'])
            username = msguser['name']
        except KeyError:
            if 'username' in msg:
                username = msg['username']
            else:
                return

self._client.users doesn't contain this particular user, and so msguser['name'] gives a TypeError because msguser is none. As far as I can tell, client.users is set on login. Presumably that means that any users appearing after the bot has logged in are not known to the bot and will cause this problem.

We can do the following regarding this issue:

merge #24 to avoid the crash
add another thread that periodically updates the users list .

A cleaner alternative might be to detect when the user is not in the list and then refresh the list. That avoids the extra thread, and also avoids a workaround that might hide real bugs. But up to you.

Just experienced this issue, however no user joined the channel after starting the bot. Probably #24 as suggested by @lins05. I agree with @larsga to update the user list on demand.

The bot no longer crashes. #61 tracks the invisible users problem, something we definitely need to fix.


Upgrade to service-my-wallet-v3

I think that docs/wallet.md and docs/create.md should also be updated to reflect these changes.

utACK. Someone who has write access should merge and tag with 2.0 (as there are desired breaking changes)


[NVIDIA XLA:GPU] Fused MHA Support in XLA GPU: SE

cuDNN currently supports the following patterns for Multi-headed attention:
BMM1 - BMM2
BMM1 - Scale - Bias - Mask - Softmax - BMM2
BMM1 - Scale - Bias - Mask - Softmax - Dropout - BMM2
BMM1 - Scale - Mask - Softmax - BMM2
BMM1 - Scale - Mask - Softmax - Dropout - BMM2
BMM1 - Softmax - Dropout - BMM2
This PR adds support for the stream executor for these patterns.

@akuegel @cheshire Based on https://github.com/tensorflow/tensorflow/pull/60139#issuecomment-1486799319, I created this new PR.

@akuegel Possible to merge this now? Hopefully no more issues :)


Be smarter about integer deserialization

Now, deserialize_any will try to find the best representation possible.
This is not a very pretty implementation, but it should work well.
Fixes #100
Fixes #155

Thanks for the review, addressed your comments.

bors r+


Seneca auth

@mcdonnelldean here is the seneca-auth stuff in place and I took the liberty to add lab and linting tools + fixed the errors :) Hope this is what you expected. Let me know if you have any concerns

Epic stuff, can you add a travis file too? So I need to pull concorda for this to work right?

@mcdonnelldean no, not yet. This just uses seneca-auth to handle the login and logout. I'll add the travis file so don't merge this just yet :)

@ckiss Perfect

@mcdonnelldean travis file is added now. can you please review and merge?

Looks good but let's hold off until @mirceaalexandru makes his agreed changes based on the mail today


Embed Item Quality Data in Message

Rather than having qualities like Accurate link to an item (which the user may or may not have permission to view, and which is tough to get right), we should embed the description in the message so it can be easy displayed to users regardless of further migrations or system changes.

Done, will be released in 1.807.


User can't go back to login page from 'Forgot Password' page.

When a user clicks "Forgot Password", then after being directed to that page, user can't go back to Login page back.

So, I want to work in this issue, by putting a button so that user can go back to sign in page, under GSSoC'24.

@TanayMaurya84 okay go ahead


x/crypto/acme/autocert: implement HTTP-01 challenge responses

Presently only the TLS-SNI-02 & TLS-SNI-01 Challenges are supported:
https://github.com/golang/crypto/blob/12c985a/acme/autocert/autocert.go#L455..L463
Adding support for the HTTP-01 challenge would be helpful for servers that sit behind a TLS-terminating proxy (such as Google Cloud HTTP(S) Load Balancer), and therefore cannot receive TLS-SNI requests directly. My intended use case is to have a server running on GKE behind such a load balancer (Kube Ingress) provision/update certs and then use the Kube API to update TLS secrets for the load balancer to serve.
Would you be open to a change that adds this support? Probably by way of adding an optional *http.ServeMux to the autocert.Manager struct, and updating the function linked above?
/cc @x1ddos @bradfitz

If you're doing to do something custom and wire up a weird contraption, the https://godoc.org/golang.org/x/crypto/acme package has HTTP-01 support already. Have you got that working first?
autocert is supposed to be magic for a common config. Maybe your use case will also become a common config, but it's not clear it is already.
 You can implement the autocert cache interface and put the certs on GCS or something.
In any case, what API do you have in mind? I'd say make it work first and then propose the least invasive API change possible.

The API change I had in mind is adding this exported field to autocert.Manager:
// ServeMux optionally specifies a request multiplexer to use for HTTP-01 challenges.
// If specified, HTTP-01 challenges will be preferred over TLS-SNI challenges.
ServeMux *http.ServeMux

Usage would then look something like:
func main() {
    m := autocert.Manager{
        Prompt:     autocert.AcceptTOS,
        HostPolicy: autocert.HostWhitelist("example.org"),
        Cache:      autocert.DirCache("/mnt/autocert/cache"),
        ServeMux:   http.DefaultServeMux, // Used to serve HTTP-01 challenge responses.
    }
    go func() {
        t := time.NewTicker(time.Minute)
        for range t.C {
            if cert, err := m.GetCertificate(); err != nil {
                // Log error.
            } else {
                // Store cert in place used by TLS-terminating proxy.
            }
        }
    }()
    http.ListenAndServe(":http", nil)
}

This could be done just using the acme package, but I'd like to take advantage of the conveniences autocert provides.
In any case, I'll try and get it working first then report back. Thanks for the feedback!

The concrete type http.ServeMux should never appear in API surface.
And instead of naming the field ServeMux, you'd write what it's used for. Actually, try writing the docs. What would you say?
type Manager struct {
     // Foo optionally specifies a Handler to ...
     // If nil, ...
     Foo http.Handler

Also, that one minute polling loop is kinda gross. Any polling is gross: it's either too fast or too slow. You want to have no TLS for a minute while you wait to install the cert?
Any API needs to work instantly and not involve loops with sleeps.

Wouldn't it be even simpler to just have Manager implement http.Handler? Then one could do:
m := &autocert.Manager{Prompt: autocert.AcceptTOS}
http.Handle("/.well-known/acme-challenge/", m)
http.ListenAndServe(":80", nil)

Manager would still need to know which challenge type is preferred, tls-sni or http-01, but this feels less intrusive towards existing API.

That seems like too big of a cut through the abstractions. I'd rather not add a ServeHTTP method and make users know or think about the ACME protocol. That sends the wrong message about how Manager is supposed to be used.

RE: Polling - yes, it's ugly, the client could just poll once every RenewBefore or so, but it would be nicer if Manager exposed a way of watching for updates to certificates. Something like:
func (m *Manager) WatchCertificate(hello *tls.ClientHelloInfo) <-chan *tls.Certificate
But this would be a fairly invasive change to Manager.
RE: ServeMux/Handler.  I don't understand how one can add a Handler for a path (e.g./.well-known/ to a Server by passing Manager an http.Handler. I think it would have to be the other way around, having Manager implement http.Handler like Alex suggests, or perhaps adding a function to Manager's interface like:
func (m *Manager) HTTP01ChallengeHandler() http.Handler
Given this would expose details of the ACME protocol to users of autocert.Manager, and that's not desirable, I think I'll just use the acme package directly as suggested by Brad in the first reply. Thanks all!


Expirator - an app which can be used as Trial/Free cleanup job

Description
Changes proposed in this pull request:

Expirator - an app which sends expiration requests for a given plan and expiration period
Most of the code is taken from trialcleanup (which will be removed finally)

Related issue(s)
Related: https://github.com/kyma-project/kyma-environment-broker/issues/593

The code is taken from trailcleanupjob, changed logger, added PlanID as a configuration env.


Create CI (GitHub actions) for the generated code

🎯 Goal: - Currently we have a CI only for the root project, the root, we obviously need to include one with the generated code. For a start using GitHub actions, in the future we might add more vendors. This CI should include few typical steps like testing, linting and vulnerabilities checker
🤔 Things to consider:
- The basics - Add a simple GitHub action workflow as described in this article, we actually have one in the root of our project. For a start, we surely want it to run the tests (already exists in every package and Microservice)

Where - In the code templates folder where the generated solution is
How to test - The workflow that you're adding is not under the root, so it won't run for this repository. How can we ensure that it works? This is one of the challenges in this task
Monorepo challenges - The generated code is not a single component rather multiple that are linked in a monorepo (Lerna). What exactly the CI should execute? This worth a discussion, as a simple first step it might run the tests of all sons:

lerna command test

@shanizlo Leme know if you want to hop online someday and chat about it. We can sync in 15min if you wish, or you can just process it on your own time


Implement PEP 625 (naming of sdists and projects)

PEP 625 and the current wheel spec. specify that project names and sdists should use lower case names. Implementing this will change the names of sdists and wheels which may cause problems for external tooling and so should probably be done as part of a feature release.
setuptools v69.3.0 implemented this support which impacts the names of the generated sip module sdists, but does not impact the names of the corresponding wheels (which is determined by the project name set by sip-module).
The project name in the generated sip modules' pyproject.toml files should be lower case and lower case should be used in the name of any generated sdist and wheel files.

Ultimately the specification of the format of a distribution name (and therefore sdist and wheel file names) is in the Escaping and Unicode section of the binary distribution section of the Python Packaging User Guide. Although it specifies the use of lower case characters it also states that tools consuming wheels must also be prepared to accept upper case characters.
We therefore choose the leave the current implementation as it is to avoid potential compatibility problems for existing projects. It is up to project authors to implement distribution names as they see fit.

PyPI is now sending emails warning that in future sdists must conform to PEP 625.

sdists now have lower case file names but the project names have been left untouched. Hopefully this is enough to satisfy PyPI.


Create SpaceHunt.html

Simon's HTML main page

Will work on these fixes and complete movement behavior tomorrow. Good suggestions for sure. 👍

OK I've got my user story to an initial working point. I wanted to get system status info to display with output tags but couldn't figure it out, so the system status displays with input text tags.


Change to ./bin

See if it builds

Could you try to print the file tree, to see if the file is downloaded correctly???
Because on bin-manager it works, so it's really strange.


Adding SRI to all fields.

See #561

@jdorfman This is available on http://dev.bootstrapcdn.com/, please review and I'll release to prod shortly.

:+1: @jmervine thanks for your hard work here.

My pleasure! Thanks for getting the ball rolling. 😜

@jmervine I don't see the SRI fields anymore

@jdorfman intentional, it is now on all of them isn't it.
Can I also say this has really helped educate people about SRI:
http://stackoverflow.com/questions/32039568/what-are-the-integrity-and-crossorigin-attribute

@jonathanKingston @jmervine apologies!  I need new glasses. =p


Wiki lookup plugin additions and improvements

As requested in issue #8431
The request is to make it possible to use the wiki lookup button to cast on items, I am not sure how that would be possible in some interfaces or tabs. I have instead been working on making it possible with adding an option while right clicking on items in some interfaces or tabs. Also, Wiki lookup on quests seems to be already working this way.
What i did:

Change right click options from "Guide, quick guide, and wiki" to "Lookup Guide, Lookup Quick Guide and Lookup Guide to be consistent with casting the wiki lookup text"
Add wiki lookup functionality to the following tabs and interfaces:
Items in equipment tab
Items in bank
Items in inventory while bank is open
Items in smithing interface
Items in ge and search
Items in ge inventory
Items in crafting interface
Items in music tab
Items in prayer tab
Items in spellbook tab
Items in shop
Items in shop inv

entory
Some example screenshots:

Suggestions and help would be appriciated, rather new on the runelite and open source development alltogether.

I don't think this is the best solution but because this is the way it already was implemented in the quest, achievement and skilltab i figured why not implement it also in other tabs and interfaces.
The reason i dont think this is the best solution because it is not consistent the way it works to lookup wiki on items/npcs/objects. this way some wiki lookup functionality is partly hidden (I personally did not know there was a right click lookup wiki guide functionality on the quest tab untill i read about it.
Any suggestions/ideas?


Issue 1: Default global content styling

[x] Adjust width of body for site pages as on original site.
[x] Adjust default content styles (default colors, font sizes etc.) in styles.css
[x] Add fallback font
[x] Add favicon
[x] Adjust media breakpoints, as/if required

Fix #1
Test URLs:

Before:
https://main--piramal--aemsites.hlx.page
https://main--piramal--aemsites.hlx.page/mirko-test-page-styles
After:
https://issue-1--piramal--aemsites.hlx.page
https://issue-1--piramal--aemsites.hlx.page/mirko-test-page-styles

Also, @msagolj
as per : https://pagespeed.web.dev/analysis/https-issue-1--piramal--aemsites-hlx-page/2oootukycw?utm_source=psi&utm_medium=redirect&form_factor=mobile
there is an accessibility issue coming up around the sequence of headings (h3 -> h1 -> h4) - so levels are getting skipped.
This needs to be updated to maintain the sequence.(highlighted parts in DOM below)

Also, @msagolj as per : https://pagespeed.web.dev/analysis/https-issue-1--piramal--aemsites-hlx-page/2oootukycw?utm_source=psi&utm_medium=redirect&form_factor=mobile
there is an accessibility issue coming up around the sequence of headings (h3 -> h1 -> h4) - so levels are getting skipped. This needs to be updated to maintain the sequence.(highlighted parts in DOM below)

@asthabh23 these are all headings where H level can be edited by the author. h3 -> h1 should be ok, h1 to h4 are titles so they can be changed. in their original site they dont see the issue because because the h4 is text so that on them to decide what supposed to be a title or just text.
In general their site is inconsistent whith using  or  for titles e.g. 'Happy Customers ' is an H2 while 'Primal Since 1980' is a styled DIV


'env: node\r: No such file or directory' - related to yarn, with reslate v3.0.0-5

Bug Description
This command: npx reslate build
gives this error:
 env: node\r: No such file or directory
Reverting to version 3.0.0-4 solves this problem.
Steps to reproduce:

install reslate (yarn add reslate)
npx reslate init
npx reslate build

Screenshots

Browser:

OS: MacOS BigSur
Version: 11.4

Thanks for the report. Just tried it on MacOS 11.4 and could not reproduce (using npm).
Does this happen if you omit the yarn add step, or use npm i or npm i -g instead?

I also get it on linux. I am doing yarn add -D reslate

Are you able to replicate with npm, and not yarn?

You are correct. If I use just npm it works.
Any ideas why?
Anyway thanks for the quick help.

Most reports of this issue seem related to CR/LF line ending issues on the scripts created on install. Why yarn would seem to be getting this wrong, or doing it in a way which is incompatible with npx or cross-var is still unclear.

# Check that dos2unix exists
if [ ! `command -v dos2unix` ]; then
    sudo apt install -y dos2unix
fi

# For each *.js in cwd, recursively change line endings to Unix style
for filejs in `find . -name "*.js"`; do
    echo "Converting line endings in $filejs"
    dos2unix $filejs
done


NetBios: fixes #434, added two functions to common.

Pull Request (PR) description
Restructured GET and SET functions based on the following:

An adapter can only be configured via WMI method if IPEnabled=$true. Otherwise the registry key must be configured. This satisfies disabled and disconnected states.
Current setting can only be obtained via the registry since WMI is unreliable due to various adapter states.
Only configure an adapter when not in the desired state.
Retain wildcard support.

This Pull Request (PR) fixes the following issues

Fixes #434

Task list

[ ] Added an entry under the Unreleased section of the change log in the CHANGELOG.md.
Entry should say what was changed, and how that affects users (if applicable).
[ ] Resource documentation added/updated in README.md in resource folder.
[ ] Resource parameter descriptions added/updated in schema.mof
and comment-based help.
[ ] Comment-based help added/updated.
[ ] Localization strings added/updated in all localization files as appropriate.
[ ] Examples appropriately added/updated.
[ ] Unit tests added/updated. See DSC Resource Testing Guidelines.
[ ] Integration tests added/updated (where possible). See DSC Resource Testing Guidelines.
[ ] New/changed code adheres to DSC Resource Style Guidelines and Best Practices.

This change is

Hi @phbits - thanks for submitting this - there are a few style issues causing the HQRM failures. I can identify them for you if you like? Sorry for taking a long time to get onto these - I'm completely snowed under with day job till Dec. But I'll be back across all this then.

Hi @phbits - I'm back on board now and have some capacity to work on DSC again. Are you planning to work on this? There are a few style and other issues that are causing this to fail - do you want me to do the review and identify key issues?

Hi @PlagueHO! I'm happy to continue working on this. The HQRM errors about a new line after an opening brace weren't making any sense to me so I put it on hold to learn Pester as the Unit tests will need to be fixed due to this PR. Let me take another look at the HQRM requirements and I'll update this thread.

@PlagueHO - Thanks for the thorough review and feedback! I finished the unit tests yesterday and all the tests are completing successfully. Will start working on your suggested HQRM edits.

@PlagueHO - All the tests are completing successful for this PR so I'm going to work on the DnsServerAddress PR.

Hi @phbits - can you resolve the conflicts on this one? Check that the changes in the CHANGELOG.md appear in the [Unreleased] section.

Hi @phbits - looks like there are still conflicts in the CHANGELOG.md

@PlagueHO , I resolved the CHANGELOG conflict. Will resolve the DnsServerAddress PR after this PR is closed.


razor pageremote type=post

In this.remote = function:
return new Promise(function (ok, reject)
{
var request = new XMLHttpRequest();
if (params.type === 'post') | not 'Post'. better yet: params.type.tolowercase

Here:
https://github.com/haacked/aspnet-client-validation/blob/4956e54606903351c9b9cc38a86a941aae6b1282/src/index.ts#L366-L369
Agree that a case-insensitive match would be preferred.


CLI asks for v4 public ip from a public ipv6 prefix

az feedback auto-generates most of the information requested below, as of CLI version 2.0.62

Describe the bug
When attempting to create a public IP from a public IP prefix that is IPv6, the command fails if the version parameter is not included.
To Reproduce
Create public IPv6 prefix, then request an IP from the prefix without specifying version parameter.  The request will eventually timeout and fail, because NRP is requesting a v4 IP.
PS C:\Users\blehr> az network public-ip prefix create --name IPv6prefix --resource-group willdeletethis --location eastus --version IPv6 --length 126
{
"customIpPrefix": null,
"etag": "W/\"3ad152a3-6aa2-43c4-93f9-c821ca3b5928\"",
"extendedLocation": null,
"id": "/subscriptions/6bb4a28a-db84-4e8a-b1dc-fabf7bd9f0b2/resourceGroups/willdeletethis/providers/Microsoft.Network/publicIPPrefixes/IPv6prefix",
"ipPrefix": "2a01:111:f100:2002::8975:2d90/126",
"ipTags": [],
"loadBalancerFrontendIpConfiguration": null,
"location": "eastus",
"name": "IPv6prefix",
"prefixLength": 126,
"provisioningState": "Succeeded",
"publicIpAddressVersion": "IPv6",
"publicIpAddresses": null,
"resourceGroup": "willdeletethis",
"resourceGuid": "6779f972-4a2b-47ae-b74e-839c625fe70d",
"sku": {
"name": "Standard",
"tier": "Regional"
},
"tags": null,
"type": "Microsoft.Network/publicIPPrefixes",
"zones": null
}
C:\Users\blehr> az network public-ip create --name IPv6testing --resource-group willdeletethis --location eastus --sku Standard --public-ip-prefix /subscriptions/6bb4a28a-db84-4e8a-b1dc-fabf7bd9f0b2/resourceGroups/willdeletethis/providers/Microsoft.Network/publicIPPrefixes/IPv6prefix

This second command will fail after approximately 30 minutes with an Internal Server Error.
In general, with an existing public ip prefix, you should not need to call out: Version, Allocation Method, SKU, Tier, or Zonal properties – these all should be assumed to match the prefix.  This should be fixed for all cases.
Expected behavior
The Public IPv6 address should be allocated as requested.
Environment summary
azure-cli 2.20.0 on local Powershell 7.

network

@00Kai0 Let me know any questions you have re: this ask

@yonzhan This has been opened since September of last year.  When will it be actioned?


Update NextJS and Mantine to latest

Description
It would be nice to update to nextjs 13 and mantine 5.10 at least, to get #598 solved but there are some bugs
Note
Check out the linked branch next-13
Priority
Medium (Would be very useful)

Moved to 0.11.4 due to time constraints


Not Saving Attributes

So far, I've focused on sending a response as jsonapi.  The other side of the coin is accepting requests in jsonapi and saving attributes correctly.  Thus far, this does not happen.

Where are you with this?  I think this is a great project.

Hi, thanks for the interest.  So far I've not had a chance to really figure this out.  I have a feeling I'll need to override blueprints, but I'm trying to avoid that.  If you have any thoughts (or PRs!) please feel free to send them this way. :smile_cat:

There is an RFC in jsonapi-serializer to add deserialization, which would help this effort greatly.

I made a PR https://github.com/SeyZ/jsonapi-serializer/pull/43 for that RFC, if you would like to try it. thats what we are using in our servers rigth now

Hey, will this project be continued?

At this time I am no longer using Sails or Ember, so I am unlikely to continue working on it any time soon, unfortunately.   I'd be happy to turn it over to anyone else who's interested in carrying the torch.

@IanVS out of curiosity what have you moved onto (for your api)?

I've joined a startup using PHP on the backend (but still emitting json-api).

interesting. thanks!

https://github.com/SeyZ/jsonapi-serializer supports deserialization since 2.1.0. It wasn't documented before 3.0.0.

@jelhan, very cool.  Want to take a shot at a PR?

@IanVS I'm not quite sure yet. Currently I'm investigating what work is necessary to get this working with Ember.js. Just sharing results.

I started work on a PR to support creating, updating and deleting resources.

Creating and deleting resources is implemented. Have started work on update here: https://github.com/jelhan/sails-hook-jsonapi/tree/update

Updating has just been merged. Closing this one for now.


Full size expand no longer works on Gelbooru

Just a few weeks ago it worked, but now the thumbnails cannot be clicked.
Firefox Version 123.0 64-bit, Tampermonkey with script installed from Sleezy Fork

@bird-ee
This is because Gelbooru's API returns a non-standard time, causing the script to report an error.
Fixed in v0.32.1


Fix crash on XCode 9.3

Thank you for sending the PR! We appreciate you spending the time to work on these changes.
Help us understand your motivation by explaining why you decided to make this change.

Test Plan

Related PRs

Release Notes

[CATEGORY] [TYPE] [LOCATION] - Message

Warnings

:warning:

:lock: package.json - Changes were made to package.json. This will require a manual import by a Facebook employee.

:warning:

:exclamation: Big PR - This PR is extremely unlikely to get reviewed because it touches 386532 lines.

:warning:

:grey_question: Base Branch - The base branch for this PR is something other than master. Are you sure you want to merge these changes into a stable release? If you are interested in backporting updates to an older release, the suggested approach is to land those changes on master first and then cherry-pick the commits into the branch for that release. The Releases Guide has more information.

  Generated by :no_entry_sign: dangerJS


Introduces the concept of scope and improves the :has() pseudo-class

Trying to get :scope and :has() working correctly (#439). I've introduced the concept of scopes to queries. Because an ISelector doesn't maintain a query state (and thus shouldn't know the scope before matching), the scope gets passed down from the outer-most .QuerySelector() call.
I think I'm pretty close, though I still can't get some of my new tests for :has() to pass. I suspect I need to spend a little more time tweaking HasFunctionState to properly use the new scoping stuff.
Mainly I just wanted your feedback at this point. I know this is something you were looking at in #352 so I figured I'd surface this in case it's close to the direction you were thinking of going.

I think I'm getting close. It can pass the simple :scope test I added and it also passes all the additional :has() tests except one.
The failing test is from the spec and uses an adjacent sibling selector p:has(+ p). It's tricky because the HasFunctionState implementation typically looks at children of the current node for matches. In this case however, we'd need to look at the current node itself to find the match.
My thought at the moment is to change the behavior under certain conditions to go up a level and match against self. I'll play a little more with this tomorrow to see if I can get a clear test board.

Alright, very cool! Keep this one updated - it would be great to have it in the next release!

All tests now pass. I resolved the issue of supporting sibling matches relative to the scope of :has() by checking if the scope is being using either implicitly (as is the case when the pseudo-class selector starts with a delimiter) or explicitly (I.e., :scope in the pseudo-class selector). In both cases, we search for matches a level up so we check for matches not only in the target element's child nodes, but it's siblings too. This should be safe since we've ensured the query scope is in use, and that refers to the target element.
At this point I'll consider this PR ready for review and merge. It's a big change, so it'll obviously need an extra set of eyes. Hopefully I've covered all the bases.

Alright, I looked over it - looks very good. Besides the cosmetic points of fitting to the overall source code style of AngleSharp I guess the point to discuss is if we really want to expose the scope parameter in some QuerySelector/All signatures. I would not include it. I would only have the Match signature changed - if one really wants to use the scope parameter one could always construct an ISelector and use it directly.
Apart from that discussion it all seems very good - thanks for the PR! Looks fantastic 👍

Thanks :) The scope needs to be set when we make the outer call to QuerySelector() since it tracks the document/element that we're executing the query against. To make sure I understand, you're thinking that instead of passing the scope through the internal calls to the QuerySelector() helpers that we should wrap the whole selector in something like a ScopedSelector and use that to pass the scope instead of the method signatures? Let me take a stab at that and see what I come up with...

So I didn't get too far. The problem is with the methods in the internal QueryExtensions class. For example, take the following implementation (before I modified it):
public static IElement QuerySelector(this INodeList elements, String selectors)
{
    var sg = CreateSelector(selectors);

    if (sg == null || sg is UnknownSelector)
        throw new DomException(DomError.Syntax);

    return elements.QuerySelector(sg);
}

It gets the selector string and is responsible for creating the representative ISelector. So if I were going to use a new ScopedSelector to wrap the underlying selector, this is where I'd instantiate it and set the element scope. However, notice there's no element being passed in the method signature, only the set of nodes I'm evaluating the query against. So in order to create the scoped selector and give it a scope I still need to pass the scope IElement to this method.
So it's possible to use a wrapping ISelector that maintains the scope, but I'd still need to add the extra IElement to all the QuerySelector() helper methods that are responsible for building the selector from the selector string. AFAIK, these are all internal so it shouldn't change the API (for that matter, I don't think any of the QuerySelector() methods I originally added scope to were public).
Given that I'd need to add the scope element to at least a few QuerySelector() methods, is it worth removing it from the others and using an ISelector to pass scope or should we leave it as-is? Or equally likely, maybe I've overlooked something?

The QuerySelector/All methods (and extension methods, which exist for convenience) represent the DOM methods with the same names. I would rather create a public extension method for the ISelector named MatchAll, which is performing a selection with scope against a set of nodes.

Makes sense to keep the methods that match the DOM consistent. How would that work for calls like someElement.QuerySelectorAll(":scope > p")? Ideally, the scope for this query should be set to someElement even though we never explicitly created an ISelector or called any methods on one?
What if we added new alternative internal helpers called something like QuerySelectorWithScope() and then called those from inside the public QuerySelector() methods? We could also use the idea of a ScopedSelector to avoid modifying the other internal QuerySelector methods. That would keep the edits to both the public and internal APIs to a minimum.
I'll try and work up and iteration of this idea today or tomorrow to help continue the discussion.

Think I just realized what you were going after. Essentially the same as above, except instead of a QuerySelectorWithScope() helper extension, we'll have an extension on ISelector that does the wrapping and introduction of scope. Is that about right?

Regarding your example
eElement.QuerySelectorAll(":scope > p")

does such a query run in the browser? In the browser the executing element seems to be the scope.
I compared
document.body.querySelectorAll('body > a') //a
document.body.querySelectorAll(':scope > a') //b

both yield the same results. Other experiments resulted in the same output. So I think yes - the scope should be set to the element where the query is run against. If the element is run against the document then the IDocument node needs to be the scope.
Regarding the second point: Yes, exactly! Instead of the QuerySelectorWithScope we introduce an extension to ISelector, which is more natural and should also be exposed to the outside (i.e., public).

Okay, pushed a new commit that I hope is a bit closer to what you were thinking. I added the change as a second commit instead of amending so there'd be a record of the first version for now. I can squash if/once you're ready to merge.

Looks very good! Thanks for your patience with me 👍 !

Hey, cool! Wasn't expecting a merge so quickly :smile: And thank you - I use this a ton, so glad I finally found a way to contribute to such a great project.


Docs not getting generated.

I've created setup for my own flow, but here i'm not able to see my generated docs.
Everything is working fine path is being added but swagger-autogen is not able to pick my comments on route's function handler.
This is what my route.js file looks like.

AuthController.js is looking like this

other two helper functions

and

swagger.js file

Hi @uttam-internetsoft. Thank you for opening the issue. This is a known issue in #79. Because of this, I'll close it and any progress I'll post it there. Thanks!


Update setup.md to fix typo

In line 76 the env var CONFIG_FILE is set to the path of config.ini  , however in line 96 the wrong name and python syntax is used

Thanks for your contribution. Sorry for the delay in getting to this PR.
As part of #85, READMEs/setup.md was split into several README files and heavily updated in general.
This typo no longer exists in any of them.
> grep -r CONFIG_FILE|grep -v Binary|grep -v ENV_CONFIG_FILE
tox.ini:    CONFIG_FILE = {toxinidir}/config_local.ini
READMEs/services.md:export CONFIG_FILE=config.ini
READMEs/services.md:config = Config(os.getenv('CONFIG_FILE'))
READMEs/developers.md:export CONFIG_FILE=config.ini
READMEs/parameters.md:1. no input, so it uses CONFIG_FILE envvar
READMEs/parameters.md:## 3. No input, so it uses CONFIG_FILE envvar
READMEs/parameters.md:export CONFIG_FILE=config.ini

Therefore we can close this PR.

Thanks for your contribution. Sorry for the delay in getting to this PR.
As part of #85, READMEs/setup.md was split into several README files and heavily updated in general.
This typo no longer exists in any of them.
> grep -r CONFIG_FILE|grep -v Binary|grep -v ENV_CONFIG_FILE
tox.ini:    CONFIG_FILE = {toxinidir}/config_local.ini
READMEs/services.md:export CONFIG_FILE=config.ini
READMEs/services.md:config = Config(os.getenv('CONFIG_FILE'))
READMEs/developers.md:export CONFIG_FILE=config.ini
READMEs/parameters.md:1. no input, so it uses CONFIG_FILE envvar
READMEs/parameters.md:## 3. No input, so it uses CONFIG_FILE envvar
READMEs/parameters.md:export CONFIG_FILE=config.ini

Therefore we can close this PR.


Make EOS version number extraction handle RC version numbers

@MitchPierias

Will go ahead and close this for now. Happy to pick this back up if you're keen to get it in.


Render the application status on the program admin application list item

Description
Render the application status if it's set (or default to a value of "None").
Screenshot

Release notes:
Render the application status on the program admin application list item.
Checklist

[X] Added the correct label: < feature | bug | dependencies | infrastructure | ignore-for-release >
[X] Created tests which fail without the change (if possible)
[ ] Extended the README / documentation, if necessary

Issue(s)
#3268

Codecov Report

Merging #3385 (663d17d) into main (8d9bbb3) will not change coverage.
The diff coverage is n/a.

@@            Coverage Diff            @@
##               main    #3385   +/-   ##
=========================================
  Coverage     80.16%   80.16%
  Complexity     2349     2349
=========================================
  Files           288      288
  Lines          9286     9286
  Branches        620      620
=========================================
  Hits           7444     7444
  Misses         1597     1597
  Partials        245      245

:mega: We’re building smart automated test selection to slash your CI/CD build times. Learn more


Branch impact events are not running on the first commit to a branch

they are running on the second commit though.  I'm wondering whether this is an ingester bug.

Yes, the compliance check goals are not being updated on the first commit of a branch because the algorithm to find the before Commit does not work when this is the first Commit on a branch.

Blocked on requiring an update to our PushImpact Event generation.

working for latest in master


Docker example

Example of creating a docker image and running it

@sopov can you merge this? Thanks.

It worked, thanks 🙏

@sopov ping!

Why docker is needed for this? if you have go installed we are good to go right?

@sravan1000 in my case and that of many others: not having (or wanting to have) the environment to be able to compile code in go or not having knowledge about go (my case).
With a Docker image all this is simplified and you simply have to run it in a container passing it the appropriate parameters.
Also, this PR creates an example only, it is not my intention to upload it to docker hub (although it could be done).

I just added the workflow that creates the image automatically in ghcr and the new instructions to get the value of SecureID following https://github.com/sopov/resumeio2pdf/issues/27#issuecomment-2016434712

@sopov merge, please!


Remove deprecated chart code

Close 218 - Remove old charting code.

Remove files that are not used
Remove functions that are not used from Charts/utils
remove Highcharts and Highcharts-React from the project dependencies

 Removing old chart code (Highcharts)


NIXIO ImageSequence support

Rebased and cleaned the branch for PR #760.

The travis tests succeeded but this was not displayed properly by github.


Replace eslint with xo

I was doing some exploring and found xo and it looks pretty neat to me. The biggest drawback seems to be that you need to add a xo plugin to your $EDITOR but if that's a big drawback I think we could just extend eslint-config-xo.
I disabled most of the rules but left those that didn't need a lot of work and made sense to me and fixed the issues that xo was reporting.

Large changeset and doesn't really matter IMHO. XO is great for exactly that reason.
My vote is to fix the conflicts and merge asap

@benediktvaldez What do you think about people needing to config a different flow for their editor to use xo over eslint?

The simplest way to solve that would be to create a .eslintrc file in the root with
{
	"extends": "xo"
}

This should work in most editors that support eslint

Ah lol, of course 😄

I say we go for this and then we can change to the xo-config if we want to

Nothing changed in the tests here which makes me believe that the tests might be flaky after #403  😭


istioctl describe mTLS output a bit confusing

$ ~/istio/istio-1.5.0-beta.5/bin/istioctl x describe pod `kp sleep`
Pod: sleep-6bdb595bcb-drr45
   Pod Ports: 15090 (istio-proxy)
Suggestion: add 'version' label to pod for Istio telemetry.
--------------------
Service: sleep
   Port: http 80/HTTP targets pod port 80
Pod is DISABLE, clients configured automatically

Pod is DISABLE, clients configured automatically

I happen to know this is referring to TLS, but its not explicit to users that do not what this means.

This may be a duplicate of https://github.com/istio/istio/issues/20898 or https://github.com/istio/istio/issues/18988
The intention was to make the output grammatical as part of the fixes for the breakage that occurred in 1.4.  Now that https://github.com/istio/istio/pull/21268 has merged work should resume on this.

Background.  I re-installed Istio many, many times.  These are the outputs I observed and will be fixing as part of this work:
+------------+---------+----------------+--------------------------------
|mtls.enabled|mtls.auto| meshpolicy def | describe in 1.5.0-beta.5 says
+------------+---------+----------------+------------------------------------------------
|    false   |  false  |                | Pod is DISABLE and clients are None
+------------+---------+----------------+------------------------------------------------
|    false   |  true   |                | Pod is DISABLE, clients configured automatically
+------------+---------+----------------+------------------------------------------------
|    true    |  false  |                | Pod is STRICT and clients are ISTIO_MUTUAL
+------------+---------+----------------+------------------------------------------------
|    true    |  true   |                | Pod is STRICT, clients configured automatically
+------------+---------+----------------+------------------------------------------------
|    false   |  false  | mode:PERMISSIVE| Pod is PERMISSIVE and clients are None
+------------+---------+----------------+------------------------------------------------
|    false   |  true   | mode:PERMISSIVE| Pod is PERMISSIVE, clients configured automatically
+------------+---------+----------------+------------------------------------------------
|    true    |  false  | mode:PERMISSIVE| Pod is PERMISSIVE and clients are ISTIO_MUTUAL
+------------+---------+----------------+------------------------------------------------
|    true    |  true   | mode:PERMISSIVE| Pod is PERMISSIVE, clients configured automatically
+------------+---------+----------------+------------------------------------------------

A few questions about your table @esnible:

Is it safe to assume mtls.enabled is referring to the following:

apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: default
  namespace: istio-system
spec:
  mtls:
    mode: STRICT

Is mtls.auto is referring to spec.meshConfig.enableAutoMtls in the IstioOperator CR?

Lastly, for the meshpolicy column are you setting mtls.mode to PERMISSIVE like this:

apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: test
  namespace: test
spec:
  selector:
    matchLabels:
      foo: bar
  mtls:
    mode: PERMISSIVE

Thank you, looking forward to seeing those PRs land in a release.


/usr/bin/python: No module named flamegraph

i cannot start it because No module named flamegraph.
but i have installed: pip install git+https://github.com/evanhempel/python-flamegraph.git
'''pip install git+https://github.com/evanhempel/python-flamegraph.git
Collecting git+https://github.com/evanhempel/python-flamegraph.git
Cloning https://github.com/evanhempel/python-flamegraph.git to /tmp/pip-req-build-0o57y8hm
Running command git clone -q https://github.com/evanhempel/python-flamegraph.git /tmp/pip-req-build-0o57y8hm
/home/yiwei/.local/lib/python3.8/site-packages/pkg_resources/init.py:123: PkgResourcesDeprecationWarning: 0.23ubuntu1 is an invalid version and will not be supported in a future release
warnings.warn(
/home/yiwei/.local/lib/python3.8/site-packages/pkg_resources/init.py:123: PkgResourcesDeprecationWarning: 1.13.1-unknown is an invalid version and will not be supported in a future release
warnings.warn(
/home/yiwei/.local/lib/python3.8/site-packages/pkg_resources/init.py:123: PkgResourcesDeprecationWarning: 0.1.36ubuntu1 is an invalid version and will not be supported in a future release
warnings.warn(
Building wheels for collected packages: flamegraph
Building wheel for flamegraph (setup.py) ... done
Created wheel for flamegraph: filename=flamegraph-0.1-py3-none-any.whl size=5793 sha256=29005f3f6de44d87677934b3929f9ff5f527d75c4e229a204d17d24b7bde124e
Stored in directory: /tmp/pip-ephem-wheel-cache-x0ptsvmg/wheels/ef/34/64/fb50a5c67924972e8361d0641f1f1b56fa9f1ff8daad2550bb
Successfully built flamegraph
Installing collected packages: flamegraph
Successfully installed flamegraph-0.1
'''
what can i do here?

Hi @Miaoishappening,
I can not help you directly with your problem, I just wanted to chime in that with Python 3.8 this profiler does not work in general due to #18 and you'd have to use a version with the fix from #17, e. g. the fork at https://github.com/rossmcdonald/python-flamegraph. Installing that did work for me right away in Python 3.9.
However, I've been looking at a lot of different Python profilers recently, and to be honest I'd recommend simply using a different one in the first place, for flamegraphs py-spy could be a good alternative due to it being actively maintained, having a bigger community and being more advanced in terms of features.
Best wishes,
Aaron