Using Mujoco with ML Agents

The way Mujoco Unity plugin is currently working is simply to attach Mujoco in runtime and execute physics on Mujoco Nodes.
While that is a good approach for running simulation on runtime, but that doesn't help in running ML Agents training.
ML Agents training is probably the main reason why researchers and game developers would benefit from Mujoco being open sourced and available in Unity as a plugin.
In order for ML Agents training to work properly, we need the means to:-
1- Load a Scene
2- Reset a Scene ( for running episodes)
3- Handling Multiple Scenes (for Multi Agents)
Currently, I probably can do this by modifying the MjScene.cs file, but that is neither ideal nor re-usable, so I am hoping in future versions you will be able to expose such functionality.
I will keep playing with it anyway and will let you know here if I reached a solid approach or further ideas.

In terms of resetting a scene, the method I've been using is storing the kinematic state of the ragdoll (or copying it from another one) in an enumerable, and restoring it in a similar way MjScene recreates a scene when needing a reset (i.e. on episode begin).
public static unsafe (IEnumerable<double[]>, IEnumerable<double[]>) GetMjKinematics( MjBody rootBody)
        {
            MujocoLib.mjModel_* Model = mjScene.Model;
            MujocoLib.mjData_* Data = mjScene.Data;
            var joints = rootBody.GetComponentsInChildren<MjBaseJoint>().OrderBy(j=>j.MujocoName);
            var positions = new List<double[]>();
            var velocities = new List<double[]>();
            foreach (var joint in joints)
            {
                switch (Model->jnt_type[joint.MujocoId])
                {
                    default:
                    case (int)MujocoLib.mjtJoint.mjJNT_HINGE:
                    case (int)MujocoLib.mjtJoint.mjJNT_SLIDE:
                        positions.Add(new double[] { Data->qpos[joint.QposAddress] });
                        velocities.Add(new double[] { Data->qvel[joint.DofAddress] });
                        break;
                    case (int)MujocoLib.mjtJoint.mjJNT_BALL:

                        positions.Add(new double[] { Data->qpos[joint.QposAddress],
                                                     Data->qpos[joint.QposAddress+1],
                                                     Data->qpos[joint.QposAddress+2],
                                                     Data->qpos[joint.QposAddress+3]});

                        velocities.Add(new double[] { Data->qvel[joint.DofAddress],
                                                      Data->qvel[joint.DofAddress+1],
                                                      Data->qvel[joint.DofAddress+2]});
                        break;
                    case (int)MujocoLib.mjtJoint.mjJNT_FREE:
                        positions.Add(new double[] {
                                                        Data->qpos[joint.QposAddress],
                                                        Data->qpos[joint.QposAddress+1],
                                                        Data->qpos[joint.QposAddress+2],
                                                        Data->qpos[joint.QposAddress+3],
                                                        Data->qpos[joint.QposAddress+4],
                                                        Data->qpos[joint.QposAddress+5],
                                                        Data->qpos[joint.QposAddress+6]});
                        velocities.Add( new double[] {
                                                        Data->qvel[joint.DofAddress],
                                                        Data->qvel[joint.DofAddress+1],
                                                        Data->qvel[joint.DofAddress+2],
                                                        Data->qvel[joint.DofAddress+3],
                                                        Data->qvel[joint.DofAddress+4],
                                                        Data->qvel[joint.DofAddress+5]});
                        break;
                    }
                }
            return (positions, velocities);
        }

        public static unsafe void SetMjKinematics(this MjScene mjScene, MjBody rootBody, IEnumerable<double[]> positions, IEnumerable<double[]> velocities)
        {
            MujocoLib.mjModel_* Model = mjScene.Model;
            MujocoLib.mjData_* Data = mjScene.Data;
            var joints = rootBody.GetComponentsInChildren<MjBaseJoint>().OrderBy(j=>j.MujocoName);
            foreach ((var joint, (var position, var velocity)) in joints.Zip( positions.Zip(velocities, Tuple.Create), Tuple.Create))
            {
                switch (Model->jnt_type[joint.MujocoId])
                {
                    default:
                    case (int)MujocoLib.mjtJoint.mjJNT_HINGE:
                    case (int)MujocoLib.mjtJoint.mjJNT_SLIDE:
                        Data->qpos[joint.QposAddress] = position[0];
                        Data->qvel[joint.DofAddress] = velocity[0];
                        break;
                    case (int)MujocoLib.mjtJoint.mjJNT_BALL:
                        Data->qpos[joint.QposAddress] = position[0];
                        Data->qpos[joint.QposAddress + 1] = position[1];
                        Data->qpos[joint.QposAddress + 2] = position[2];
                        Data->qpos[joint.QposAddress + 3] = position[3];
                        Data->qvel[joint.DofAddress] = velocity[0];
                        Data->qvel[joint.DofAddress + 1] = velocity[1];
                        Data->qvel[joint.DofAddress + 2] = velocity[2];
                        break;
                    case (int)MujocoLib.mjtJoint.mjJNT_FREE:
                        Data->qpos[joint.QposAddress] = position[0];
                        Data->qpos[joint.QposAddress + 1] = position[1];
                        Data->qpos[joint.QposAddress + 2] = position[2];
                        Data->qpos[joint.QposAddress + 3] = position[3];
                        Data->qpos[joint.QposAddress + 4] = position[4];
                        Data->qpos[joint.QposAddress + 5] = position[5];
                        Data->qpos[joint.QposAddress + 6] = position[6];
                        Data->qvel[joint.DofAddress] = velocity[0];
                        Data->qvel[joint.DofAddress + 1] = velocity[1];
                        Data->qvel[joint.DofAddress + 2] = velocity[2];
                        Data->qvel[joint.DofAddress + 3] = velocity[3];
                        Data->qvel[joint.DofAddress + 4] = velocity[4];
                        Data->qvel[joint.DofAddress + 5] = velocity[5];
                        break;
                }

            }
            // update mj transforms:
            MujocoLib.mj_kinematics(Model, Data);
            mjScene.SyncUnityToMjState();
        }

In terms of loading a scene, the plugin works somewhat well with prefabs. You can store a scene in one and instantiate it during runtime, but with weird behaviour if you create multiple instances.

In terms of resetting a scene, the method I've been using is storing the kinematic state of the ragdoll (or copying it from another one) in an enumerable, and restoring it in a similar way MjScene recreates a scene when needing a reset (i.e. on episode begin).
Example static functions to use:

Thanks @Balint-H for the tip! Very much appreciated :)
By inspecting the MjScene.cs file, I got a few ideas similar to this one, and it's most probably ideal to save the state as you mentioned and make sure to call SyncUnityToMjState
I did it forcefully with a DestroyScene/CreateScene - the easy but not optimized way. It works well so far, but I will consider your approach when building a multi-agent approach.

Lastly multi agent learning works with ML-Agents with the num-envs argument when running from build. This wastes some resources due to running multiple instances of mujoco in the background, but works. You can also place multiple agents/robots in the scene, and I had success with training with them with a slight speedup compared to just relying on num-envs.

I am trying a slightly different approach, training multiple agents simultaneously in the same scene, like the classic 3dball example. This approach speeds up training alot in some scenarios, also it should be one instance of Mujoco, and multiple mujoco envs. I am using Model/Data Array instead of a single Model/Data. POC seems to work, but code needs some modification to run efficiently. I will try to share the code once it works, hopefully soon :) Cheers

Lastly multi agent learning works with ML-Agents with the num-envs argument when running from build. This wastes some resources due to running multiple instances of mujoco in the background, but works. You can also place multiple agents/robots in the scene, and I had success with training with them with a slight speedup compared to just relying on num-envs.

I am trying a slightly different approach, training multiple agents simultaneously in the same scene, like the classic 3dball example. This approach speeds up training alot in some scenarios, also it should be one instance of Mujoco, and multiple mujoco envs. I am using Model/Data Array instead of a single Model/Data. POC seems to work, but code needs some modification to run efficiently. I will try to share the code once it works, hopefully soon :) Cheers

Hi @michelhabib, what you have described sounds like it could work and can be very useful - did you manage to get it work in the end?


No sound on speakers with PipeWire.

Problem
I compiled and installed PipeWire on my Raspberry Pi 4 (running Raspberry Pi OS Bookworm) so that I could completely disable bluetooth from it (and have a more recent version that supported module-alsa-source and module-alsa-sink). Then, I configured PipeWire as of the PulseAudio integration wiki page, changing pulseaudio to pipewire-pulse in the bluepulse systemd service. I'm trying to use BlueALSA as sink, and I'm testing it with my phone (Galaxy S21 5G)
The problem is that BlueALSA receives the audio, but doesn't play it on the speakers. The bluepulse script creates the source for my phone when audio starts playing and, as of pavucontrol, PipeWire does receive audio data, but this audio is not played into the speakers.
I think that this is more of a PipeWire bug than a BlueALSA bug, but I'm adding this here so that a possible solution could be documented in the wiki.
You can only hear the phone's audio when you run bluealsa-aplay XX:XX:XX:XX:XX:XX, otherwise it is sent to the ALSA source and it's not played onto the speakers.
Reproduction steps
All I did is enable all the PipeWire services and BlueALSA (bluealsa -S -p a2dp-sink -c sbc -c mp3 -c aac -c aptx -c aptx-hd -c ldac --ldac-quality=high --ldac-abr) using systemd.
Then, I rebooted and triple-checked that everything was running and, even though everything was working, BlueALSA gave this problem. I also tried to restart all the services, but nothing changed.
Setup

Raspberry Pi OS Bookworm
BlueALSA v4.1.1
BlueZ v5.66
ALSA v1.2.8
PipeWire v0.3.85
Branch and commit: f7e6ee5 (HEAD, tag: v4.1.1) Bug fix version bump (release 4.1.1)
Configure options:

--enable-manpages

--enable-aac

--enable-ldac

--with-libopenaptx

--enable-mp3lame

--enable-mpg123

--enable-aptx --enable-aptx-hd

--enable-msbc

--enable-rfcomm

--enable-hcitop

--enable-systemd

--enable-cli

--enable-debug

I am intrigued by this. I've always found Pipewire Bluetooth support to be very good, and never imagined anyone would want to use BlueALSA on the same system. Anyway, your main problem here (there may be others, as this setup has never been tried before AFAIK) is with Wireplumber.

of pavucontrol, PipeWire does receive audio data

So you have not told Pipewire what to do with that audio stream. In a default install with Pipewire's internal bluetooth, then the default Wireplumber config sets up a loop from bluetooth source to default (speaker) sink. I think you will have to find a way to configure  Wireplumber so that it does something similar when a new ALSA module source appears.

I've always found Pipewire Bluetooth support to be very good, and never imagined anyone would want to use BlueALSA on the same system.
I am trying to use BlueALSA on PipeWire because, AFAIK, it does not support decoding LDAC, and because I am more familiar with BlueALSA anyway.

So you have not told Pipewire what to do with that audio stream. In a default install with Pipewire's internal bluetooth, then the default Wireplumber config sets up a loop from bluetooth source to default (speaker) sink. I think you will have to find a way to configure Wireplumber so that it does something similar when a new ALSA module source appears.
I see. I'm trying to configure WirePlumber for that right now, but I can't seem to get it working at the moment.

but I can't seem to get it working at the moment.

As noted in the wiki, the Pulseaudio module-loopback does not work with the bluealsa plugin as a module-alsa-source. Perhaps the Pipewire equivalent (whatever that is, I'm not a Pipewire expert) has similar difficulties. I happen to be looking into ways that the bluealsa plugin can be made to behave more like a hw ALSA device, especially for ALSA capture, but that work is likely to take several more months before it comes to anything useable,
In the meantime, if the Pipewire loopback cannot be made to work, then you will have to consider some other way of transferring the incoming stream to an output device. bluealsa-aplay may well be the best option for that.

but I can't seem to get it working at the moment.

As noted in the wiki, the Pulseaudio module-loopback does not work with the bluealsa plugin as a module-alsa-source. Perhaps the Pipewire equivalent (whatever that is, I'm not a Pipewire expert) has similar difficulties. I happen to be looking into ways that the bluealsa plugin can be made to behave more like a hw ALSA device, especially for ALSA capture, but that work is likely to take several more months before it comes to anything useable,

Yes, PipeWire presents the same issues as PulseAudio. I tried using both module-loopback from pactl and pw-loopback, and both returned the same result.

In the meantime, if the Pipewire loopback cannot be made to work, then you will have to consider some other way of transferring the incoming stream to an output device. bluealsa-aplay may well be the best option for that.

I see. For now I am going to disable PipeWire and enable bluealsa-aplay, until this is resolved.

I've updated the wiki page with some notes on using pulse's module-loopback with BlueALSA sources, and also mentioned using bluealsa-aplay as an alternative. The bash script has been updated to include an option to automatically load module-loopback.
@Vinchethescript Does that description match what you did with load-module module-loopback?
What result did you get when using that module with Pipewire?
Now that you have demonstrated that the pulse load-module module-alsa-source command works with Pipewire, I would like to extend the wiki article to include what you have learned. Please could you tell:

how did you disable Pipewire's internal Bluetooth support?
did you need to modify the bash script in any way, or perform any additional manual configuration?
did you try using bluealsa-aplay --pcm=pipewire instead of loading the alsa-module-source module?

Any other useful information that you can provide, including links to references to may have used, would be helpful.
I plan to upgrade my laptop to Ubuntu 24.04 later this year, so will lose my last pulseaudio installation, and will then investigate more the co-existence of BlueALSA and Pipewire.

I've updated the wiki page with some notes on using pulse's module-loopback with BlueALSA sources, and also mentioned using bluealsa-aplay as an alternative. The bash script has been updated to include an option to automatically load module-loopback.

I tried to use that script even with the loopback, but it just exits with a syntax error because s32le  is not present into sample_size.

@Vinchethescript Does that description match what you did with load-module module-loopback? What result did you get when using that module with Pipewire?

I got the same results as running it into PulseAudio.

how did you disable Pipewire's internal Bluetooth support?

I had to recompile PipeWire from scratch using the guide available on their repository.

No, I did not do any manual configuration and I did not edit the bluepulse bash script.

did you try using bluealsa-aplay --pcm=pipewire instead of loading the alsa-module-source module?

I just tried it, and it does show up on pavucontrol this time, meaning the audio passes through PipeWire, and there is sound on the speakers! I noticed, though, that there's more latency when audio is playing through PipeWire, while there's less latency when using ALSA directly.

Thanks, that is all very helpful. Given that recompilation of pipewire is required, I think that this all too much to be added on to the existing pulseaudio wiki page.  Would you agree that co-existence of Pipewire and BlueALSA is just not worth the effort?

s32le  is not present into sample_size

oops! I'll upload a new version with that corrected ASAP.

Would you agree that co-existence of Pipewire and BlueALSA is just not worth the effort?

I think it depends on which codec someone needs, unless PipeWire supports decoding and playing high quality codecs such as LDAC. AFAIK, PipeWire doesn't support it, which is why I've been trying to use BlueALSA on PipeWire (I used a different library to compile BlueALSA with LDAC support, which includes decoding).

I've just updated the wiki page https://github.com/arkq/bluez-alsa/wiki/PulseAudio-integration with information on how to disable pipewire bluetooth support. I still have not found any way to get the pipewire loopback module to work with bluealsa, but for me using bluealsa-aplay with pipewire gives good results.

The wiki page Pipewire or PulseAudio Integration is now complete, documenting the working setup that I now have with Ubuntu 24.04LTS (pipewire 1.0.5, wireplumber 0.14.17, bluealsa 4.2.0). I think there is no way to get loopback working from a BlueALSA source because the BlueALSA PCM simply cannot deliver samples at a steady enough rate, but bluealsa-aplay is doing a good job for me in its place. Sure the latency is high, but as Bluez is unable to report stream delay to the remote source using AVDTP it has never been possible to synchronize with video on the phone anyway.
I'm closing this issue, but if anyone finds issues with the new wiki text please do open a new issue so that we can try to resolve them. (but please report issues specifically related to the bluealsa-agent program to the bluealsa-autoconfig issues list).


No transaction history saved.

Ran bot but after purchasing stock no transaction history saved.

my apologies, this piece of functionality is something that I somewhat overlooked because I didn't feel like it was very important and ended up implementing in a misleading way - the "tradehistory" file doesn't log every purchase you make, but instead logs the results of how much profit/loss was incurred when the bot decides to sell a stock. For example, if the bot buys a share of APL on Monday and then sells it on Friday, it will only write to the file upon selling the stock - saying you made/lost $X. If the bot buys APL and then holds on to it, nothing is logged because in the bot's mind, you haven't gained or lost any actual money until you decide to sell it. I do think that it is better to log every transaction though. I'll try to remedy this at some point.

Your explanation is good enough for me. So if I understand, the bot will calculate the profit/loss based on buy/sold info that it gets from Robinhood so it doesn't need to save data at time of purchase? If that's the case, I can live with that.


Error in Unity

Library\PackageCache\com.aarthificial.pixelgraphics@dbe057aecd\Runtime\Forward\VelocityRenderFeature.cs(38,13): error CS0619: 'ScriptableRenderPass.ConfigureTarget(RenderTargetIdentifier)' is obsolete: 'Use RTHandle for colorAttachment'

If this is due to the transition to a new version of Unity, then you can simply replace the line in the source code. Thus:
Before:
 _pass.ConfigureTarget(renderer.cameraColorTarget);
After:
_pass.ConfigureTarget(renderer.cameraColorTargetHandle);


Da update for caching structured obj

pr #358 left a few items incomplete on the V02-caching and V02-structured-obj computation modes. Those are not target operational modes so the PR was merged in spite of that.
This PR fixes those items (using the loop-less da procedure; using _i for temporary indexes) and also updates the V02-structured DA algorithm with a minor but important update to the timestep indexing.

This PR implements three sections of nearly identical code in lines 455-473 and lines 1377-1430 in mc_reach.pyx. I'm thinking that we can remove some of the code duplication and hopefully streamline things (both generally desirable goals) for all the inputs if we create a function to handle those calculations.
The challenge would be that there are slight differences in the handling of timesteps and in the declaration of the flowveldepth arrays. Can anyone think of a way to make a cython function to treat a cdef float[:,::1] and a cdef np.ndarray[float, ndim=3] as synonymous? We have those two constructions in our different compute functions representing the same variables (flowveldepth) and I would like to prepare a single function to initialize them both. In other words, I would like to create a function into which I can pass either the memoryview with 2 dimensions or the nd array with 3 dimensions and, given the congruency of their data storage, populate them equivalently.
One option would be to modify the caching-based version of our function to use the 3-dimensional declaration (so they would all be synchronous.) But that loses the advantages of the typed memoryview, which are more general and more efficiently used with cython. See the following:

"Typed memoryviews allow efficient access to memory buffers, such as those underlying NumPy arrays, without incurring any Python overhead. Memoryviews are similar to the current NumPy array buffer support (np.ndarray[np.float64_t, ndim=2]), but they have more features and cleaner syntax."
https://cython.readthedocs.io/en/latest/src/userguide/memoryviews.html

With that in mind, I'd prefer to leave the memoryview in place and find a way to convert the array declaration, at least for purposes of handling inside this function.

ping @groutr

Works on Cheyenne with the fcast test case.


error while serving ionic 2 project

I have the ionic version 2.0.0-beta.30
when I serve my project I get this error
`Caught exception:
Error: watch node_modules/angular2/src/animate/css_animation_options.js ENOSPC
Caught exception:
node_modules/angular2/src/platform/dom/util.js ENOSPC
at exports._errnoException (util.js:860:11)
at FSWatcher.start (fs.js:1313:19)
at Object.fs.watch (fs.js:1341:11)
at createFsWatchInstance (/home/ameni/ionic/covoiturage/node_modules/chokidar/lib/nodefs-handler.js:37:15)
at setFsWatchListener (/home/ameni/ionic/covoiturage/node_modules/chokidar/lib/nodefs-handler.js:80:15)
`
and many other errors like that. Can I anyone help me to solve this problem

Could you provide me with the output from ionic info from within your project directory? Thanks, this should help in troubleshooting.

Thanks for opening the issue. It seems that issue has been resolved. I am going to close this issue but please feel free to reopen another issue if this occurs again.


reduced font size, rotated labels and updated team colours

Yeah that's always been there, I think we just leave it


sized_grain: Suggestions list

[x] Remove AddGrain support since its redundant.
[ ] Grainfactory3 AVS has recently gotten significant updates, port over it's changes
[x] Change default resample kernel to lanczos and allow it to be configurable.
[x] Remove the reliance on resample kernel arguments to adjust sharpness, we could do a simple blur/sharpen convolution instead.
[ ] Add support for vs-noise type=4

* [ ]  Add support for vs-noise type=4

Since type=4 (among others) is signal dependent; application on a blank clip is useless.

its*

Can i get a link to Grainfactory3 AVS?

https://github.com/Dogway/Avisynth-Scripts/blob/master/FilmGrain%2B.avsi

Moved the last missing thing in #92


get traction_engine working

tracktion_engine builds on Android, and it is most likely that it can handle audio plugin related bits in JUCE.
Then, if we implement juce_audio_processor for AAP it is likely to work. augene project can be used as a reference player. It is very simple auduo plugin based music player based on tracktion_engine.
Problem: even if we get this working, there is no "instrumental" sample yet.

So far everything avoid dynamic JNIEnv instancing from JavaVM.
bd74234 has some JUCEy treat:

JUCE integration has moved to aap-juce repo. Closing.


Hi, I wanna join as a contributor in refactoring

Is there any way we can work together to do this. I've gone through your code and the avl tree but a lot of errors come up since I started learning dart last year but I'm really loving it.
Since dart is also an being uptaken by developers alot, we can help the OpenGeo community.
this code to update it to the latest dart version and if you could help, I could maintain the code after understanding it well.

Hi, I worked on this repo two or three years ago. Unfortunately, I didn't have enough spare time to continue the work. You probably already noticed that this is very much work in progress. Your help would be welcome! It would be great, if the code turned into something useful for the OpenGeo community.

Since dart_jts has been created, i bet i I can close this


Play/Pause Buttons aren't updated when using the slickPause/slickPlay Methods

When using the slickPause method such as $(".slider").slick('slickPause') to pause an autoplaying slider the play/pause buttons on the front-end are not automatically updated, even though the autoplaying slider does in fact pause.
My solution currently is to do this right after calling the slickPause method. But would be nice if it happened as a result of that method as well.
$(".slider .slick-play-icon").css("display", "inline");
$(".slider .slick-pause-icon").css("display", "none");

Didn't see this issue was already opened by another user. Closed for redundancy


Libvirt: Change the libvirt Volumename for s390x architecture

For s390x Architecture,  there is need for differentiate the Volume Name.
In s390x environment, to differentiate hyper protect peer pod image and  other image, we planned to use the volumeName as the differentiator. Hence we planned to change the volume Name from "cidata" to "cc_cidata".
Existing s390x code base uses the VolumeName as differentiator.  Hence we need to use same approach.

The tests are all failing:
=== RUN   TestLibvirtCreateSimplePod
=== RUN   TestLibvirtCreateSimplePod/SimplePeerPod_test
time="2024-12-12T15:55:58Z" level=info msg="ServiceAccount default updated successfully."
    assessment_runner.go:274: timed out waiting for the condition
--- FAIL: TestLibvirtCreateSimplePod (600.04s)
    --- FAIL: TestLibvirtCreateSimplePod/SimplePeerPod_test (600.04s)
=== RUN   TestLibvirtCreatePodWithConfigMap
=== RUN   TestLibvirtCreatePodWithConfigMap/ConfigMapPeerPod_test
    assessment_runner.go:274: timed out waiting for the condition
--- FAIL: TestLibvirtCreatePodWithConfigMap (600.04s)
    --- FAIL: TestLibvirtCreatePodWithConfigMap/ConfigMapPeerPod_test (600.04s)
=== RUN   TestLibvirtCreatePodWithSecret
=== RUN   TestLibvirtCreatePodWithSecret/SecretPeerPod_test
    assessment_runner.go:274: timed out waiting for the condition
--- FAIL: TestLibvirtCreatePodWithSecret (600.03s)
    --- FAIL: TestLibvirtCreatePodWithSecret/SecretPeerPod_test (600.03s)
=== RUN   TestLibvirtCreatePeerPodContainerWithExternalIPAccess
    assessment_helpers.go:421: Failing on CI

We rely on the /run/media/cidata/user-data location to get the data in the peer pod vm, so I don't see how this change will work?

yeah, I think the mount unit would need to be adjusted accordingly. but then we can maybe rename it for all archs that use config disks?

This changes has been addressed in the s390x specific code for podvm


change pipeline

[corretto8] - Change pipeline

Change added to #2 so closing this.

This change has been added to #2 in commit 28aa59dee204593be25ece8e9ea6f7def95d8b0f


[FEATURE] Add support for custom authorization headers in chapters section of the tests.

Is your feature request related to a problem?
As of now, there are two ways http_client sets the authorization (see here):

Default username and password (admin and myStrongPassword123!)
A username and password supplied through CLI (process.env.OPENSEARCH_USERNAME and process.env.OPENSEARCH_PASSWORD)

This will apply to all tests and there is no option to apply it to a specific chapter or test file.
What solution would you like?
This issue requests addition of another way of passing custom authorization headers in request block of chapters. This will then be used by http_client to execute requests. It should look something like this:
  - synopsis: ...
    path: ...
    method: ...
    request:
      headers:
        authorization: Basic <base64-encoded-creds>
      payload:
        ...
    response:
      ...

What alternatives have you considered?
NA
Do you have any additional context?

https://github.com/opensearch-project/opensearch-api-specification/blob/main/tools/tests/tester/helpers.ts#L45
https://github.com/opensearch-project/opensearch-api-specification/pull/439#discussion_r1711800406

https://github.com/opensearch-project/opensearch-api-specification/pull/488


Add a "variant: " prefix to variant descriptions in test names

This tweaks the suffix added to test names when using variants to be slightly more specific ( (variant: foo) instead of just  (foo):

This is to help editors that have access only to the test name that want to support running tests easily from the editor, but without leaving the regex passed to flutter test --name open-ended, which might match other tests than the one the user expected to run (if they have their own suffix in parens).
Right now, VS Code uses a regex like:
flutter test --name "^Counter increments smoke test$" test/widget_test.dart

But this fails for tests with variants. The goal is to change this regex to something like:
flutter test --name "^Counter increments smoke test( \(variant: .*\))?$" test/widget_test.dart

This will still avoid matching other tests with a suffix in parens, but allow the variants to run, fixing #86659.
IntelliJ doesn't currently use the same restrictive regex, but that leads to behaviour that could be considered a bug and is likely to be updated with https://github.com/flutter/flutter-intellij/issues/5650.
Pre-launch Checklist

[x] I read the Contributor Guide and followed the process outlined there for submitting PRs.
[x] I read the Tree Hygiene wiki page, which explains my responsibilities.
[x] I read and followed the Flutter Style Guide, including Features we expect every widget to implement.
[x] I signed the CLA.
[x] I listed at least one issue that this PR fixes in the description above.
[x] I updated/added relevant documentation (doc comments with ///).
[x] I added new tests to check the change I am making or feature I am adding, or Hixie said the PR is test-exempt.
[x] All existing and new tests are passing.

If you need help, consider asking for advice on the #hackers-new channel on Discord.

@gspencergoog @jonahwilliams I'm assuming nobody outside of this repo is relying on the exact format of this suffix, but if you think that might not be correct, please let me know!

Gold has detected about 2 new digest(s) on patchset 1.
View them at https://flutter-gold.skia.org/cl/github/86701

I'm not sure how to interpret the skia-gold comment/results above - is it indicative of an error? (I see it's on a few other recent PRs though, so might not be related to my change).

I'm not sure what to make of the golden file changes either. Clearly your PR should not be changing golden file results, so I am inclined to ignore them.

This should be safe to merge, it looks like the flutter-gold check has resolved now. There were unchecked images that were merged in accidentally. Confirmed the reported images here are ok.

Thanks!


Added 7 quotes, modified pr_template

Check one of the below boxes as per your PR

[ ✅] Adding more quotes issue
[ ] Fixes another issue #00 (<=== Add issue number here)

Make sure these boxes are checked before your pull request (PR) is ready to be reviewed and merged. Thanks!

[ ✅] PR is descriptively titled 📑 and links the original issue above 🔗
[✅ ] code is in uniquely-named branch and has no merge conflicts 📁
[ ] screenshots/GIFs are attached 📎 if applicable
[ ] ask @avats-dev for help, in a comment below

I'm happy to help you get this ready -- don't be afraid to ask for help, and don't be discouraged !

Thanks!

@sriraj-gopu i can merge your quotes json as it is good, but revert changes you made in PR template as it needs to be edited as per the user in the pr comment. Leave it as it is. Thanks.

@avats-dev I have reverted the PR template back to its original state. Thanks.

@avats-dev Hey could you check and confirm. Thanks

@avats-dev Hey, are there any other changes. Could you check and confirm. Thanks.


"Can't apply field access to Null Type" only on Gateway update

I'm developing a custom connector to web service that uses GraphQL as it's API, but it all relies on sending requests and parsing Json responses.
The data extracted by the connector is structured like this:

It work as intended in any circumstance on the Desktop version of Power BI. But when I use a Personal Gateway to update the report I get "Can't apply access to Null Type", as if the "Data" column from the NavTable was empty (which it clearly isn't as shown bellow):

Here is the error that shows on trying to update the dataset on Power BI Online via Personal Gateway:

@migueesc123 Thanks for the recomendation, but my point is: if there's an expression error, why does it work fine on the Desktop version? And besides, the table where the error occurs is kinda random, this time was the Table Cards but everytime I try to run the update it shows a different table, same error.

it's really difficult to pinpoint the exact reason without looking at the code, but if you're not getting issues with the test connection part, then it's most likely how your navigation table gets created.
It's the same principle that you get with other connectors - some things might work on the desktop, but they won't necessarily work on the service itself because a number of things that could rely on also a number of things. Again, without looking at the code it's pretty hard to pinpoint the root cause of it, but creating a support ticket will give get the right team on track so they can have the right diagnostics on what's going on at the Power BI Service level.


Linter does not look at inline Polymer html templates

Polymer templates that are declared as html tagged template literals on the _template or static template fields on a Polymer element don't seem to be checked by the Polymer linter.
Needs investigation

Hi,
This issue is still valid. Do we have any progress on this?
Thanks
@rictic


clean out as many of the displayNewLine() functions as possible because we have a nicer looking alternative

i did it, and now there's only two left that i was instructed to ignore anyways


Remote Session - randomized keyboard output

Hi,
I experienced a bug in RDPWrap.
In Short

After pressing the Windows key/Home key and typing letters to find something, the keyboard input seems to be randomized. I type aaa on the remote client's keyboard and q3h shows up on the remote host's search bar.
I like the option to have concurrent session on one RDP host, like:

Another user uses the RDP host PC to work as user localuser.
I login as user rdpuser from another machine.
We can both work happily next to each other.
But: The issue described above affects localuser, too. He presses Windows key to use Windows' search just to see its keyboard randomized, too.

Example with screenshot:
I just typed aaa:

This is my setup:

I installed RDPWrap on one Windows 10 PC, listening on Port 3389.

Installation and configuration were without any problems or red flags.
I created a dedicated RDP user called rdp on RDP host PC.1

I'm using the pre-installed (old) Remote Desktop application on another Windows 10 PC.

Connection is established with [IP_ADDRESS]:3389
Login with user rdp.

No issues so far.
I were not able to reproduce the behavior described above by 100%.
Could this be an issue regarding the separation of keyboard inputs?

This sometimes happens for me in Windows 7 RDP as well.
It may be RDP Client issue, but usually disconnecting & connecting again helps. Also it may be related to some "sticky" keyboard keys.
Anyway, that's not RDP Wrapper issue.


NIFI-4087 Fix to allow exclusion of filename from tika criteria.

Thank you for submitting a contribution to Apache NiFi.
In order to streamline the review of the contribution we ask you
to ensure the following steps have been taken:
For all changes:

[X] Is there a JIRA ticket associated with this PR? Is it referenced
in the commit message?

[X] Does your PR title start with NIFI-XXXX where XXXX is the JIRA number you are trying to resolve? Pay particular attention to the hyphen "-" character.

[X] Has your PR been rebased against the latest commit within the target branch (typically master)?

[X] Is your initial contribution a single, squashed commit?

For code changes:

[X] Have you ensured that the full suite of tests is executed via mvn -Pcontrib-check clean install at the root nifi folder?
[X] Have you written or updated unit tests to verify your changes?
[ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under ASF 2.0?
[ ] If applicable, have you updated the LICENSE file, including the main LICENSE file under nifi-assembly?
[ ] If applicable, have you updated the NOTICE file, including the main NOTICE file found under nifi-assembly?
[X] If adding new Properties, have you added .displayName in addition to .name (programmatic access) for each of the new properties?

For documentation related changes:

[ ] Have you ensured that format looks appropriate for the output in which it is rendered?

Note:
Please ensure that once the PR is submitted, you check travis-ci for build issues and submit an update to your PR as soon as possible.

looks like a good idea and good change.  I'll let travis-ci do its thing and then take a look a bit later.  Thanks for contributing

Hi Joe. I can go ahead and push an update to remove the comment line. I do see though that the appveyor build failed but the travis-ci one did not. Looking at the appveyor build it looks like the failure wasn't related to the change. So is that safe to ignore that for now?

yep.  if travis is happy on at least one then we're likely in good shape.

OK sounds good. Just let me know if I need to do anything else for this (already pushed an updated commit to remove the comment you mentioned)

will give this a manual test run soon.  For future contributions it is probably easiest if you add commits to your PR rather than force pushing over top at least during the review phase.  This makes it easier on the reviewer because they can see the diffs between review/change cycles.  At the end the reviewer might ask you to squash/rebase or they can do it for you when they merge.
If you have any questions on that let us know.
Thanks again for contributing


change default web-shell storage to volatile

There are pouch specific bugs that occur, for example: "select favorite food" recipe - select some foods, then try to unselect - there is a warning in the log, and the store isn't updated any more ever

You can remove the // <alternate> bits if you want, I know I started that, but I'm not sure it's helping anybody.

Actually, I think it is helping to see that this is the right place to change for alternatives :)


Cloud Storage not persisting on error

I've ran into an issue that is only present in production using the Google Cloud Storage API.  I have an importer that loads 100 entries at a time.  When developing locally (with require('fs')) and an error occurs on say page 5, the previous pages worth of entries (400 total) are still persisted to disk.  When using GCS, this is not the case (the file doesn't even appear in the bucket).
Here is a snippet of the problematic part of the pipeline.
    stream = importer.streamPages({perPage: perPage, importDate: importDate}, maxPages);

    let writeStream = null;
    if (process.env.NODE_ENV === 'production') {
      writeStream = bucket.file(importFileName).createWriteStream();
    } else {
      writeStream = fs.createWriteStream(`data/${importFileName}`);
    }

    stream
      .on('error', err => this.logError(err))
      .pipe(ndjson.stringify())
      .pipe(writeStream);

The API can receive a file in two ways; a single request that contains the entire contents of the file (simple), or a series of requests (resumable). By default, createWriteStream uses resumable uploads, so that if an error occurs during the upload, you can start the transfer over, and only send data starting from where the last upload quit.
With a simple upload (you can pass resumable: false to createWriteStream), if an error occurs, the API never sees a completed HTTP request, so no file is ever created (@jgeewax, is this right?).

@stephenplusplus hmm, I'm using the default resumable stream.  If an error occurs, is there a way I can resume and just "commit" the current progress.
In my example, importer.streamPages() walks a paginated api, emitting a perPage amount of objects (it's an object stream) on each data event.  It would be nice if I could do something like this.
stream
      .on('error', err => { this.logError(err); this.done(); )

That sounds like a cool idea. I'll see if I can find a way to do that :+1:

I think what you can do is:
stream
      .on('error', err => { this.logError(err); this.end(); })

When a readable stream gracefully ends, it's saying "And you, next stream in the pipe, have received all I have to give you." Our write stream will continue normally, and write that data to the remote object. Try it out and let me know!

Thanks, I'll give that a shot and let you know.  In my example, I'll need to not sure an arrow function so this is the stream and not the closure, so I figure it will be more like...
let logError = this.logError;
stream
      .on('error', function(err) { logError(err); this.end(); })

I currently don't have a test for this, but will try to make one (against Cloud Storage) and see how it goes, as otherwise I'll have to wait for the failure to occur again (import.streamPages retries a call 5 times before failing, which a gateway time-out caused the issue before)

Cool, thanks for sharing details about your use case. It always helps to hear real-world scenarios.
RE: the this stuff, good catch, I didn't think of that. Here's another option:
stream
      .on('error', err => { this.logError(err); stream.end(); })

And just for reference, here is the script I used to test:
'use strict';

var through = require('through2')
var gcloud = require('gcloud')({ projectId: '...' })
var gcs = gcloud.storage()
var bucket = gcs.bucket('...')
var file = bucket.file('new-file-' + Date.now())

var srcStream = through.obj(function (chunk, enc, next) {
  next(null, JSON.stringify(chunk))
})
var writeStream = file.createWriteStream()

srcStream.on('error', function () {
  this.end()
})

srcStream.pipe(writeStream)

srcStream.write({ a: 'b' })
srcStream.write({ a: 'b' })
srcStream.write({ a: 'b' })
srcStream.destroy(new Error('error'))

@stephenplusplus thanks for your help.  I've made some progress on this, but my limited knowledge of Node streams has me hitting a wall.
I found some of my issue to be in my retryStream function, that uses a Passthrough stream/transform that I was previously just logging "Max attempts reached" when the streamFunc reached it's maximum number of retries.
Now, calling .end() on the PassThrough stream fixes the problem I was having with the file not showing up in GCS (after adding an arbitrary 1000ms setTimeout() to my test before calling done() so the node/mocha process didn't terminate before gcloude-node had time to write the file up to GCS.  I tried using setImmediate() but it didn't work).
Anyways, the problem I have is I really need to bubble an error out of retryStream through streamPage and streamPages so I can add an 'error' event listener at the end of the pipeline so I can properly log/email the problem before writing what I have grabbed so far to GCS (and a importStream which also processes the records and persists them to Cloud SQL) .
If I remove the error handler from BaseImporter#streamPage, then I get Uncaught Error: Uncaught, unspecified "error" event. (Max attempts reached), but if I leave it on, the error gets swallowed and the test times out because end or error never makes it.
If I attempt to "re-emit" the error (.on('error', function(err) { console.error('streamPage error', err); this.emit('error', err) })) I get a  Uncaught RangeError: Maximum call stack size exceeded
This is quite likely out of the scope of this issue, but thought I'd at least present my problem in case you had any suggestions.
Here are snippets of the related functions.   Thanks for any help you have on this.
retry-stream.js
'use strict';
const PassThrough = require('stream').PassThrough;

module.exports = function retryStream(streamFunc, retries) {
  var resultStream = new PassThrough();
  let attempt = 1;
  retries = retries || 3;

  function attemptStream() {
    let stream = streamFunc()
      .on('error', (err) => {
        console.warn(`Attempt (${attempt} of ${retries}) failed`, err);
        if (attempt < retries) {
          console.log('Retrying');
          attempt++;
          attemptStream();
        } else {
          console.error('Max attempts reached');
          // resultStream.end(); // <-- Works
          resultStream.emit('error', 'Max attempts reached'); // <-- Would like to work
        }
      })
      .on('response', () => {
        // console.log('success');
        stream.pipe(resultStream);
      })
  };
  attemptStream();

  return resultStream;
}

base.js
const got = require('got');
const MultiStream = require('multistream');
const through2 = require('through2');
const JSONStream = require('JSONStream');
const JSONPath = require('jsonpath');
...

class BaseImporter {
...
  streamPage(query, options) {
    const pageOptions = this._getPageOptions(query, false);
    const queryString = Object.keys(pageOptions.query).map(key => `${key}=${pageOptions.query[key]}`).join('&');
    // console.log('Fetching url', `${this.config.baseUrl}?${queryString}`);

    if (options && options.retries) {
      return retryStream(() => got.stream(this.config.baseUrl, pageOptions), options.retries)
        .on('error', err => console.error('streamPage error', err))
        .pipe(JSONStream.parse(this.config.jsonPath))
    } else {
      return got.stream(this.config.baseUrl, pageOptions)
        .on('error', err => console.error(err))
        .pipe(JSONStream.parse(this.config.jsonPath))
    }
  }

  streamPages(query, maxPages) {
    query.page = query.page || 1;
    let totalRaceCount = 0;
    let isDone = false;

    let factory = (streamCallback) => {
      if (isDone) {
        // console.log('Done');
        streamCallback(null);
        return;
      } else if (maxPages && query.page > maxPages) {
        console.log('Maximum pages reached.');
        streamCallback(null);
        return;
      } else {
        // console.log('Next page')
      }

      let raceCount = 0;
      let stream = this.streamPage(query, { retries: 5 })
        //.on('error', (err) => console.error('streamPages error', err))
        .on('end', () => {
          // console.error('streamPage end')
          console.log(`Fetched ${raceCount} races`);
          if (raceCount < query.perPage) {
            // console.log('Last page');
            isDone = true;
          }

          streamCallback(null, stream);
        })
        .pipe(through2({ objectMode: true, highWaterMark: query.perPage },
          function(chunk, enc, transformCallback) {
            // console.log('transform');
            // console.log('chunk', chunk);

            raceCount += 1;
            totalRaceCount += 1;

            this.push(chunk)
            transformCallback()
          })
        )

      query.page += 1;
    }

    return MultiStream(factory, { objectMode: true, highWaterMark: query.perPage })
      .on('error', (err) => console.error('MultiStream error', err))
      .on('close', () => console.log(`Total races: ${totalRaceCount}`))
  }
...
}

test.js
describe.only('BaseImporter GCS', function() {

  beforeEach(function() {
    nock('http://localhost')
      .get('/')
        .query({page: 1, perPage: 5})
        .reply(200, (uri, requestBody) => {
          // console.log('returning page1, 200')
          return [{}, {}, {}, {}, {}]
        })
      .get('/')
        .query({page: 2, perPage: 5})
        .reply(200, (uri, requestBody) => {
          // console.log('returning page2, 200')
          return [{}, {}, {}, {}, {}]
        })
      .get('/')
        .query({page: 3, perPage: 5})
        .times(10)
        .reply(500, () => {
          // console.log('returning 500')
        })
  })

  afterEach(function() {
    nock.cleanAll();
  })

  describe('#streamPages()', function(done) {
    it('should still write contents to GCS for previous pages after exhausting retries for later page', function(done) {
      this.timeout(5000);

      const importer = new BaseImporter({ baseUrl: 'http://localhost', jsonPath: '*' });

      const gcloud = require('gcloud');
      const ndjson = require('ndjson');
      const gcsConfig = require('../../../app/config').get('storage:googleCloudStorage');
      const gcs = gcloud.storage({ projectId: gcsConfig.projectId });
      const bucket = gcs.bucket(gcsConfig.bucket);
      const importFileName = `test/${new Date().toISOString()}`;
      const writeStream = bucket.file(importFileName).createWriteStream();
      // const writeStream = require('fs').createWriteStream('test.txt');

      const stream = importer.streamPages({perPage: 5}, 3)
        .on(
        .on('end', () => {
          // Give GCS a second to upload file before ending test (and killing process)
          setTimeout(done, 1000);
        })
        .pipe(ndjson.stringify())
        .pipe(writeStream)

        ;
    });
  });

});

No problem, I'm happy to take a look! I'll let you know if I can come up with anything

@stephenplusplus you are crazy responsive :)

Also just repo-dropping this because I also go through some headaches when trying to deal with streams, so I wrote up some guides for under/non-documented use cases: https://github.com/stream-faqs - I haven't dug into your code yet, but it might have a helpful section or two!

You mean https://github.com/stephenplusplus/stream-faqs and I've been there - https://github.com/techniq/wiki/wiki/Node.js :)

Haha, I just make sure to always keep a GH tab open:

And oh, nice! :)

As well a nice game.  You have a daughter I image (or really like Frozen :) )

btw, I'm on Hangouts/Slack/etc if you want to chat outside of the issue to keep the noise down.  I'm leaving the office in about 10 min to work my second shift (Chief Family Officer) but will be back on ~9:00pm EST.

Sounds about identical to my plans! I'll ping back I'm back on the case. No
worries about noise here, I think it'll only help others.
And yes, a daughter who loves Frozen enough for both of us:
https://youtu.be/tGKybwbsA34
On Fri, Jan 29, 2016, 4:51 PM Sean Lynch<EMAIL_ADDRESS>wrote:

btw, I'm on Hangouts/Slack/etc if you want to chat outside of the issue to
keep the noise down. I'm leaving the office in about 10 min to work my
second shift (Chief Family Officer) but will be back on ~9:00pm EST.
—
Reply to this email directly or view it on GitHub
https://github.com/GoogleCloudPlatform/gcloud-node/issues/1097#issuecomment-176985760
.

I'm not quite sure this is what you need, but end takes an error, first emitting it, then attempting to gracefully end the stream. So, instead of:
// resultStream.end(); // <-- Works
resultStream.emit('error', 'Max attempts reached'); // <-- Would like to work

Try:
resultStream.end(new Error('Max attempts reached'));

But for the issue of bubbling the error, I would try using pumpify.
I tried running the code, but I don't think I got it to work the way I was supposed to! I saw how removing the setTimeout caused the file not to be created, but I'm a little confused by the pipeline/process to really know where that problem is caused.
And just in case you haven't seen this one, take a look at retry-request... it might save you the headache of the retry stuff.
And since all I'm doing is dropping module names, another: paged-http-stream might be nice here because of how much it simplifies the pagination process.
Sorry that this may have added more confusion than you were looking for :( If you can put together a gist or repo I can clone that gets me up and running a little easier, I'll be happy to dive back in.

Thanks @stephenplusplus, that gives me plenty to look at.  I've already had to switch gears onto a different project so it will probably be a few days till I can spend more time on this.  I've been using got (for Promise support, as well as stream) and originally it had retry support but not for 500 errors (which was odd to me), but I think they may have changed that in later releases to also retry on 500s.  If so, I might also try to look if it can be solved with it.
If I continue to hit a road block, I'll definitely try to get a working gist setup.  When I posted the snippets I tried to keep them concise to the issue but definitely omitted some parts that you probably need to get it functional.
Thanks again for all your help.

No problem. I'll close this issue since I don't think gcloud needs to change, your app just needs to find the right place to handle the aborting of the stream. I'm still happy to help find where that is, so whenever/whatever I can do, just ping me back in here and we can keep working on it.

@stephenplusplus No problem, and good news, I spent some time on this yesterday and figured it out.  My retryStream now emits a new Error() when it exhausts the number of retries.  One of the big gotchas (and still not 100% sure why it's occurring) was needing to call stream.emit('error', err) within my streamPage function instead of this.emit('error', err) even when using a normal function and not an arrow function.
If i used a function and this.emit, it would stack overflow (Uncaught RangeError: Maximum call stack size exceeded)
      const stream =  retryStream(() => got.stream(this.config.baseUrl, pageOptions), options.retries)
        .on('error', function(err) { this.emit('error', err) })
        .pipe(JSONStream.parse(this.config.jsonPath))

but this works
  const stream =  retryStream(() => got.stream(this.config.baseUrl, pageOptions), options.retries)
        .on('error', err => stream.emit('error', err))
        .pipe(JSONStream.parse(this.config.jsonPath))

I also refactored my streamPages to no longer use MultiStream and a factory function but instead rewrote it to just use a PassThrough stream as I do with retryStream (I didn't know PassThrough existed when I first wrote that and wasn't sure how to dynamically chain a n number of streams into 1).
Here are my current functions in case your interested.
retry-stream.js
'use strict';
const PassThrough = require('stream').PassThrough;

module.exports = function retryStream(streamFunc, retries) {
  var resultStream = new PassThrough();
  let attempt = 1;
  retries = retries || 3;

  function attemptStream() {
    let stream = streamFunc()
      .on('error', (err) => {
        console.warn(`Attempt (${attempt} of ${retries}) failed`, err);
        if (attempt < retries) {
          // console.log('Retrying');
          attempt++;
          attemptStream();
        } else {
          // console.error('Max attempts reached');
          resultStream.emit('error', new Error('Max attempts reached'));
        }
      })
      .on('response', () => {
        // console.log('success');
        stream.pipe(resultStream);
      })
  };
  attemptStream();

  return resultStream;
}

...
class BaseImporter {
...
  streamPage(query, options) {
    const pageOptions = this._getPageOptions(query, false);
    const queryString = Object.keys(pageOptions.query).map(key => `${key}=${pageOptions.query[key]}`).join('&');
    // console.log('Fetching url', `${this.config.baseUrl}?${queryString}`);

    if (options && options.retries) {
      const stream =  retryStream(() => got.stream(this.config.baseUrl, pageOptions), options.retries)
        // .on('error', function(err) { console.error('streamPage error', err); this.emit('error', err) })
        .on('error', err => stream.emit('error', err))
        .pipe(JSONStream.parse(this.config.jsonPath))
      return stream;
    } else {
      return got.stream(this.config.baseUrl, pageOptions)
        .on('error', err => console.error(err))
        .pipe(JSONStream.parse(this.config.jsonPath))
    }
  }

  streamPages(query, options) {
    const PassThrough = require('stream').PassThrough;
    let resultStream = new PassThrough({ objectMode: true, highWaterMark: query.perPage });

    let totalRaceCount = 0;

    const getPage = (page) => {
      let raceCount = 0;
      query.page = page;
      options = options || {};
      options.retries = options.retries || 5;
      let stream = this.streamPage(query, options)
        .on('error', (err) => {
          // console.error('streamPages error', err)
          resultStream.emit('error', err);
        })
        .on('end', () => {
          // console.log('streamPages end')

          console.log(`Fetched ${raceCount} races`);
          if (raceCount < query.perPage) {
            // console.log('Last page');
            resultStream.end();
          } else if (options.maxPages && page >= options.maxPages) {
            // console.log('Maximum pages reached.');
            resultStream.end();
          } else {
            // console.log('Next page')
            getPage(++page);
          }
        })
        .pipe(through2({ objectMode: true, highWaterMark: query.perPage },
          function(chunk, enc, transformCallback) {
            // console.log('transform');
            // console.log('chunk', chunk);

            raceCount += 1;
            totalRaceCount += 1;

            resultStream.write(chunk)
            transformCallback()
          })
        )
    };
    getPage(query.page || 1);

    return resultStream
      .on('error', err => console.log(`Error`, err, `total races at error: ${totalRaceCount}`))
      .on('end', () => console.log(`Total races: ${totalRaceCount}`))
  }
...
}

base.test.js
describe('BaseImporter GCS', function() {

  describe('#streamPages()', function(done) {
    it('should still write contents to GCS for previous pages after exhausting retries for a later page', function(done) {
      this.timeout(5000);

      nock('http://localhost')
        .get('/')
          .query({page: 1, perPage: 5})
          .reply(200, (uri, requestBody) => {
            // console.log('returning page1, 200')
            return [{}, {}, {}, {}, {}]
          })
        .get('/')
          .query({page: 2, perPage: 5})
          .reply(200, (uri, requestBody) => {
            // console.log('returning page2, 200')
            return [{}, {}, {}, {}, {}]
          })
        .get('/')
          .query({page: 3, perPage: 5})
          .times(3)
          .reply(500, () => {
            // console.log('returning 500')
          })

      const importer = new BaseImporter({ baseUrl: 'http://localhost', jsonPath: '*' });

      const gcloud = require('gcloud');
      const ndjson = require('ndjson');
      const gcsConfig = require('../../../app/config').get('storage:googleCloudStorage');
      const gcs = gcloud.storage({ projectId: gcsConfig.projectId });
      const bucket = gcs.bucket(gcsConfig.bucket);
      const importFileName = `test/${new Date().toISOString()}`;
      const writeStream = bucket.file(importFileName).createWriteStream();
      // const writeStream = require('fs').createWriteStream('foo.txt');

      const stream = importer.streamPages({perPage: 5}, { retries: 3, maxPages: 5 })
        .on('error', err => {
          expect(err).to.exists
          expect(err.message).to.equal('Max attempts reached')
          stream.end()

          // Give GCS a second to upload file before ending test (and killing process)
          setTimeout(done, 1000);
        })
        .on('end', () => {
          expect.fail('"end" should not be emitted')
        })
        .pipe(ndjson.stringify())
        .pipe(writeStream)
        ;
    });
  });

});

and actually, I just tried to refactor...
streamPage(query, options) {
    const pageOptions = this._getPageOptions(query, false);
    const queryString = Object.keys(pageOptions.query).map(key => `${key}=${pageOptions.query[key]}`).join('&');
    // console.log('Fetching url', `${this.config.baseUrl}?${queryString}`);

    if (options && options.retries) {
      const stream =  retryStream(() => got.stream(this.config.baseUrl, pageOptions), options.retries)
        .on('error', err => stream.emit('error', err))
        .pipe(JSONStream.parse(this.config.jsonPath))
      return stream;
    } else {
      return got.stream(this.config.baseUrl, pageOptions)
        .on('error', err => console.error(err))
        .pipe(JSONStream.parse(this.config.jsonPath))
    }
  }

to
streamPage(query, options) {
    const pageOptions = this._getPageOptions(query, false);
    const queryString = Object.keys(pageOptions.query).map(key => `${key}=${pageOptions.query[key]}`).join('&');
    // console.log('Fetching url', `${this.config.baseUrl}?${queryString}`);

    let stream = (options && options.retries)
      ? retryStream(() => got.stream(this.config.baseUrl, pageOptions), options.retries)
      : got.stream(this.config.baseUrl, pageOptions);

    return stream
      .on('error', err => stream.emit('error', err))
      .pipe(JSONStream.parse(this.config.jsonPath))
  }

and I'm back at getting the Uncaught RangeError: Maximum call stack size exceeded

Using this example:
const stream =  retryStream(() => got.stream(this.config.baseUrl, pageOptions), options.retries)
    .on('error', function(err) { this.emit('error', err) })
    .pipe(JSONStream.parse(this.config.jsonPath))

but this works
const stream =  retryStream(() => got.stream(this.config.baseUrl, pageOptions), options.retries)
    .on('error', err => stream.emit('error', err))
    .pipe(JSONStream.parse(this.config.jsonPath))

What's happening is in the first example, this is the same context that the error was emitted on. So it's going "I got an error? I better tell me about it! I got an error? I better tell me about it! I got an..."
In the second example, when you emit on stream, (this is the important part:) stream is the last stream in the pipe that was given when you assigned the variable stream. So stream is equal to JSONStream.parse(this.config.jsonPath)
Every time you call pipe, it's basically:
Stream.prototype.pipe = function(nextStream) {
  // Glue this instance of a stream to `nextStream`...
  return nextStream;
};

So when you call .pipe after .pipe, the variable you're assigning them to will only reference the last one.

Interesting. So is there a better way to emit an error without it being catch by the same error handling?  I'm assuming this would fail (without testing it myself)
const stream =  retryStream(() => got.stream(this.config.baseUrl, pageOptions), options.retries)
    .on('error', err => stream.emit('error', err))

If the error is a deal-breaker, you should end the stream from where the problem began by calling this.end(err);
Here's an example of what happens when ending a source stream with an error:
var through = require('through2')
var sourceStream = through()

stream
  .on('error', console.error.bind(console, '\nError on source stream'))
  .pipe(through())
  .on('error', console.error.bind(console, '\nError on destination stream'))
  .on('end', console.log.bind(console, 'ended'))
  .pipe(process.stdout)

sourceStream.push('a')
sourceStream.push('b')
sourceStream.push('c')
sourceStream.end(new Error(':('))

$ node .
abc
Error on source stream [TypeError: Invalid non-string/buffer chunk]
ended

In your case, the upload stream should see the peaceful end from your stream, and upload the data to GCS.

At this point I'm going to just go forward with what I have.  Node Streams are still perplexing with how they work to me, but at this point I have something I can work with.
Thanks again for all your help.

Oh man, you're totally right. I was mixing up destroy with end. I should just close up stream-faqs now and call it stephen-is-a-noob.
Alright, the example I should have given is:
var through = require('through2')
var stream = through()

stream
  .on('error', console.error.bind(console, '\nError on source stream'))
  .pipe(through())
  .on('error', console.error.bind(console, '\nError on destination stream'))
  .on('end', console.log.bind(console, 'ended'))
  .pipe(process.stdout)

stream.push('a')
stream.push('b')
stream.push('c')
stream.emit('error', new Error(':('))
stream.end()

So emit the error like you are, but also call end, and the stream that you ended communicates to the other streams that they've received all the data they're going to get, so they can close properly. That should result in GCS getting the data.
Sorry to lead you down a dark path!

No worries.  Thanks :)

@stephenplusplus Guess who's back :)
We went into production with the changes we discussed here, but ran into a major issue.

The error handler is after the piping to the GCS write stream.
...
    // Download and import new entries
    stream = importer.streamPages({perPage: perPage, importDate: importDate}, { retries: 5, maxPages: maxPages});

    let writeStream = null;
    if (process.env.NODE_ENV === 'production') {
      writeStream = bucket.file(importFileName).createWriteStream();
    } else {
      writeStream = fs.createWriteStream(`data/${importFileName}`);
    }

    stream
      .pipe(ndjson.stringify())
      .pipe(writeStream)
      .on('error', err => {
        this.logError('Error writing stream', err, sourceName);
        stream.end();
      });
...

I've been unable to reproduce the issue locally on my workstation when using production resources (GCS, Google Cloud SQL, etc) via NODE_ENV npm start.  I think the following things might be highly related.

We're currently using a f1-micro instance currently (so only 600MB mem)
After the refactoring to streamPages, I don't think we're getting any back-pressure on the pipeline from the writeStream's (it's fetching 100 new races/records as fast as the event loop will allow it as far as I can tell from the logs).  I know this is due to how I'm performing the check within .on('end', ...) of the single streamPage call, but I'm not sure how to apply this logic so it will wait for the downstream writers/back-pressure before enqueuing the next getPage/streamPage.
Hunch: We're blowing through the micro instances memory causing the socket hangup to GCS to occur

  streamPages(query, options) {
    const PassThrough = require('stream').PassThrough;
    let resultStream = new PassThrough({ objectMode: true, highWaterMark: query.perPage });

    let totalRaceCount = 0;

    const getPage = (page) => {
      let raceCount = 0;
      query.page = page;
      options = options || {};
      options.retries = options.retries || 5;
      let stream = this.streamPage(query, options)
        .on('error', (err) => {
          // console.error('streamPages error', err)
          resultStream.emit('error', err);
          resultStream.end();
        })
        .on('end', () => {
          // console.log('streamPages end')

          console.log(`Fetched ${raceCount} races`);
          if (raceCount < query.perPage) {
            // console.log('Last page');
            resultStream.end();
          } else if (options.maxPages && page >= options.maxPages) {
            // console.log('Maximum pages reached.');
            resultStream.end();
          } else {
            // console.log('Next page')
            getPage(++page);
          }
        })
        .pipe(through2({ objectMode: true, highWaterMark: query.perPage },
          function(chunk, enc, transformCallback) {
            // console.log('transform');
            // console.log('chunk', chunk);

            raceCount += 1;
            totalRaceCount += 1;

            resultStream.write(chunk)
            transformCallback()
          })
        )
    };
    getPage(query.page || 1);

    return resultStream
      .on('error', err => console.log(`Error`, err, `total races at error: ${totalRaceCount}`))
      .on('end', () => console.log(`Total races: ${totalRaceCount}`))
  }

Any ideas?

Could you plop a gist or repo together with the bare minimums so I can jump in? (please include package.json!) I think we can figure it out, but I'm doing a lot of scrolling back and forth, so I think I'm probably just slowing us both down.

Will do.  Thanks

Probably take me an hour or so as I need to step into a meeter.  Are you on the Google Cloud team btw (just curious)?

Technically speaking, no. I'm only helping the team (@jgeewax) with this project.

I'm struggling to find a good way to reduce this into a simple reproducible example.  Since there are a lot of things at play (URL fetching of a large amount of data, slow consuming stream to a Cloud SQL database, and it's only reproducible (thus far) on a memory-limited micro instance.  You wouldn't be interested in doing a Hangout/screen share to walk through the code on my machine?

Sorry about the code back and forth. I was still trying to wrap my head around how all of the pieces of the pipeline played together. Stubs to stand-in for actual API services are fine. I was able to put the pieces together and get something running, but wasn't able to produce any meaningful results.

I'm not sure how to apply this logic so it will wait for the downstream writers/back-pressure before enqueuing the next getPage/streamPage.

You shouldn't need to do this, as far as I know. The upload stream should only take on what it can handle.
My guess is it relates to the v1-micro instance, like you suggested. If not from the low memory, maybe another limitation on the amount of sockets allowed to remain open. Is it possible to try on a more equipped VM for sake of comparison? The default maxSockets allowed to be open should be Infinity, so it shouldn't be coming from the Node side (require('http').Agent.defaultMaxSockets === Infinity).
Sorry I can't be more help here as this goes deeper into the lower levels of the stack.
@jgeewax @callmehiphop @mziccard can you think of anything?

Good news, I went back to an implementation of streamPages that uses MultiSteram and I did not receive the socket hang up error during our nightly tasks.  Using MultiStream, streamPages's factory is only called after the downstream consumers (writing to GCS and importing into Cloud SQL) have completed the current batch.  I couldn't find any kind of memory usage graph on https://console.cloud.google.com to confirm my hunch, but I'm pretty sure continually downloading all the pages from the rest service was blowing through the f1-micro's small amount of memory.
I had moved away from MultiStream to try to simplify our stream pipeline when troubleshooting the originally cause for the issue (not writing to GCS if an error was raised from one of the API page calls).  With my new units tests in place, I was able to make the appropriate changes and get it all to work.  The big change was adding the following to the streamPage() (single page) call within the streams factory of MultiStream:
        .on('error', function(err) {
          this.end();
          streamCallback(err);
        })

Here's the full impl if you're interested.  As always, thanks for the help.
  streamPages(query, options) {
    query.page = query.page || 1;
    let totalRaceCount = 0;
    let isDone = false;

    let factory = (streamCallback) => {
      if (isDone) {
        console.log('Done');
        streamCallback(null);
        return;
      } else if (options.maxPages && query.page > options.maxPages) {
        console.log('Maximum pages reached.');
        streamCallback(null);
        return;
      }

      let raceCount = 0;
      let stream = this.streamPage(query, options)
        .on('error', function(err) {
          // console.error('streamPages error', err)
          this.end();
          streamCallback(err);
        })
        .on('end', () => {
          // console.error('streamPage end')
          console.log(`Fetched ${raceCount} races`);
          if (raceCount < query.perPage) {
            // console.log('Last page');
            isDone = true;
          } else {
            // console.log('Next page')
            query.page += 1;
          }

          streamCallback(null, stream);
        })
        .pipe(through2({ objectMode: true, highWaterMark: query.perPage },
          function(chunk, enc, transformCallback) {
            // console.log('transform');
            // console.log('chunk', chunk);

            raceCount += 1;
            totalRaceCount += 1;

            this.push(chunk)
            transformCallback()
          })
        )
    }

    return MultiStream(factory, { objectMode: true, highWaterMark: query.perPage })
  }

That's great to hear! And thanks for posting here, so others can see how mind-crippling streams are. At this point, I'm not even sure the people who wrote them fully understand them!

haha, no kidding.  They're like regex to me, as soon as I think I'm on top of them, I take a week off, and feel like I know half what I did last time I looked at them.
On a similar note, I have hopes the observable / web streams.  Haven't played with RxJS yet, but figure I will soon when I get into Angular 2.  I considered using it at one point (or Highland.js) but felt like it was overkill at the time for what I was doing and was trying to keep things as simple as possible.


Visual: text gets stretched after a while (until restarting application)

Pertains to Version 5.0d16 (1592).

Is this still happening? Is it reproducible?

It was, but I haven’t seen it in the betas.
It seemed to be an accumulating thing over a longer stretch of time. I’ll observe and let you know.

On 15 Aug 2019, at 22:27, Brent Simmons<EMAIL_ADDRESS>wrote:
Is this still happening? Is it reproducible?
—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub, or mute the thread.

Quoth Brent Simmons (15 Aug 2019, 22:27):

Is this still happening? Is it reproducible?

It still happens.  These two screen shots are consecutive; I made one
when I suspected I saw the stretch, quit the app, relaunched it and
captured the very same area.
Before quit/relaunch:

After relaunch:

The first one stops at "Manton Reece", the second has two extra entries
-- in the identical screen real estate.
It's nothing I do, specifically, that has this effect.  It's just a
matter of time.

So weird. I can’t reproduce it, and nobody else has reported it yet.
Is there anything odd about your machine or your setup? The monitor, graphics card, some uncommon system extension, or…?

Quoth Brent Simmons (20 Aug 2019, 21:21):

So weird. I can’t reproduce it, and nobody else has reported it yet.

Yeah.  It's a subtle phenomenon.

Is there anything odd about your machine or your setup? The monitor,
graphics card, some uncommon system extension, or…?

Well, I might be a relatively zealous reader of feeds.  Strange though
this sounds, I have sometimes wondered if I saw spacing change when
hitting a refresh that didn't result in new unread items.  Other than
that, it's a simple MacBook Pro, nothing funky, using only the attached
Retina display.  Well, as I'm a security professional it is somewhat
hardened, networkwise, but no funky stuff there either.  And there's
VMware Fusion, VirtualBox, Docker and so on.  The only thing I'm doing
that's out of the ordinary is probably Soundflower [1] which is not
something I'm inclined to relate to anything graphical.
When I first noticed the items in the feed list receiving extra spacing,
I still ran Mojave.  It's Catalana (ß6) now, but that apparently didn't
impact this issue.  And I don't think I've seen it happen on my iMac at
home — which I use less.
I have developer tools installed, and if there's anything you'd need to
collect runtimewise I probably can.  And you might consider filing it
away as a cosmetic oddity affecting one particular idiot's system only,
and as long as it hits nobody else that's of course entirely true.
Cheers, Jeroen.
[1] https://rogueamoeba.com/freebies/soundflower/

Quoth Brent Simmons (20 Aug 2019, 21:21):

So weird. I can’t reproduce it, and nobody else has reported it yet.

Still happening...
Grab before restarting NNW:

Grab of identical screen area after:

Wanted to point out this is still happening with 5.0.3.

It also sometimes happens in the article list, but with the text stretched horizontally.

I can’t reproduce it and I have no idea at all what the problem is or how to fix it. Moving it to 5.1 milestone.

Hi,
I just saw this too today.
I have an external monitor, and the Macbook turned off the displays (normal power saving stuff), but I have set it to not hibernate or sleep the Macbook.
Could it be related to something like this, when the Macbook turns on the screens again?

jungnissen (24 Sep 2020, 8:41):

I just saw this too today.
I have an external monitor, and the Macbook turned off the displays
(normal power saving stuff), but I have set it to not hibernate or
sleep the Macbook.
Could it be related to something like this, when the Macbook turns on
the screens again?

Hard to say.  I haven’t seen it happen, but then again there have been
a few NNW updates.  Restarting NNW has the side effect of this issue not
occurring.
It may be related to sleep/wake events.  As of march, my MacBook
hasn’t traveled much; the daily commute has disappeared.
Sorry, but I can’t corroborate or correlate at this moment.  I also
can’t rule it out.

I do keep on getting distorted text / icons in the middle bar. Current versions are NNW 6.1 (6106), MacOS 13.0.1, Mac Mini (M1). System uptime (and rough runtime for NNW) is 29 days, and the distortion does seem to get wider as time goes on???

Not sure if this is the same issue as before, as its a different part of the UI, but it seemed to be the nearest thing to note the issue.
It seems to be purely cosmetic, NNW isn't crashing or anything.

I bet this has to do with having scrollbars always visible.

I do have my system set to always show scrollbars.

fredoralive (25 Jan 2023, 9:56):

I do have my system set to always show scrollbars.

I, however, do not.  Then again, I haven’t seen the problem I reported (inter-feed spacing gradually increasing as time goes by) recur for a while now.


fix(build-ui-and-server): change the setup and build ui command

Notes for Reviewers
This PR fixes #
Signed commits

[x] Yes, I signed my commits.

we have a matching target for ui-provider-build, right?


SW-1803 Map accession CSV country names to codes

In the template CSV we don't specify that the user has to feed us valid ISO
country codes in upper case, and requiring users to do that would be needlessly
annoying. Update the CSV importer to accept country names and lower-case country
codes, and add a validation check to detect unrecognized countries.

Current dependencies on/for this PR:

main

PR #661

PR #662   👈

This comment was auto-generated by Graphite.

Graphite Merge Job
Current status: ⏳ Queued to merge
This pull request is currently queued to be merged as part of a stack.
This comment was auto-generated by Graphite.

Job Reference: jfFbzR15tOWaNawBoxGE


Refactor command.rs

[ ] Added an entry to CHANGELOG.md if this change could be valuable to users

This seems unnecessary

This seems unnecessary

the code is currently messy and hard to find around. Refactoring makes it easier for newcomers like myself to find a specific item in a long list, given that I haven't memorized the whole list

This seems unnecessary

the code is currently messy and hard to find around. Refactoring makes it easier for newcomers like myself to find a specific item in a long list, given that I haven't memorized the whole list

this just sorts everything and adds noise in git log, I don't see how that hinders ability to use search or go to symbol

Imagine having a dictionary where the words are just put there in random order. It would be kinda hard to get an overview of what's available when learning the codebase

but this is not a dictionary, it's a codebase, which you can easily ctrl+f or go by reference/symbol

but this is not a dictionary, it's a codebase, which you can easily ctrl+f or go by reference/symbol

if I know the name of the item, then I can use ctrl+F. But what if I'm exploring the list to see what's available?

Then you can read? Still it doesn't require everything to be sorted

Then you can read? Still it doesn't require everything to be sorted

yeah I can read, but I think that having it sorted makes it easier for me to navigate it. Let's suppose I'm looking for an item that begins with Update.... If it's sorted, then I have them all in one place instead of having to jump around to different parts of the code

this refactoring also sorts items by how many arguments they take in. Let's suppose I'm looking for an item that takes 3 arguments as input. I have them all in one place so that there are fewer items I need to go through

this just sorts everything and adds noise in git log,

the noise in git log can be solved by git rebase, where you merge this commit into one of its neighbors in the git log


No links in urlset

Hi!
After using your template generated sitemap has no body (generates only urlset without urls)
Is there any issue with current template? How can I resolve it?

Same here.

Hey guys, @fedojo @Jim-Zenn I know it was months ago but are you able to give more information?
When I generated a sitemap I got this output:
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url>
    <loc>http://yoursite.com/2017/10/27/hello-world/</loc>
    <lastmod>2017-10-28T00:39:58.000Z</lastmod>
  </url>
</urlset>

Closing due to inactivity, please re-open if it is still outstanding.


give operator-pending mode a rectangle cursor

As discussed in https://github.com/atom/vim-mode/pull/777

I like the hollow rectangle but Jack's original version has a thick-underline cursor:
.half-cursor(@visibility: visible) {
  border: none;
  border-bottom: 10px solid @syntax-cursor-color;
  background: none;
  visibility: @visibility;
  opacity: 0.5;
}

Happy to go with whatever people think looks good.

Now we just need an expecting-a-character cursor that can be used for f, t, and register commands.

Agreed, updated.

:clap: Nice.

Rebased to master.


IE11 has issue with checkbox with drop down multiselect

Sometimes the checkbox shows and sometimes it doesn't.
(https://codepen.io/) or JS Bin

Bug reports must include a live demo of the problem. Per our contributing guidelines, please create a reduced test case via CodePen/JS Bin or Stackblitz and report back with your link, Bootstrap version, and specific browser and OS details.
This is an automated reply

Closing due to lack of demo. Can reopen when we see one.


Small python 3 syntax improvement in docs

While browsing docs noticed in one doc small Python3 incompatibility. Switched to using print() as function call vs as statement

+1


basic example

I could use some help setting up a basic problem on a unit cube (or square).  I've been able to run the following but trying a forcing function of say all ones has been difficult.
h0 = 0.1 * np.ones(10)
grid = emg3d.utils.TensorMesh((h0,h0,h0), np.zeros(3))
model = emg3d.utils.Model(grid)
sfield = emg3d.utils.Field(grid)
efield = emg3d.solver.solver(grid, model, sfield)

A couple questions:

with model, it seems that the permitivity and permeability are both 1 (with 1/\sigma being a parameter).  Is it possible to set all of \eta, \mu_r, \omega, \mu_0 through this interface or is this intended to be a simpler model?
With the source field it appears that theses are edge values, but I can't seem to get a Field(fx,fy,fz) initiated.

emg3d.utils.TensorMesh is VERY basic; it is mainly used by the solver itself, but it can also be used to set up a grid. However, for fancier/easier set-ups it is easier to use discretize.TensorMesh (pip install discretize).  http://discretize.simpeg.xyz

emg3d.utils.Model - Model builder:

In the current stable version, the magnetic permeability is fixed to 1. On master (and hence from v0.6.3 onwards), you can set the magnetic permeability by providing emg3d.utils.Model(... mu_r). Only isotropic though.

You cannot set the eta_(x,y,z), but all of the resistivities, by providing emg3d.utils.Model(...res_x, res_y, res_z). Now, res_(x,y,z) can be floats, or arrays of dimension (nx, ny, nz)=grid.vnC, or even an array of length nx*ny*nz=grid.nC (Fortran-ordered).

You can provide the frequency with the freq parameter, which then will set omega (2pifreq).

mu_0 cannot be set, it is a physical constant.

Do you think it needs more in the documentation?
https://emg3d.readthedocs.io/en/latest/code.html#emg3d.utils.Model
Example:
h0 = 0.1 * np.ones(10)
grid = emg3d.utils.TensorMesh((h0,h0,h0), np.zeros(3))

res_x = 1
res_z = np.arange(grid.nC)+1
freq = 10
model = emg3d.utils.Model(grid, res_x=res_x, res_z=res_z, freq=freq)

This sets-up a model with frequency = 10 Hz. x-directed resistivity is 1 Ohm m, z-directed resistivity is increasing for each cell from 1 to number of cells Ohm m. As I did not provide res_y it assumes a VTI-model, hence same resistivity in x- and y-direction.
On top of the API-docs, the example might give some hints: https://github.com/empymod/emg3d-examples.
If you think I need to document that better I'll be happy to do so. Maybe add a getting-started notebook in the examples. However, I am off to holiday for 10 days, so I'll get back to it then.
Also, an easy getting-started example with some comments is given in the docs: https://emg3d.readthedocs.io/en/stable/usage.html

3. Source field
This sfield = emg3d.utils.Field(grid) actually gives you a Field(fx, fy, fz); try sfield.fx, sfield.fy, and sfield.fz.
However, to set-up a source field you need the routine
sfield = emg3d.utils.get_source_field(grid, src=[0, 0, 0, 0, 0], freq=freq)
or you can, as you did, initiate an empty field and then put your source wherever you want in that.

Let me know if these helps, and if you think I need more on top of:

https://emg3d.readthedocs.io/en/stable/usage.html
https://github.com/empymod/emg3d-examples
https://emg3d.readthedocs.io/en/latest/code.html#emg3d.utils.Model
https://emg3d.readthedocs.io/en/latest/code.html#emg3d.utils.Field
https://emg3d.readthedocs.io/en/latest/code.html#emg3d.utils.get_source_field
https://emg3d.readthedocs.io/en/latest/code.html#emg3d.utils.TensorMesh

The only thing you would have to change in your example is
sfield = emg3d.utils.get_source_field(grid, src=[0.5, 0.5, 0.5, 0, 0], freq=1)
This sets up a x-directed (azimuth=0, dip=0) source at location (0.5, 0.5, 0.5); frequency=1 (same as you used to set-up the model).
Then
In [9]: efield = emg3d.solver.solver(grid, model, sfield)

:: emg3d START :: 22:03:58 ::

   MG-cycle       : 'F'                 sslsolver : False
   semicoarsening : False [0]           tol       : 1e-06
   linerelaxation : False [0]           maxit     : 50
   nu_{i,1,c,2}   : 0, 2, 1, 2          verb      : 2
   Original grid  :  10 x  10 x  10     => 1,000 cells
   Coarsest grid  :   5 x   5 x   5     => 125 cells
   Coarsest level :   1 ;   1 ;   1

   > CONVERGED
   > MG cycles      : 16
   > Final l2-norm  : 5.012e-12

:: emg3d END   :: 22:03:58 :: runtime = 0:00:00

You get some additional goodies if you initiate your grid with discretize instead of emg3d:
grid = discretize.TensorMesh((h0,h0,h0), np.zeros(3))
Then you can plot your model and the resulting field:
grid.plot_3d_slicer(efield.fx.ravel('F'), view='abs', vType='Ex')
which results in

which is an interactive figure, depending on your backend, so you can slice through it.
Similarly, you could plot the model parameters:
grid.plot_3d_slicer(model.res_x)
Now, these are discretize-features, not emg3d-features, but they come in handy.

Needs %matplotlib notebook in a notebook, in IPython it should work out of the box.

Hi @lukeolson. I am back from holiday and catching up. Did my comments answer your initial questions?

Yep, this helps.  One suggestion is a to provide a very basic example with some of these comments.  I used
import emg3d
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

h0 = 0.1 * np.ones(10)
grid = emg3d.utils.TensorMesh((h0,h0,h0), np.zeros(3))

res_x = 1
res_z = np.arange(grid.nC)+1
freq = 10
model = emg3d.utils.Model(grid, res_x=res_x, res_z=res_z, freq=freq)
sfield = emg3d.utils.get_source_field(grid, src=[0.5, 0.5, 0.5, 0, 0], freq=1)

efield = emg3d.solver.solver(grid, model, sfield)

One question from this: why is the grid 10x10x10 in this case, but the field variables (e.g. Ex) 10x11x11?

@lukeolson:
With regards to your question:
The grid is 10 x 10 x 10 cells. It is a Yee-grid, so the electric fields are located on the edges: See https://emg3d.readthedocs.io/en/stable/theory.html#muld06-fig1
That means that

Ex has dimension 10x11x11  (cells x edges x edges)
Ey has dimension 11x10x11  (edges x cells x edges)
Ez has dimension 11x11x10  (edges x edges x cells)

Does that make sense?
I close this issue then. There is a very similar basic example given in https://emg3d.readthedocs.io/en/stable/usage.html#example

Ah -- a Yee-grid.  Yep that makes sense.

I expanded a bit on my minimum example, and also created a notebook out of it, so it should be easier to get started:
https://github.com/empymod/emg3d-examples/blob/master/0a_Minimum_working_example.ipynb


[TheForest] Oxide events fails at user login in this order: IOnUserApprove, OnPlayerConnected, OnPlayerSpawn

Step 2: Describe your setup

Oxide version ("latest" is not a version): v2.0.3345
Game name and version: TheForestDedicatedServerv0.62 and TheForest client v0.62
OS version: Windows2012Server( server) Windows10 ( client )

Step 3: Describe the problem
Just want to test last changes...
Steps to reproduce:

Download https://github.com/OxideMod/Oxide/archive/2.0.3345.zip
Open With Visual Studio and compile
Copy bundles/TheForest to Server folder
Start Server

Observed results:
Oxide events fails at user login in this order: IOnUserApprove, OnPlayerConnected, OnPlayerSpawn
Starting dedicated server
Dedicated server info:
 IP:<IP_ADDRESS>, steamPort:8766, gamePort:27015, queryPort:27016
 players:4, admin password:'yes', password:'no', autosave interval:30
Game setup: Standard Multiplayer Server, Continue Normal game, slot Slot1
Skipping Steam initialization
Setting breakpad minidump AppID = 242760
Set a Logon
Loading Oxide Core v2.0.0...
Loading extensions...
Loaded extension CSharp v2.0.0 by Oxide Team
Loaded extension MySql v2.0.0 by Oxide Team
Loaded extension SQLite v2.0.0 by Oxide Team
Loaded extension Unity v2.0.0 by Oxide Team
Loaded extension TheForest v2.0.0 by Oxide Team
Using Covalence provider for game 'The Forest'
[Rcon] Server started successfully on port 25580
Loading plugins...
Loaded plugin Unity v2.0.0 by Oxide Team
Loaded plugin The Forest v2.0.0 by Oxide Team
Connected to Steam successfully
The Forest Server is VAC Secure!
Game server SteamID:XXXXXXXXX(ofusced ID)
Will force game to new because GameSetup.IsSavedGameTrue LevelSerializer.CanResume False
CanResume:False, BoltNetwork:True, isServer:True, StartGameSetup:New
****** Game Activation Sequence ******
Dedicated Server Running
Address: [EndPoint <IP_ADDRESS>:27015 | 27015]
Max Players: 4LO.com
Save Interval: 30 minutes
Game autosave started
Failed to call hook 'IOnUserApprove' on plugin 'TheForestCore v2.0.0' (NullReferenceException: Object reference not set to an instance of an object)
  at Oxide.Game.TheForest.Libraries.Covalence.TheForestPlayerManager.PlayerJoin (UInt64 userId, System.String name) [0x00000] in <filename unknown>:0
  at Oxide.Game.TheForest.TheForestCore.IOnUserApprove (.BoltConnection connection) [0x00000] in <filename unknown>:0
  at (wrapper managed-to-native) System.Reflection.MonoMethod:InternalInvoke (object,object[],System.Exception&)
  at System.Reflection.MonoMethod.Invoke (System.Object obj, BindingFlags invokeAttr, System.Reflection.Binder binder, System.Object[] parameters, System.Globalization.CultureInfo culture) [0x00000] in <filename unknown>:0
Connection:1
Steam auth - clientId<PHONE_NUMBER>6224106 status True
ClientApproved<PHONE_NUMBER>6224106
13882433816594799366/Netzulo joined
Failed to call hook 'OnPlayerConnected' on plugin 'TheForestCore v2.0.0' (NullReferenceException: Object reference not set to an instance of an object)
  at Oxide.Game.TheForest.Libraries.Covalence.TheForestPlayer..ctor (.BoltEntity entity) [0x00000] in <filename unknown>:0
  at Oxide.Game.TheForest.Libraries.Covalence.TheForestPlayerManager.PlayerConnected (.BoltEntity entity) [0x00000] in <filename unknown>:0
  at Oxide.Game.TheForest.TheForestCore.OnPlayerConnected (.BoltEntity entity) [0x00000] in <filename unknown>:0
  at (wrapper managed-to-native) System.Reflection.MonoMethod:InternalInvoke (object,object[],System.Exception&)
  at System.Reflection.MonoMethod.Invoke (System.Object obj, BindingFlags invokeAttr, System.Reflection.Binder binder, System.Object[] parameters, System.Globalization.CultureInfo culture) [0x00000] in <filename unknown>:0
New player join dedicated server. Total 1 player(s).
Failed to call hook 'OnPlayerSpawn' on plugin 'TheForestCore v2.0.0' (NullReferenceException: Object reference not set to an instance of an object)
  at Oxide.Game.TheForest.Libraries.Covalence.TheForestPlayerManager.FindPlayerById (System.String id) [0x00000] in <filename unknown>:0
  at Oxide.Game.TheForest.TheForestCore.OnPlayerSpawn (.BoltEntity entity) [0x00000] in <filename unknown>:0
  at (wrapper managed-to-native) System.Reflection.MonoMethod:InternalInvoke (object,object[],System.Exception&)
  at System.Reflection.MonoMethod.Invoke (System.Object obj, BindingFlags invokeAttr, System.Reflection.Binder binder, System.Object[] parameters, System.Globalization.CultureInfo culture) [0x00000] in <filename unknown>:0
24/7[EUR]NETZULO.com                                               219fps, 3m16s 1/4 players 7kb/s in, 462b/s out
Slot 1[Normal]                                                     Oxide 2.0.3345 for <IP_ADDRESS>.0

Expected results:

What did you expect to happen?

Login user and events to notify covalence work
Start server
Relevant code (if applicable):
Adding this before this lane , make PlayerJoin works but fail on another site : https://github.com/OxideMod/Oxide/blob/develop/Games/Unity/Oxide.Game.TheForest/Libraries/Covalence/TheForestPlayerManager.cs#L41
            playerData = ProtoStorage.Load<Dictionary<string, PlayerRecord>>("oxide.covalence") ?? new Dictionary<string, PlayerRecord>();
            if (data)
            {
                Debug.LogWarning($"PlayerJoin, record select: update");
                record.Name = name;
                playerData[id] = record;
                allPlayers.Remove(id);
                allPlayers.Add(id, new TheForestPlayer(userId, name));
            }

Step 4: Additional information

Maybe related #1137

Commits related :

355f304dd58691bdada663886f257a05b66b0c0a
6ec1e51b34533e2785e0aac28ff1c8c19708d33c
f0a7272adb5e5350d0087d591082a78d26e55aff
a2087feaf8ecba48ed14bd3cf72e7958449c93c3

if modify this, PlayerManager it's working ^^, but still fail event OnPlayerConnected

 public IPlayerManager CreatePlayerManager()
        {
            PlayerManager = new TheForestPlayerManager();
            PlayerManager.Initialize();
            return PlayerManager;
        }

Closed ! #1142

Reopen, until new release , waiting for mannually changes , #1142

Fixed locally. :P

ok ok , but submit changes please, and commands please... are totally broken from server console.. BoltConnection connection param from server console always null :)))))))))))))))))

Yeah, might have to change OnServerCommand to not include the BoltConnection.

pleaseeeeeee!!!!!