Passport Login in Botkit

I'm on my way to implement User Login (fetch user's info) to Botkit, using Passport and in my case Facebook Passport.
To keep a decent UX the challenge is to have everything happens with a single tap/click on the Login Button, which is what this code does. The beginning of the code works fine, I can access to the Facebook auth dialog by just clicking the login button and save some info in the database, but I get a Timeout error with the redirection to auth/facebook/test/callback.
I think the problem lies with some specs in Botkit or my implementation of the code in it.
If we can fix this that'd be a cool core feature for Botkit. Anyone knows some Passport?
facebook_bot.js
var passport = require('passport');
var FacebookStrategy = require('passport-facebook').Strategy;

// The problem is probably there, it could bethe function not resolving?
function testFacebookAuthCallback(req, token, refreshToken, profile, done) {
        var providerUserProfile = {
              firstName    : profile.name.givenName,
              lastName     : profile.name.familyName,
            };
             responseStore.toDatabase(profile.id, providerUserProfile ,function(user){
      if(err) done(err);
      if(user){
        done(null,user);
      } else {
        done(null);
      }
    });
  }

const testFacebookOptions = {
  clientID: process.env.app_id,
  clientSecret: process.env.app_secret,
  callbackURL: 'https://myherokuapp.com/auth/facebook/test/callback',
  passReqToCallback: true,
  state: true,
  enableProof: false
};

passport.use('facebook', new FacebookStrategy(testFacebookOptions, testFacebookAuthCallback));

passport.serializeUser((user, done) => {
  done(null, user.id);
});

passport.deserializeUser((serializedUser, done) => {
  done(null, obj);
});

//...//

convo.say({
		attachment: {
	      type: 'template',
	      payload: {
	        template_type: 'button',
	        text: 'Login',
	        buttons: [
	          {
	            type: 'web_url',
	            title: 'Log in with Facebook',
	            url: `https://myherokuapp.com/auth/facebook/${senderId}/?pageId=${pageId}`
	          }
	        ]
	      }
	  }
  });

responseStore.js  (mongodb)
var mongodbStorage = require('botkit-storage-mongoose')({mongoUri: process.env.MONGODB_URI});

toDatabase: function(key, value, response) {

            controller.storage.users.get(response.user, function(err, user) {
            if (!user) {
            user = {
                id: response.user,
            };
        }

        user[key] = value;
        controller.storage.users.save(user, function(err, id) {});
    });
}
};

facebook.js
var passport = require('passport');
var cookieParser = require('cookie-parser');
const path = require('path');

facebook_botkit.createWebhookEndpoints = function(webserver, bot, cb) {

// ...//

webserver.get('/auth/facebook/test/:senderId', (req, res, next) => {
  req.session.sender = req.params.senderId;
  req.session.pageId = req.query.pageId;
  passport.authenticate('facebook', {scope: ['email', 'user_friends']}, {
      state: {
        sender: req.params.senderId,
        pageId: req.query.pageId
      }
  })(req, res, next);
  console.log('oh yeah');
});

webserver.get('/auth/facebook/test/callback',
  passport.authenticate('facebook'), function(req, res, next) {
  const ua = req.headers['user-agent'].toLowerCase(),
    isMobile = // here is a regex to detect mobile
  req.mobile = !!isMobile;
  next();}, (req, res) => {
  if (!req.user) {
    return res.send('Something gone wrong. Try again.');
  }
  if (req.mobile) {
    res.redirect(`http://m.me/${req.session.pageId}`); / redirect to messenger for mobile users
  } else {
    res.redirect('/closeWindow'); // will close the login window for Desktop users
  }
});

webserver.get('/closeWindow', (req, res) => {
  res.sendFile(path.resolve('static/closeWindow.html'));
});

//...//

facebook_botkit.setupWebserver = function(port, cb) { //...//
        facebook_botkit.webserver.use(bodyParser.json());
        facebook_botkit.webserver.use(bodyParser.urlencoded({ extended: true }));
        facebook_botkit.webserver.use(express.static(static_dir));
        facebook_botkit.webserver.use(cookieParser());
        facebook_botkit.webserver.use(session({ secret: 'whatever', resave: true, saveUninitialized: true}));
        facebook_botkit.webserver.use(passport.initialize());
        facebook_botkit.webserver.use(passport.session());

Did you have any progress on this? Started working on something similar but I'm using another bot framework and got to the same conclusion would have to build an express + passport backend that does the auth.
On the botside would probably open this page in a webview ? have the user login with facebook and on window close I think it's possible to pass back the data to the bot using the messenger extension https://developers.facebook.com/docs/messenger-platform/webview/extensions ?
http://espeo.eu/blog/user-permissions-messenger-bots/

Hi @cAstraea I 'm still struggling on some part of it but the following code will work, except it will redirect to somePage even in case oflogin failure. I haven't been working this out yet.
This code also lets you send a message to the user once he is logged in, by sending the user id as a parameter in the auth url.
I code directly in facebook.js so you ll need your own fork of botkit :(
facebook_bot.js
passport.use(new FacebookStrategy({
    clientID: process.env.app_id,
    clientSecret: process.env.app_secret,
    callbackURL: process.env.app_host + '/auth/fb/callback',
    profileFields: ['id', 'displayName', 'photos', 'email'],
    passReqToCallback : true,
    state: true,
    enableProof: false
  },
  function(req, accessToken, refreshToken, profile, done) {
    profile._json.access_token = accessToken;
    profile._json.refresh_token = refreshToken;
    var save = {
      ['facebook_profile'] : profile
    };
    responseStore.toDatabase(save, senderId); // This is my function to save user in Mongo (NO Mongoose)
    done(null,profile);
}));

// Not using these yet
passport.serializeUser(function(user, done) {
  done(null, user.id);
});

passport.deserializeUser(function(obj, done) {
  done(null, obj);
});

// within a convo
facebookLogin = function(response, convo) {
  convo.say({
    attachment: {
      type: 'template',
      payload: {
        template_type: 'button',
        text: 'Please login',
        buttons: [
          {
            type: 'web_url',
            title: 'Log in with Facebook',
            url: process.env.app_host + '/auth/fb/?senderId=' + senderId, // send senderId to Express server to send a message to user at auth completion.
            webview_height_ratio: 'full',
            messenger_extensions: true,
          }
        ]
      }
    }
  });
};

facebook.js
var passport = require('passport');
var cookieParser = require('cookie-parser');
var session = require('express-session');
var ejs = require('ejs');

**within the function facebook_botkit.createWebhookEndpoints = function(webserver, bot, cb) { **
        // Facebook login dialog, specifies scope
        webserver.get('/auth/fb',  (req, res, next) => {
            req.session.senderId = req.query.senderId;
            passport.authenticate('facebook',{ scope: ['user_friends', 'email']},
            {
                state: {
                    senderId: req.query.senderId // senderId will be used after auth to reply to the user
                }
            })(req, res, next);
        });

        // Redirection after login
        webserver.get('/auth/fb/callback',

                passport.authenticate('facebook', {
                    successRedirect : '/somePage', // The webview I want to open if user logs in
                    failureRedirect : '/https://m.me/' + process.env.app_name // redirect to Messenger if failure
                }));

        //  The webview opens and triggers the bot to send a message / start a new convo
        webserver.get('/somePage', isLoggedIn, function(req, res) {
            res.render('somePage.ejs',{
                    user : req.user
                });
            req.on("end", function() {
                console.log(req.session.senderId);
                var redirect = {
                  ['user'] : req.session.senderId,
                  ['channel'] : req.session.senderId
                };
                console.log('Redirecting to user ' + redirect.user);
                bot.startConversation(redirect, someNewConvo);
            });
        });

** within the function facebook_botkit.setupWebserver = function(port, cb) { **
        facebook_botkit.webserver.use(bodyParser.json());
        facebook_botkit.webserver.use(bodyParser.urlencoded({ extended: true }));
        facebook_botkit.webserver.use(express.static(static_dir));
        facebook_botkit.webserver.use(cookieParser());
        facebook_botkit.webserver.use(session({ secret: 'its_my_secret', resave: true, saveUninitialized: true}));
        facebook_botkit.webserver.use(passport.initialize());
        facebook_botkit.webserver.use(passport.session());

I see thanks for your input. I would need a way to merge the 2 processes :(
http://recordit.co/osdMl0MUCL ==> Managed to get it working with 2 login processes but sadly the success redirect doesn't return to the existing window but redirects in the popup :(
Is there no way to do the passport.authenticate inside the link route ? Been trying all night with no success.  This is what I have currently.
https://gist.github.com/cAstraea/95712209f943680393f780a10bf8fc5e
Anyone can take a guess ? Tried to move the passport.authenticate inside app.get('/authorize but don't know how to handle the  const redirectURISuccess = ${redirectURI}&authorization_code=${authCode};
res.redirect(redirectURISuccess);
I'm thinking would need to do this inside the app.get(/auth/fb/callback' ... but don't know how ..

Yay got it to work I hope :-) http://recordit.co/a3CGBcGFPX

Hi @cAstraea can you please share your final solution i have similar problem, thanks

Hello.
I used another framework (botly) but the same thought process applies all the work is done in the app.get routes.
https://gist.github.com/cAstraea/fbf1f8012bc0712c1d1078451ca1a7c0
the callbackURL in passport auth is the /auth/fb/callback
you need to add this to oauth whitelist in the facebook developer section

Valid OAuth redirect URIs

Great thanks for sharing 👍

@cAstraea Can you elaborate on the user interaction for account linking you are using?
I'm building something for slack which has different concerns, but am curious what the user experience you created looks like.

Hello. The user would be in contact with a third party q&a system where he can request help from a live person if the answer received doesn't fit. I'm thinking it would be used as a way of onboarding users and also provide customized recommendations based on their likes and whatnot.


Where is the possumbot folder

I cant find the possumbot folder I need help

i don't know, maybe its on your harddrive?


kv_client_table_test should init KVClientTable with type double

In the base/abstract_partition_manager.hpp, KVPairs was defined as std::pair<third_party::SArray<Key>, third_party::SArray<double>>, where the value is a SArray contains double type values.
In the worker/kv_client_table_test.cpp, the Add test init KVClientTable with type float, during the runtime, when kv_client_table try to call partition_manager_->Slice(std::make_pair(keys, vals), &sliced);, it will cause error because there will be a conflit between the float and double type value.

Personally, I think simply change the float to double while initialize kv_client_table will be better,  it will not cause lots of change in the project, and enough to satisfy the current training requirement.
In the future when kv_client_table need support different type of values, we can change the KVPairs to a general pair type.
I will send a pull request if you agree too.

Sounds reasonable. Please do so!


Update README with more info regarding MSR types

Includes some notes to help parse what each type is for, as well as a little info on them that I have knowledge of.

Made revisions based on feedback.


Reverting to standard behavior when touchID is not available

Once rekey is setup, I can no longer add regular keys to my ssh-agent. This is a problem when using a laptop in clamshell mode, or when remotely connecting to a machine through VNC, for instance.
MacBookGray:~ $ ssh-add ~/.ssh/id_rsa
Enter passphrase for /Users/blaise/.ssh/id_rsa:
Could not add identity "/Users/blaise/.ssh/id_rsa": agent refused operation

Any suggestion?

i will check this tomorrow and give you a solution. for sure this is failing because is trying to add the key to the sekey agent, and should add to the normal ssh. maybe is a way to return to the normal ssh client to refuse to add and add to the normal one. i been using multiple keys with id_rsa on my user path and worked buy i never added with the ssh-add.

I'm running into the same issue. What is the status on this? 😄

I have the same issue on macos with brew install would be nice when i have to pull from a ton of repos to just ssh-add - great job on the software though its fab!


Implement Chat GetPermalink

Reference: https://api.slack.com/methods/chat.getPermalink
Acceptance Criteria:

[x] Spring Slack Client is extended with given method
[x] Spring Slack Test Client is extended with given method

closed by #279


Fixed the Calendar Issue #314

this fixed issue #314
Removed the addition to the months, as it was not being converted correctly. It now displays the calendar 100% correctly - (even with the trophies, that's a neat feature PoV!)

Yes, this fixes Chrome. Duh, somehow I didn't even notice this was broken.


net/http: ResponseWriter doesn't overwrite content-type after WriteHeader is called.

Please answer these questions before submitting your issue. Thanks!
What version of Go are you using (go version)?
1.7
What operating system and processor architecture are you using (go env)?
ubuntu, amd64
What did you do?
in a http server handler,
first call w.WriteHeader(http.StatusOK)
then call w.Header().Set("Content-Type", "application/json")
then w.Write(somedata compressed by gzip)
What did you expect to see?
The Content-Type in response message captured by troubleshooting tool of chrome/firefox becomes application/x-gzip
What did you see instead?
The Content-Type should be application/json as I set.
On the other hand, if I change the order. call w.Header().Set("Content-Type", "application/json") ahead of the w.WriteHeader(http.StatusOK), the Content-Type keeps what it is. Seems setting status ahead of write content-type trigger the logic to DetectContentType automatically. It's overwrite whatever content-type I set later.
Because of that, gzip content can't be parsed in firefox. Of course, I worked around it by changing the code to set content-type first. But it apparently a trap since there is no doc saying you should always set content-type before set status by WriteHeader.

@GordonJiang this behavior is documented in https://golang.org/pkg/net/http/#ResponseWriter

as the documentation for property Header().
Closing this issue unless someone else thinks we should do more.
/cc @bradfitz.

Working as intended.
WriteHeader is supposed to write (all) the headers, so when it's called, it
will write the current Content-Type. Later update to the Header map won't
affect header that has already been written.


React-native-swiper overlaying images from sides, Looks dirty

I am using react-native-swiper library in react-native-snap-carousel, but native-swiper showing me overlaying image from side, and looks so bad, i have tried, but unable to do.
see screenshot:

Here is the code:
SliderEntry.js
render() {

        return (
            <View style={styles.slideInnerContainer} >
                <View>
                        <Swiper
                            width={300}
                            showsButtons
                        >
                            <View>
                                <Image
                                  source={{ uri:
                                  'http://primecourierservices.co.uk/wp-
                                   content/uploads/2018/11/van-courier-
                                   service-london-uk.jpg' }}
                                   style={{width: 330, height: 250
                                  }}
                                />
                            </View>
                            <View>
                                <Image
                                  source={{ uri:
                                  'http://primecourierservices.co.uk/wp-
                                   content/uploads/2018/11/van-courier-
                                   service-london-uk.jpg' }}
                                   style={{width: 330, height: 250
                                  }}
                                />
                            </View>
                            <View>
                                <Image
                                  source={{ uri:
                                  'http://primecourierservices.co.uk/wp-
                                   content/uploads/2018/11/van-courier-
                                   service-london-uk.jpg' }}
                                   style={{width: 330, height: 250
                                  }}
                                />
                            </View>
                        </Swiper>
                </View>
                <View>
                    <Text style={{marginBottom: 10}}>{uppercaseTitle}</Text>
                    <View style={{marginLeft: 19, marginTop: 5, marginBottom: 6}}>
                        <Text>
                            {van}
                        </Text>
                    </View>
                </View>
            </View>
        );
    }

Can anyone help me with this? Please

This repo dead: https://github.com/leecade/react-native-swiper/issues/713

This repo dead: #713

What u mean?

@zainshabir00 nobody can solve your problem

but why?

@zainshabir00 nobody can solve your problem

it seems your swiper width is less than the image width. 300 and 330.

@zainshabir00 the repo has 500 open issues
it mean that when installing the library, you inherit 500 issues from the maintainer
it will take us less to build the functionality from scratch

@zainshabir00, u can use my fork
"react-native-swiper": "git+https://github.com/ARDcode/react-native-swiper.git",


Add hbelmiro as member to kubeflow org

Resolve: https://github.com/kubeflow/internal-acls/issues/654
Contributions

Kubeflow 1.9 Release Team member
https://github.com/kubeflow/website/pull/3681
https://github.com/kubeflow/pipelines/pull/10459
https://github.com/kubeflow/examples/pull/1063

pytest output:
github-orgs git:(add-hbelmiro) pytest test_org_yaml.py
============================================================================================================ test session starts ============================================================================================================
platform linux -- Python 3.12.1, pytest-7.3.2, pluggy-1.2.0
rootdir: /home/hbelmiro/dev/kubeflow/kubeflow-internal-acls/github-orgs
collected 1 item

test_org_yaml.py .                                                                                [100%]

============================================================================================================= 1 passed in 0.17s =============================================================================================================

/lgtm
Helber has been a good contributor not only for pipelines but for Kubeflow release as well. He volunteered to work as Docs lead.

/assign @terrytangyuan


Document ID extraction

Need to update document id extraction and downloadable link generation.

Function updated.
Now it's use Regular Expression and have unstable situation check (e.g., ID extraction fails).


Run notebooks and upload them

We should run all of the notebooks and commit them ran. Github has this (new?) super nice feature of showing notebooks and if it's ran it shows the output. It's much better than just showing the code.

Done.


CALCITE-1550: make WeakHashMap thread safe (synchronized).

It looks like this is the same issue reported in https://issues.apache.org/jira/browse/CALCITE-1594 and has been fixed in the Calcite master branch.  Can you try with cherry-picking that fix ?  I suppose the repro requires running the concurrency tests on the perf cluster (or were you able to repro on your VM ?).  If CALCITE-1594 helps resolve the issue, you can mark CALCITE-1550 closed as duplicate.


download-product generated config for assign-stemcell does not always match the slug in Ops Manager

The config that is generated takes the product slug given for download-product
The config that is needed for assign-stemcell needs the tile slug, not the pivnet slug.
This could be solved by grabbing the product name from the downloaded tile (like we do the stemcell) rather than inferring it from the download-product config

This will be released in the next version of om (Currently: 1.0.1. Potentially 1.1.0 depending on if new features are added -- see Changelog for more details)


⚠ Bofrost.be has degraded performance

In b1c9c98, Bofrost.be (https://www.bofrost.be) experienced degraded performance:

HTTP code: 200
Response time: 2883 ms

Resolved: Bofrost.be performance has improved in 347f632.


I wonder if i can configure specific setting in Blueprint api

Ive found that using Blueprint API is very convenient way to develop sails server
However, if i set "rest" to true in config/blueprint.js, i should block some Restful api i didn't develop  by using config/policy.js.
If i coud set Resful API to only enable the route api i defined,  it would be very convenient
For example, if i make User model and define "find" and "create" actions in UserController.js , only POST and GET routes should be enabled
Are there any way i mentioned above ?
If there are not, do you have any plan to develop that?

You can do this with the sails routes config: http://sailsjs.org/#!/documentation/concepts/Routes


Add Black beautifier for Python

It's a feature request. Could Black be supported in the future?

It would be great to add Black support!
Please feel free to submit a Pull Request to Atom-Beautify. See https://github.com/Glavin001/atom-beautify/blob/master/docs/add-languages-and-beautifiers.md#how-to-add-a-beautifier-for-a-language for details and please ask for help if you need any.
Unfortunately, bandwidth is low right now for Atom-Beautify. This is because @Unibeautify -- a complete rewrite of Atom-Beautify -- is my top priority. @szeck87 has been a great help keeping an eye on important Atom-Beautify issues.
I've already created https://github.com/Unibeautify/beautifier-black as Black support for Unibeautify would be great! Please let me know if you'd like to contribute to Atom-Beautify or Unibeautify or both 😃. Thanks in advance!


Add staged=true to release_notes endpoint docs

Looks like I found how the app is requesting a newer version's release notes!
Made the necessary modifications to @core-hacked's already contributed docs.
Here's a response from a request with staged=true appended:
{
    "response": {
        "release_notes": [
            {
                "title": "Camera-Based Detection",
                "subtitle": "Sentry Mode Improvements",
                "description": "When Camera-Based Detection is enabled, Sentry Mode will use the vehicle’s external cameras in addition to vehicle sensors to detect a security event while parked. If disabled, the vehicle will only save clips to the USB drive if a physical threat is detected. To adjust, tap Controls > Safety > Sentry Mode > Camera-Based Detection.",
                "customer_version": "2022.44.2",
                "image_url": "https://vehicle-files.teslamotors.com/release_notes/{id}?__gda__=exp={unix_timestamp}~acl=/release_notes/{id}~hmac={id}"
            },
            {
                "title": "New Language Support",
                "subtitle": "User Interface Improvements",
                "description": "You can now select Thai as your touchscreen language. To update your language settings, tap Controls > Display and select your preferred language from the ‘Touchscreen Language’ drop-down menu.",
                "customer_version": "2022.44",
                "image_url": "https://vehicle-files.teslamotors.com/release_notes/{id}?__gda__=exp={unix_timestamp}~acl=/release_notes/{id}~hmac={id}"
            }
        ],
        "deployed_version": "2<IP_ADDRESS>",
        "staged_version": "2022.44.2"
    }
}

Here's a slimmed-down version of the same request but WITHOUT staged=true:
{
    "response": {
        "release_notes": [
            {
                "title": "Sentry Mode and Dog Mode",
                "subtitle": "Miscellaneous Improvements",
                "description": "Your vehicle and pets can be kept safe by activating Dog Mode and Sentry Mode at the same time. Sentry Mode alarms are silenced if Dog Mode and Sentry Mode are both enabled. Mobile app notifications will still be sent and video footage will still be saved.",
                "customer_version": "2022.40",
                "image_url": "https://vehicle-files.teslamotors.com/release_notes/{id}?__gda__=exp={unix_timestamp}~acl=/release_notes/{id}~hmac={id}"
            },
            {
                "title": "Improved Charging Efficiency",
                "subtitle": "Charging Improvements",
                "description": "During DC fast charging, the thermal system controls have been optimized for each charging station’s power capability, improving both charging and on-route battery preconditioning efficiency. This results in faster charge times and reduced energy costs.",
                "customer_version": "2022.40",
                "image_url": "https://vehicle-files.teslamotors.com/release_notes/{id}?__gda__=exp={unix_timestamp}~acl=/release_notes/{id}~hmac={id}"
            },
            ...there was more here.
        ],
        "deployed_version": "2<IP_ADDRESS>",
        "staged_version": null
    }
}

Thanks!

@SirArkimedes
Finally. I tried every way possible before doing my PR. I was so close too haha. I did staging=true.


Omnibar: Pixel and Clear action regressions

Task/Issue URL: https://app.asana.com/0/1205782002757341/1208820359395089/f
Description
Steps to test this PR
Looks at this task for different scenarios: https://app.asana.com/0/1174433894299346/1208812333027437/f
Pixel for clearing address bar - Site

[x] Navigate to a site
[x] Tap into address bar
[x] X to delete entry (don’t type anything new)
[x] Swipe gesture to close keyboard
[x] Verify pixel m_addressbar_focus_clear_entry_website is fired only once

Pixel for clearing address bar - SERP

[x] Navigate to a site
[x] Tap into address bar
[x] X to delete entry (don’t type anything new)
[x] Swipe gesture to close keyboard
[x] Verify pixel m_addressbar_focus_clear_entry_serp  is fired only once

Pixel for clearing address bar - NTP

[x] Navigate to a site
[x] Tap into address bar
[x] X to delete entry (don’t type anything new)
[x] Swipe gesture to close keyboard
[x] Verify pixel m_addressbar_focus_clear_entry_ntp  is fired only once

Swipe gesture to close keyboard - Site

[x] Navigate to a site
[x] Tap into address bar
[x] X to delete entry (don’t type anything new)
[x] Swipe gesture to close keyboard
[x] Verify Site url is displayed

Swipe gesture to close keyboard - SERP

[x] Enter a search query on SERP
[x] Tap into address bar
[x] X to delete entry (don’t type anything new)
[x] Swipe gesture to close keyboard
[x] Verify query is displayed

Manually close keyboard  + Scroll- Site

[x] Navigate to a site
[x] Tap into address bar
[x] X to delete entry (don’t type anything new)
[x] Manually close keyboard
[x] Scroll down the site
[x] Verify Site url is displayed

Manually close keyboard  + Scroll- SERP

[x] Enter a search query on SERP
[x] Tap into address bar
[x] X to delete entry (don’t type anything new)
[x] Manually close keyboard
[x] Scroll down the site
[x] Verify query is displayed

Close keyboard - Site

[x] Ensure 3 button navigation is enabled (via device Settings)
[x] Navigate to a site
[x] Tap into address bar
[x] X to delete entry (don’t type anything new)
[x] Close keyboard (press back)
[x] Verify Site url is displayed

Close keyboard - SERP

[x] Ensure 3 button navigation is enabled (via device Settings)
[x] Enter a search query on SERP
[x] Tap into address bar
[x] X to delete entry (don’t type anything new)
[x] Close keyboard (press back)
[x] Verify query is displayed

@malmstein While testing, I noticed the instructions mention “Swipe gesture to close keyboard”, but that’s not how the keyboard is hidden. It’s actually hidden as soon as you touch the screen. The swiping then navigates back to the previous site. I assume that’s not what you want, because the Swipe gesture to close keyboard - Site test might not show the URL, for example. Right?

Thanks for the review @0nko !


feat(): Add ngRedux and created common store directories

DAS-4096

@andrewjensen, think this is ready?

@sldavidson other than the comment in the constants file, everything looks good! Feel free to merge after you make that tweak.


Set cors headers for vtiles

Companion to #1215.
This is intended to allow testing vtiles produced by Studio locally against mapbox-gl-js styles like:
{
  "sources": {
    "sourcename": {
      "type": "vector",
      "maxzoom": 14,
      "tiles": ["http://localhost:3001/source/{z}/{x}/{y}.vector.pbf?id=tmsource:///path/to/sourcename.tm2source"]
     }
  }
  [ .. snip ... ]
}}

@yhahn question: I already see cors() used in the routes for the tile requests (https://github.com/mapbox/mapbox-studio/blob/mb-pages/lib/server.js#L110-L116). Could it be that those are not working as expected or are not related to providing cors headers?

Hrm yes, maybe a candidate for http request/response testing territory : )

Could it be that those are not working as expected or are they not at all related to providing cors headers?

Got to the bottom of this: the cors() middleware works on successful requests but on 404's it never kicks in which makes it look like it is not working while debugging.


[RESOLVED] Interface eth0(): High error rate (>2 for 5m)

Problem started at 12:21:29 on 2023.06.02
Problem name: Interface eth0(): High error rate (>2 for 5m)
Host: S100-Primario
Severity: Warning
Operational data: errors in: 3, errors out: 0
Original problem ID: 23135
Event details in Zabbix: http://<IP_ADDRESS>/tr_events.php?triggerid=23483&eventid=23135

Problem has been resolved in 6m 0s at 12:27:29 on 2023.06.02
Problem name: Interface eth0(): High error rate (>2 for 5m)
Host: S100-Primario
Severity: Warning
Original problem ID: 23135


feat(@angular/cli): allow code coverage excludes

A new CLI config entry under test allows you to list exclude globs for code coverage:
  "test": {
    "codeCoverage": {
      "exclude": [
        "src/polyfills.ts",
        "**/test.ts"
      ]
    },
    "karma": {
      "config": "./karma.conf.js"
    }
  },

/cc @delasteve

Am I correct in thinking that this is no longer possible with the latest version 6.0.0 of angular-cli?
I've tried adding a codeCoverage block to angular.json, and get an error about that being invalid according to the schema.
Is something like this available with 6.0.0, and if so how should it be used?

@pete-otaqui this should work on 6.0:
"test": {
  "builder": "@angular-devkit/build-angular:karma",
  "options": {
    "main": "src/test.ts",
    // ...
    "codeCoverageExclude": [
      "src/polyfills.ts",
      "**/test.ts"
    ]
  }
},

Thanks for that - I actually managed to work that out by searching the repo for "codeCoverageExclude", working out where from the angular.json schema that key was allowed and went form there.
Thanks again for the example

It would be nice too if the documentation  can be updated to align with the changes that @filipesilva mentioned.
Currently it only mentioned that it expects the name of the test entry-point file.


change background to remove pineapples and spoons

Base color of old background image was #FAFAEE. Simply made the background that flat color and removed the image.

#43

I believe we talked to our sponsor, we decided not to since @dan7800 didn't notice the background.


combine edges in results for multi-hop queries

https://github.com/NCATSTranslator/testing/issues/55 issues a multi-hop query. In the BTE results, we have separate entries in the results for each edge (e01 and e02 in the screenshot 1 below), rather than a single entry in results that for each path that has both edges. Compare that to the Improving Agent output in screenshot 2 below, which I think is how TRAPI intends to capture results.

Screenshot 1

Screenshot 2

Should be implemented here: https://github.com/biothings/bte_trapi_query_graph_handler/blob/main/src/query_results.js#L39

I'm currently getting the following, which doesn't match up with either of the outputs above:
{
  "message": {
    "query_graph": {
      "nodes": {
        "n0": {
          "id": "MONDO:0005812",
          "category": "biolink:Disease"
        },
        "n1": {
          "category": "biolink:PhenotypicFeature"
        },
        "n2": {
          "category": "biolink:Disease"
        }
      },
      "edges": {
        "e01": {
          "subject": "n0",
          "object": "n1"
        },
        "e02": {
          "subject": "n1",
          "object": "n2"
        }
      }
    },
    "knowledge_graph": {
      "nodes": {},
      "edges": {}
    },
    "results": []
  },
  "logs": [
    {
      "timestamp": "2021-06-15T00:16:56.980Z",
      "level": "DEBUG",
      "message": "BTE identified 3 QNodes from your query graph",
      "code": null
    },
    {
      "timestamp": "2021-06-15T00:16:56.980Z",
      "level": "DEBUG",
      "message": "BTE identified 2 QEdges from your query graph",
      "code": null
    },
    {
      "timestamp": "2021-06-15T00:16:56.980Z",
      "level": "DEBUG",
      "message": "BTE identified your query graph as a 1-depth query graph",
      "code": null
    },
    {
      "timestamp": "2021-06-15T00:16:56.981Z",
      "level": "DEBUG",
      "message": "REDIS cache is not enabled.",
      "code": null
    }
  ]
}

Maybe related, I'm getting this message:

biothings-explorer-trapi:batch_edge_query Failed to filter 27279 results due to TypeError: Cannot read property 'map' of undefined +1ms

First, your query is not correct (not TRAPI v1.1). Second, there are known issues with BTE and one of its KP APIs right now. To avoid hitting it, I would do a query like this. This two hop takes my local instance ~32 seconds to return.
{
    "message": {
        "query_graph": {
            "nodes": {
                "n0": {
                    "categories": ["biolink:Gene"],
                    "ids": ["NCBIGene:3778"]
                },
                "n1": {
                    "categories": ["biolink:Gene"]
                },
                "n2": {
                    "categories": ["biolink:ChemicalSubstance"]
                }
            },
            "edges": {
                "e01": {
                    "subject": "n0",
                    "object": "n1"
                },
                "e02": {
                    "subject": "n1",
                    "object": "n2"
                }
            }
        }
    }
}

@colleenXu, I added your query as a new example file:
https://github.com/biothings/BioThings_Explorer_TRAPI/blob/fix_issue_164/examples/v1.1/query_multihop_gene_gene_chemical.json
I also added a test (currently failing) for the desired result for this issue. @newgene (and anyone else), can I get a confirmation that this test correctly describes the desired result?
https://github.com/biothings/BioThings_Explorer_TRAPI/blob/646c18c3a1c0dbac3e25612864b1a17f930c9be1/__test__/integration/routes/v1query.test.js#L63
If the test is good, I'll refactor the code to make it pass.

Feel free to suggest modifications to the test, if you think it should be more tightly defined.

Yes, I think that test accurately describes the desired result. If you want to be even more explicit before writing code, you could create two example outputs, one which fails the test and a reformatted version that passes. But not necessary -- only if useful for you.

@ariutta Is there a way to make the test more detailed? For example, given this query, this entire result object should exist in the results? I give an example below.
@andrewsu Please advise, since I think this is also desired behavior that we need a test for. I think the TRAPI standard says incomplete paths aren't in the results. This would be a change to BTE's current behavior.
For example, in the example below, KCNMA1 (NCBIGene:3778) -> pulmonary hypertension (MONDO:0005149) exists as an e0 edge....but there's no matching e1 edges for the second hop (pulmonary hypertension -> other genes) (I think there's no matching edges because of the predicate restriction). There should therefore be no results object with pulmonary hypertension as its n1, since there's no complete path that includes it.

For example, this is a query I ran that makes more biological sense (take the gene KCNMA1, find diseases that are caused by it, look at what other genes cause those diseases). It takes my local instance of BTE 1 min 15 sec - 1 min 40 sec to run, and the size of the response is 4.22 MB. If it's different for you, then something is likely different in the data, so the result object below might not be there.
{
    "message": {
        "query_graph": {
            "nodes": {
                 "n0": {
                    "ids": ["HGNC:6284"],
                    "categories":["biolink:Gene"]
                },
                "n1": {
                    "categories": ["biolink:Disease"]
                },
                "n2": {
                    "categories": ["biolink:Gene"]
                }
            },
            "edges": {
                "e0": {
                    "subject": "n0",
                    "object": "n1",
                    "predicates": ["biolink:gene_associated_with_condition"]
                },
                "e1": {
                    "subject": "n1",
                    "object": "n2",
                    "predicates": ["biolink:condition_associated_with_gene"]
                }
            }
        }
    }
}

Based on the nodes/edges in the Response object, I would expect the following object (1 result) somewhere in the results array. In human-readable terms, it says KCNMA -> obesity disorder (disease) -> TULP3 (gene)
            {
                "node_bindings": {
                    "n0": [
                        {
                            "id": "NCBIGene:3778"
                        }
                    ],
                    "n1": [
                        {
                            "id": "MONDO:0011122"
                        }
                    ],
                    "n2": [
                        {
                            "id": "NCBIGene:7289"
                        }
                    ]
                },
                "edge_bindings": {
                    "e0": [
                        {
                            "id": "NCBIGene:3778-biolink:gene_associated_with_condition-MONDO:0011122"
                        }
                    ],
                    "e1": [
                        {
                            "id": "MONDO:0011122-biolink:condition_associated_with_gene-NCBIGene:7289"
                        }
                    ],
                },
                "score": "1.0"
            },

@colleenXu I think your proposed test is good but much more brittle, i.e., it's dependent on data resources continuing to return the same data.  So, let's consider this in the future.  But for the moment, I think the simpler test that Anders proposed should move us substantially toward fixing this issue.  Let's reassess after this test passes.

After discussion with Andrew today, the test could have a simple addition:
The node bindings would have n0, n1, and n2, for the specified query.

Note for @andrewsu and me:

it seems like results should only have complete answers (aka there must be an edge from n0 -> n1 AND an edge from n1 -> n2 in order for it to be in the results section).
However, it's hard to figure out exactly where this was discussed in Translator. There's the comment on my issue here, this from operations/workflow group (private, for Translator members), and that's all I found.

@colleenXu, as @andrewsu said, the test you suggested would be brittle for this repo, but it might actually be perfect for QueryResult.test.js in the bte_trapi_query_graph_handler repo. The difference is that we have control over the records in QueryResult.test.js because we're specifying them manually, not getting them from an API every time.
What do you and @andrewsu think of adding your test there? I got the test started, but it needs some updating to match what you suggested (I just duplicated the record from Kevin's test):
https://github.com/biothings/bte_trapi_query_graph_handler/blob/bb4da0559cb243be396e253322d1923e6af68b6c/__test__/integration/QueryResult.test.js#L76
I think this would help to make sure we get what is desired.

@andrewsu, I have a rough WIP going on here:
https://github.com/biothings/bte_trapi_query_graph_handler/commit/97963b6f3379de4f1ed7ff8e5bedbcd303f34271
The basic idea is that if we want QueryResult.update() to only have queryResult as input, we'll need to do some kind of mapping from previous output result index to node ID. I think my current code fails because it assumes there's no expansion between hops, so we might need to create a new array to serve as the next this.results, merge the hop from each record with a deep copy of its corresponding item in this.results to add to the new array and at the end of the update call, set this.results equal to the new array.

reassigned this to Colleen to do some confirmation testing that the behavior is as expected

Strangely, I don't see the expected "fix" and the behavior looks the same as before.

This is what I did for a quick test:

run the query below on BTE using DGIdb only (https://api.bte.ncats.io/v1/smartapi/e3edd325c76f2992a111b43a907a4870/query). Using only 1 API makes the query run quickly, running local BTE gets the same output.
The celecoxib input ID (pubchem in the query) uses CHEBI:41423 as its primary ID

Because there are edges from the input ID CHEBI:41423 -> NCBIGene:5170 (PDPK1) and NCBIGene:5170 -> CHEBI:91439 (BX-795), I expect something like this in the results section:
            {
                "node_bindings": {
                    "n0": [
                        {
                            "id": "CHEBI:41423"
                        }
                    ],
                    "n1": [
                        {
                            "id": "NCBIGene:5170"
                        }
                    ],
                    "n2": [
                        {
                            "id": "CHEBI:91439"
                        }
                    ]
                },
                "edge_bindings": {
                    "e0": [
                        {
                            "id": "CHEBI:41423-biolink:physically_interacts_with-NCBIGene:5170"
                        }
                    ],
                    "e1": [
                        {
                            "id": "NCBIGene:5170-biolink:physically_interacts_with-CHEBI:91439"
                        }
                    ]
                },
                "score": "1.0"
            },

Instead, I see these two objects in the results section:
            {
                "node_bindings": {
                    "n0": [
                        {
                            "id": "CHEBI:41423"
                        }
                    ],
                    "n1": [
                        {
                            "id": "NCBIGene:5170"
                        }
                    ]
                },
                "edge_bindings": {
                    "e0": [
                        {
                            "id": "CHEBI:41423-biolink:physically_interacts_with-NCBIGene:5170"
                        }
                    ]
                },
                "score": "1.0"
            },

            {
                "node_bindings": {
                    "n1": [
                        {
                            "id": "NCBIGene:5170"
                        }
                    ],
                    "n2": [
                        {
                            "id": "CHEBI:91439"
                        }
                    ]
                },
                "edge_bindings": {
                    "e1": [
                        {
                            "id": "NCBIGene:5170-biolink:physically_interacts_with-CHEBI:91439"
                        }
                    ]
                },
                "score": "1.0"
            },

The query:
{
    "message": {
        "query_graph": {
            "nodes": {
                "n0": {
                    "ids":["PUBCHEM.COMPOUND:2662"],
                    "categories":["biolink:ChemicalSubstance"]
                },
                "n1": {
                    "categories":["biolink:Gene"]
               },
                "n2": {
                    "categories":["biolink:ChemicalSubstance"]
               }
            },
            "edges": {
                "e0": {
                    "subject": "n0",
                    "object": "n1"
                },
                "e1": {
                    "subject": "n1",
                    "object": "n2"
                }
            }
        }
    }
}

From 2021-07-13 Translator: meeting

@newgene to look into loss of score in results section; add test
@colleenXu to do small test of performance comparing performance on prod and test and post results here

Missing Score
Old code - automatically always sets it as '1.0' here:
https://github.com/biothings/bte_trapi_query_graph_handler/blob/87878936ce00b4bd3893243e65e0bdeb778660a2/src/query_results.js#L45
New code - can we just copy that old code to this location?
https://github.com/biothings/bte_trapi_query_graph_handler/blob/7ec30dcfc14a1e2a822aedd3d54128a90a376b7a/src/query_results.js#L74

@newgene, before we close this issue, do you want to delete _createEdgeBindings and _createNodeBindings? I left them to be conservative with my change, but they aren't used anywhere except the test, so leaving them might actually be confusing.
https://github.com/biothings/bte_trapi_query_graph_handler/blob/7ec30dcfc14a1e2a822aedd3d54128a90a376b7a/src/query_results.js#L86

Performance with Anders's code
TDLR: very similar to before. I did 4 tests below.
dev has Anders's multi-hop code, doesn't have scoring (BUG)
prod doesn't have multi-hop code, does have scoring
testing involves running 1 query for prod, then waiting several min before running that query on dev (or vice versa).
1
Query 1 (see section below), to BTE-MyChem only (dev url, prod url).
Dev: 9.47 sec to return 4.73 MB response.
Prod: 9.5 sec to return 3.9 MB response
2
Query 1 (see section below), to BTE-DGIdb only (dev url, prod url).
Dev: 16.81 sec to return 11.48 MB response.
Prod: 16.24 sec to return 10.07 MB response
3
Query 2 (see section below), to BTE (dev url, prod url).
Dev: 34.66 sec to return 31.16 MB response.
Prod: 36.55 sec to return 25.37 MB response
4
Query 3 (see section below), to BTE (dev url, prod url).
Dev: 9.53 s sec to return 4.97 MB response.
Prod: 8.98 sec to return 4.7 MB response

Query 1:
{
    "message": {
        "query_graph": {
            "nodes": {
                "n0": {
                    "ids":["PUBCHEM.COMPOUND:2662"],
                    "categories":["biolink:ChemicalSubstance"]
                },
                "n1": {
                    "categories":["biolink:Gene"]
               },
                "n2": {
                    "categories":["biolink:ChemicalSubstance"]
               }
            },
            "edges": {
                "e0": {
                    "subject": "n0",
                    "object": "n1",
                    "predicates": ["biolink:physically_interacts_with"]
                },
                "e1": {
                    "subject": "n1",
                    "object": "n2",
                    "predicates": ["biolink:physically_interacts_with"]
                }
            }
        }
    }
}

Query 2:
{
    "message": {
        "query_graph": {
            "nodes": {
                "n0": {
                    "categories": ["biolink:Gene"],
                    "ids": ["NCBIGene:3778"]
                },
                "n1": {
                    "categories": ["biolink:Gene"]
                },
                "n2": {
                    "categories": ["biolink:ChemicalSubstance"]
                }
            },
            "edges": {
                "e01": {
                    "subject": "n0",
                    "object": "n1"
                },
                "e02": {
                    "subject": "n1",
                    "object": "n2"
                }
            }
        }
    }
}

Query 3:
{
    "message": {
        "query_graph": {
            "nodes": {
                "n0": {
                    "categories": ["biolink:Pathway"],
                    "ids": ["REACT:R-HSA-1368082"]
                },
                "n1": {
                    "categories": ["biolink:Gene"]
                },
                "n2": {
                    "categories": ["biolink:ChemicalSubstance"]
                }
            },
            "edges": {
                "e01": {
                    "subject": "n0",
                    "object": "n1"
                },
                "e02": {
                    "subject": "n1",
                    "object": "n2"
                }
            }
        }
    }
}

I added score, removed unused methods  _createEdgeBindings and _createNodeBindings and updated tests to reflect these changes.
Based on @colleenXu 's results, it appears there's no significant change to performance.
As far as I can see, this issue is ready to be closed.

I can confirm by sending queries to the dev instance of BTE, that scores are now appearing in the correct parts of the result object for single and multi-hop Predict-style responses.
Closing this issue

reopening this issue because I think I'm seeing the same (or a related) issue.  Here is the query (three-node, two edges):
{
    "message": {
        "query_graph": {
            "nodes": {
                "n0": {
                    "ids": [
                        "PUBCHEM.COMPOUND:5358"
                    ],
                    "categories": [
                        "biolink:ChemicalSubstance"
                    ]
                },
                "n1": {
                    "categories": [
                        "biolink:BiologicalProcessOrActivity"
                    ]
                },
                "n2": {
                    "categories": [
                        "biolink:Disease"
                    ]
                }
            },
            "edges": {
                "e0": {
                    "subject": "n0",
                    "object": "n1",
                    "predicates": [
                        "biolink:related_to"
                    ]
                },
                "e1": {
                    "subject": "n2",
                    "object": "n1",
                    "predicates": [
                        "biolink:related_to"
                    ]
                }
            }
        }
    }
}

The results objects only have nodes for n0 and n1, and only have an edge for e0.

@ariutta can you have a look please?

@andrewsu, I created a draft version of a test for the case you described above:
https://github.com/biothings/bte_trapi_query_graph_handler/blob/fix_issue_164a/__test__/integration/QueryResult.test.js#L333
The records have some placeholder values (see please_fix), but does this test otherwise correctly appear to match the real example of what you saw?
Also, am I correct to understand there should be two queryResult.update calls, not a single call with two records?
https://github.com/biothings/bte_trapi_query_graph_handler/blob/fix_issue_164a/__test__/integration/QueryResult.test.js#L422

In the two results you got, I notice n1 and e0 are different for each result, but both of the n1 values appear to be in the biolink:BiologicalProcessOrActivity category (Acclimatization and Cerebrovascular Circulation).

@andrewsu, I created a draft version of a test for the case you described above:
https://github.com/biothings/bte_trapi_query_graph_handler/blob/fix_issue_164a/__test__/integration/QueryResult.test.js#L333
The records have some placeholder values (see please_fix), but I expected this test to still correctly match the real example of what you saw. However, it doesn't, because it currently returns 0 results, and I'm not sure why.

Can you confirm you see the results above when POSTing the query to https://api.bte.ncats.io/v1/query?  I confirm that I get zero results when posting to dev (which I thought was due to the Biolink Model v2.1 update, but changing ChemicalSubstance to SmallMolecule doesn't fix it, so I think I'm missing something here...)

Should there be two queryResult.update calls (not a single call with two records)?
https://github.com/biothings/bte_trapi_query_graph_handler/blob/fix_issue_164a/__test__/integration/QueryResult.test.js#L422

Hmm, that's a bit too deep into the code for me to have an informed answer on...

In the two results you got, I notice n1 and e0 are different for each result, but both of the n1 values appear to be in the biolink:BiologicalProcessOrActivity category (Acclimatization and Cerebrovascular Circulation).

Yes, from my view, it looks like only e0 is being reported and e1 is being dropped.

At the moment, I'm getting "502 Bad Gateway"

@andrewsu @ariutta I believe this issue arises because BTE actually doesn't retrieve any nodes/answers for e1.
In this situation, BTE is currently returning everything it found for the e0 step, even though it didn't find anything for the e1 step (so there are no complete paths to put in the results). We may need to decide to change BTE's behavior to return nothing (empty knowledge_graph / results sections) if it doesn't find any complete paths.
Marco has been dealing with related behavior (with the new querying logic).

@colleenXu, I think you're correct about e1.
Going back to @andrewsu's first post on this issue, I'm wondering how we want to handle the case where one of the forks in the query graph has zero hits but the other fork has at least one hit.

If any link in the query graph is broken, we want to return zero results. If we can safely assume that the query graph is always linear, then the code in branch fix_issue_164b will give us what we want.
If we allow non-linear query graphs (two examples below), the code will always return at least one result as long as there's at least one path from a start node to an end node, e.g., if there are records connecting Gene -> Disease -> ChemicalSubstance but no records connecting Gene -> BiologicalProcess -> ChemicalSubstance, the code will still return results.
We could refactor the code to observe the query graph from the record(s) passed into queryResult.update([record]) calls, but if there are zero records for BiologicalProcess -> ChemicalSubstance, we would never call queryResult.update with a record telling us this hop in the query graph even exists.

Can we assume linear query graphs for now?

First, do we agree that "linear" is irrespective of the directionality of the edges?  so A --> B <-- C is as linear as A --> B --> C?  If yes, then I am fine making the assumption for now that all query graphs will be linear.  If that's the route we go, then it would be best if we can somehow detect non-linear query graphs and proactively return an error that says "we don't support this" rather than failing in non-obvious or unpredictable ways.

If no, how should QueryResult know what the query graph looks like in the case where there are zero records returned for one or more edges of the query graph?

Hmm, I'm afraid this is a bit too deep into the implementation for me to make any intelligent comment.  But perhaps this is an argument that the pruning of the KG needs to be performed at the time of assembling the results object, rather than iteratively done as each edge is queried?

Can we assume linear query graphs for now?
It ended up being most practical to just handle all possible query graphs rather than trying to separate out and return errors for special cases.

This issue should now be resolved in this commit:
https://github.com/biothings/bte_trapi_query_graph_handler/pull/34


[Feature Request]: Add installation option to not upgrade packages with RequireExplicitUpgrade: true by default

The request
Would be nice to have installation option like -DontUpdatePackagesWithREUByDefault that will make WAU don't upgrade packages that have  RequireExplicitUpgrade: true in their manifests.
Is your feature request related to a problem?
When option `-DontUpdatePackagesWithREUByDefault` is passed, WAU will not upgrade packages that have  `RequireExplicitUpgrade: true` in their manifests. Note: packages added to `AllowList` that have `RequireExplicitUpgrade: true` will still be upgraded.

Additional information
No response

Not stale.

Not stale.

Not stale.

Not stale.

Not stale.


blackfire: 2.8.0 -> 2.8.1

Description of changes
Updated Blackfire
Things done

Built on platform(s)

[X] x86_64-linux
[X] aarch64-linux
[X] x86_64-darwin
[X] aarch64-darwin

[ ] For non-Linux: Is sandbox = true set in nix.conf? (See Nix manual)
[X] Tested, as applicable:

NixOS test(s) (look inside nixos/tests)
and/or package tests
or, for functions and "core" functionality, tests in lib/tests or pkgs/test
made sure NixOS tests are linked to the relevant packages

[X] Tested compilation of all packages that depend on this change using nix-shell -p nixpkgs-review --run "nixpkgs-review rev HEAD". Note: all changes have to be committed, also see nixpkgs-review usage
[ ] Tested basic functionality of all binary files (usually in ./result/bin/)
22.05 Release Notes (or backporting 21.11 Release notes)

[ ] (Package updates) Added a release notes entry if the change is major or breaking
[ ] (Module updates) Added a release notes entry if the change is significant
[ ] (Module addition) Added a release notes entry if adding a new NixOS module
[ ] (Release notes changes) Ran nixos/doc/manual/md-to-db.sh to update generated release notes

[X] Fits CONTRIBUTING.md.

I wanted to play with the update-source-version and setup now for me for all platforms, a builder. As I tested on the blackfire package, I just submitted the new shas. Also tested of course on the machine :)

Thanks.


stylelint command fails while used with lint-staged

I setup style-lint from docs
It's linting correctly but when the command stylelint './src/**/*.styles.ts' is processed through lint-staged I get an error lint:css failed without output (ENOENT).
I've tried to see the output of stylelint './src/**/*.styles.ts' --formatter verbose and the result seems fair

This sounds slightly similar to what I ran into when I forgot to update my husky files. Make sure your command in .husky/pre-commit, matches your package manager. For example, I use yarn so I have yarn lint-staged instead of npx lint-staged.


Proper Error Handling for Date Parse

Add a ParseError type to date parsing that can be used by the elastic_types_parsers crate and the elastic_types crate.

Reopening this because just using chrono::ParseError isn't sufficient for custom impls of parse.


php get refused port

after fresh installation and test, i try to send msg to channel from php.
here is my code
$pub = new GripPubControl(array(
            'control_uri' => 'http://localhost:7779'
        ));

        $pub->publish_http_stream('ws', 'hello there\n');

but i got this error even open port in firewall.
Failed to publish: Failed to connect to localhost port 7779: Connection refused

whats wrong?

By default, Pushpin’s control port is 5561, so double check you’re specifying the correct port.

thx jkarneges. but when i use this port on my client side i get
Can’t establish a connection to the server at ws://localhost:5561/ws.

why?

Clients connect to a front port (7999 by default) not the control port.
The reason for multiple ports is because the control port is private, to be used from the server side only.

sorry. but how connect backend(php) to frontend(js)? my code not work. why?
frontend
        var ws = new WebSocket('ws://localhost:7999/ws');
        ws.onmessage = function (message) {
            console.log(message.data);
        };

backend
$pub = new GripPubControl(array(
            'control_uri' => 'http://localhost:5561'
        ));

        $pub->publish_http_stream('ws', 'hello there\n');

Where is /ws routing to? If it's the test handler, you need to publish to channel test. If it's routing to your backend server, then your server needs to send a subscription control message.

Also, publish_http_stream publishes data only to clients that are using the HTTP streaming transport.
To publish to clients that are connected with the WebSocket transport, you need to publish a websocket message like this:
$pub->publish('test', new PubControl\Item(
    new GripControl\WebSocketMessageFormat('hello there')
));

thx. you help is very helpful. i think documents need more clear specially in php+js programming. and the last question is how can create new channel name. any channel except 'ws' with 'test' publish not work. i read route route section but cant find my answer.

See https://pushpin.org/docs/usage/#websockets . There's a PHP example.

thx but im confused. to create new channel i used below code
<?php
// composer require fanout/gripcontrol

$in_events = GripControl::decode_websocket_events(
    file_get_contents("php://input"));
$out_events = array();
if ($in_events[0]->type == 'OPEN')
{
    $out_events[] = new WebSocketEvent('OPEN');
    $out_events[] = new WebSocketEvent('TEXT', 'c:' .
        GripControl::websocket_control_message('subscribe',
        array('channel' => 'mychannel')));
}

header('Content-Type: application/websocket-events');
header('Sec-WebSocket-Extensions: grip');
http_response_code(200);

echo GripControl::encode_websocket_events($out_events);

?>

but i get Notice: Undefined offset: 0
how can i create new channel and how assign handler to it?

I'm guessing that error is coming from $in_events[0] ? This script is meant to be invoked by Pushpin, in which case events will be provided and the code should be ok.
Try making a request directly to your php script using curl. You should see something like this:
$ curl -i -d 'OPEN'$'\r'$'\n' http://localhost:8000/yourphpfile.php
HTTP/1.1 200 OK
Date: Sat, 15 Jun 2019 22:23:31 GMT
Server: Apache/2.4.29 (Ubuntu)
Sec-WebSocket-Extensions: grip
Content-Length: 61
Content-Type: application/websocket-events

OPEN
TEXT 2c
c:{"channel":"mychannel","type":"subscribe"}

If you don't see that, then something is wrong with the script. But if you do see that, and you have Pushpin configured to route to your backend server, then making a WebSocket request through Pushpin (e.g. to ws://localhost:7999/yourphpfile.php) should work.


conditional instance restarts on config file changes

Allow conditional restarts of a redis instance on configuration changes.

Any chance this could get merged? Thanks


Passphrase required exception

Hi,
I'm trying to test out smart lock to see if I should start using it in my app. I cloned the project and ran the credentials-signin project on my phone, but when I try to save a new credential, I get the following error. I looked the error code up and it is the error code for resolution required
save:FAILURE
com.google.android.gms.common.api.ApiException: 6: Passphrase required.
  at com.google.android.gms.common.internal.zzb.zzy(Unknown Source:14)
  at com.google.android.gms.common.internal.zzbk.zzz(Unknown Source:0)
  at com.google.android.gms.common.internal.zzbl.zzr(Unknown Source:32)
  at com.google.android.gms.common.api.internal.BasePendingResult.zzc(Unknown Source:46)
  at com.google.android.gms.common.api.internal.BasePendingResult.setResult(Unknown Source:42)
  at com.google.android.gms.internal.zzavw.setResult(Unknown Source:2)
  at com.google.android.gms.internal.zzavv.zze(Unknown Source:2)
  at com.google.android.gms.internal.zzawc.onTransact(Unknown Source:46)
  at android.os.Binder.execTransact(Binder.java:682)

So, I was wondering why this error was happening and how to resolve it. After looking at the saveCredential(), it seems like the exception should've been an instance of ResolvableApiException.
Thanks,
Jeffrey

I am closing this issue/PR, as it has been migrated to the new repo linked above in the comments. Thank you!


Inconsistent return type from jax.numpy.linalg.norm

With jax=0.2.9, jaxlib=0.1.61 and jaxlib=0.1.60, the return type of jax.numpy.linalg.norm is inconsistent.
The evaluation of norm returns a _DeviceArray.  But any subsequent evaluations return a Buffer:
import jax
from jax.numpy.linalg import norm

key = jax.random.PRNGKey(0)
x = jax.random.normal(key, (32, 32))

print(type(norm(x)))  #  <class 'jax.interpreters.xla._DeviceArray'>
print(type(norm(x)))  #  <class 'jax.interpreters.xla.Buffer'>
print(type(norm(x)))  #  <class 'jax.interpreters.xla.Buffer'>

Note that this doesn't happen with sum:

print(type(jax.numpy.sum(x)))  #  <class 'jax.interpreters.xla._DeviceArray'>
print(type(jax.numpy.sum(x)))  #  <class 'jax.interpreters.xla._DeviceArray'>
print(type(jax.numpy.sum(x)))  #  <class 'jax.interpreters.xla._DeviceArray'>

This behavior is not present with jax=0.26, jaxlib=0.1.57.

The difference between Buffer and _DeviceArray is intended to be an internal difference (one is generated by Python code, the other by C++, and we are gradually moving some logic into C++). Can you say more about why it affects you? It shouldn't change anything other than the type. Note both are instances of DeviceArray, which is a superclass.

I noticed it while debugging a different issue.  This wasn't my real problem, just thought the variable nature of the return type was odd.  Feel free to close if this behavior is expected.


Interpolation syntax only needed for custom CSS Properties

I was having a hard time getting some CSS Custom Properties (CSS variables) to take on some hex values that I was holding in some SCSS vars, after updating a number of libraries in my project.
What was odd was that the following code was working fine:
$foo: #ccc;

html, body {
  background: $foo;
}

In the browser's dev tools I would see: background: #ccc; and the background would be grey, as expected.
But this code didn't work:
$foo: #ccc;

html {
  --my-grey-color: $foo;
}

Oddly enough in the browser dev tools, I'd then see --my-grey-color: $foo, w/o the SCSS variable's value interpolated.
I finally came across this comment:
https://github.com/sass/node-sass/issues/1943#issuecomment-379747451
I tried wrapping the variable reference in #{} and bingo it worked!
So here's my question.... while I'm glad I finally have a way forward, and thankful for that comment on that thread.... Why is it that the #{} is needed only when trying to interpolate values for CSS Custom Properties, but seems to be working just fine for others?

NPM version (npm -v): 6.9.0
Node version (node -v): 10.15.3
Node Process (node -p process.versions):

{ http_parser: '2.8.0',
  node: '10.15.3',
  v8: '6.8.275.32-node.51',
  uv: '1.23.2',
  zlib: '1.2.11',
  ares: '1.15.0',
  modules: '64',
  nghttp2: '1.34.0',
  napi: '3',
  openssl: '1.1.0j',
  icu: '62.1',
  unicode: '11.0',
  cldr: '33.1',
  tz: '2018e' }

Node Platform (node -p process.platform): darwin
Node architecture (node -p process.arch): x64
node-sass version (node -p "require('node-sass').info"):

 node -p "require('node-sass').info"
node-sass	4.11.0	(Wrapper)	[JavaScript]
libsass  	3.5.4	(Sass Compiler)	[C/C++]

npm node-sass versions (npm ls node-sass):

 npm ls node-sass
<EMAIL_ADDRESS>/Users/matthewmarcus/dev/omnisci/immerse2
└──<EMAIL_ADDRESS>

This is required because the way CSS custom properties are spec'd. See https://github.com/sass/sass/issues/1128


Persistent Dutch text strings under Notices

In personal notifications tab, I get this message.:
"Manuel Jiménez Friaza, satipera gaf een reactie op een bericht van satipera hace 3 minutos"
Under images shared with Diaspora, you see this message:
"volledige grootte tonen"
It is as if they had mixed translations, Dutch and Spanish...
Jeroenprat, that is who upload my Spanish translation to the github, said:
"One thing I find is that all those strings you mentioned are in the Spanish strings.php and hstrings.php. So that is not the case. Another thing I find is that the Spanish strings.php (not hstrings.php) has more lines than the Dutch (2149 vs 2124). But that should actually reverse your problem.
So please submit an issue on Github. In the meantime I will change the hub default to English and see if I experience the same problem. You are allowed to copy and paste this message into the Github issue."
EDITED:
Jeroenprat changed the default language of your hub to English. Under images shared with Diaspora reappears "View full size", but nothing changes in Personal Notifications tab.

I can reproduce this, so I can confirm this. When I change my language to Spanish, I still see Dutch hover text under Notices and Dutch on notifications/system; like this: "jeroenpraat, Mike Macgirvin gaf een reactie op een bericht van George Ellenburg hace 8 horas". When I change my language to German, I get the same behaviour: "jeroenpraat, Mike Macgirvin gaf een reactie op een bericht van George Ellenburg vor 8 Stunden". And I can go on like that. So it's not related to the Spanish language strings. I also changed the default language to English on my hub, but that doesn't change the above mentioned text.
To be sure, I cleared my browser cache, emptied store/[data]/smarty3/compiled/, restarted my browser and tested it in both the latest Firefox and Chromium.

I think I've found it. Somehow the notify text is set to someone's account language, not their channel language. When I look into the database account language's are set to the default site language when the account is created. That's no issue when that account language is not used. Let me see why the notify text is using that.

This is probably it, but I don't know how to change it. It's in enotify.php (line 55) (used for e-mail notifications?), but probably also used for the notifications on the hub. It is also a known issue so to see.
push_lang($recip['account_language']); // should probably have a channel language

The problem is that the channel language is temporary stored in the session table. The only solution is I think that the localized bits are not stored in notify_msg, but generated when viewing the notifications. I'm afraid here stops my know-how (actually I'm quite proud on how far I came). A workaround is to let the users change their account_language.  I can change it in the database myself, but that's not a solution of course.

And of course we have the problem that this in include/account.php:179
dbesc(get_best_language()),

...doesn't find the user's web browser language when signing up, but uses the default site language (I presume that one is only used when detecting the user's web browser language failed).

Accounts on Redmatrix.nl have their account_language set to their browser's language. The last account created on that hub is on August 28th and until then it worked as expected.
But, when I compare relevant files like include/account.php and include/language.php they are almost the same. Except two (I think) unrelated commits. So I'm lost here.

I did some tests. When I create a new account with the browser changed to Spanish the account_language is Spanish. So forget what I said about that get_best_language doesn't work.
I now remember why Manuel's account_language was set to Dutch, because I created (as agreed) his account. So that explains that.
The only two issues that remain are feature requests (option to change account_language and using channel language in msg_notify). I close this issue. Sorry for the confusion and (probably) email spam.


Ban punishment gets created regardless if user is already banned

This is caused by the following code not returning the await.
https://github.com/JustAnyones/Pidroid/blob/22a2938df114e009fe13a66445ef0d0361b0a6be/bot/cogs/ext/commands/mod/punish.py#L243
Will fix it after holidays.

Should be fixed by https://github.com/JustAnyones/Pidroid/commit/08ca4809924e1805551e8562657e81bcd9cf6ee3

Should be fixed by https://github.com/JustAnyones/Pidroid/commit/08ca4809924e1805551e8562657e81bcd9cf6ee3


Correctly handle ICCCM globally active input model

This issue stems from swaywm/sway#3007.
I've found that from 2020.3 onwards IntelliJ sets the input field of WM_HINTS to set to false for certain dialogs.  This is allowable under the ICCCM specifications (see https://tronche.com/gui/x/icccm/sec-4.html#s-4.1.7).  If the input model is globally active the window receive passive input and then issue a SetInputFocus request to receive focus.  However under Wayland there is no mechanism to access input passively.  This leads to the situation under Sway where these dialogs cannot receive user input.
I've created a rather broad fix (perhaps overly so) which I've tested with:

Chrome
IntelliJ
Alacritty
VSCode

I have not seen any major issues.  The fix is to always enable the input flag.  My rationale is as follows:

If wlroots is not compiled with ICCCM support this is the default so it must be somewhat reasonable otherwise I'd expect to see some open issues.
If a window really does not require input and receives it this is somewhat easier for the end-user to navigate around rather than the current situation whereby the window can never receive input.

If this fix is too broad then perhaps we can at least open the discussion on how this can be better handled.  The current situation leaves JetBrains products rather unusable.

Do you have the fix you mention on a branch somewhere? I'd love to help testing this out.

Do you have the fix you mention on a branch somewhere? I'd love to help testing this out.

We had a very hard time getting focus to work somewhat correctly with X11 windows. We'll need extensive testing before merging anything.
Some windows really don't want input focus, like tooltips. How does your patch handle them?
In the spec, with "Globally Active Input" the input field is set to False but WM_TAKE_FOCUS is enabled. Can't we special-case it?

We had a very hard time getting focus to work somewhat correctly with X11 windows. We'll need extensive testing before merging anything.
Some windows really don't want input focus, like tooltips. How does your patch handle them?
In the spec, with "Globally Active Input" the input field is set to False but WM_TAKE_FOCUS is enabled. Can't we special-case it?

I think supporting special-cases would make sense.  I agree that you don't want tooltips to take focus. I'll have more of a look at this.

I think supporting special-cases would make sense.  I agree that you don't want tooltips to take focus. I'll have more of a look at this.

Hi,
your patch breaks another behavior. 'Normal' child windows from intelli no longer get the focus. If the mouse not over the new window i can't use shortcuts (for example the quit dialog and 'enter').

Hi,
your patch breaks another behavior. 'Normal' child windows from intelli no longer get the focus. If the mouse not over the new window i can't use shortcuts (for example the quit dialog and 'enter').

Thanks for testing, @Haardt.
Just following up after some further investigation.  It appears as though the dialog boxes in question do not set WM_TAKE_FOCUS.  The relevant logging can be seen below (there are no protocols configured).
00:00:58.707 [DEBUG] [wlr] [types/wlr_surface.c:673] New wlr_surface 0x55bd86009d30 (res 0x55bd860021f0)
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:1525] New xwayland surface: 0x55bd86009d30
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:415] XCB_ATOM_WM_CLASS: jetbrains-idea jetbrains-idea
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:468] XCB_ATOM_WM_NAME: win2
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:514] XCB_ATOM_WM_TRANSIENT_FOR: 0x55bd85e87870
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:573] WM_PROTOCOLS (0)
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:606] Input disabled for 4194685 as non-modal
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:609] WM_HINTS (9)
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:661] WM_NORMAL_HINTS (18)

At this point I am not sure of the best course of action.  I will update the issue in IntelliJ with the information provided above in the hope that they correctly set WM_TAKE_FOCUS.
Any suggestions?

Thanks for testing, @Haardt.
Just following up after some further investigation.  It appears as though the dialog boxes in question do not set WM_TAKE_FOCUS.  The relevant logging can be seen below (there are no protocols configured).
00:00:58.707 [DEBUG] [wlr] [types/wlr_surface.c:673] New wlr_surface 0x55bd86009d30 (res 0x55bd860021f0)
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:1525] New xwayland surface: 0x55bd86009d30
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:415] XCB_ATOM_WM_CLASS: jetbrains-idea jetbrains-idea
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:468] XCB_ATOM_WM_NAME: win2
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:514] XCB_ATOM_WM_TRANSIENT_FOR: 0x55bd85e87870
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:573] WM_PROTOCOLS (0)
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:606] Input disabled for 4194685 as non-modal
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:609] WM_HINTS (9)
00:00:58.707 [DEBUG] [wlr] [xwayland/xwm.c:661] WM_NORMAL_HINTS (18)

At this point I am not sure of the best course of action.  I will update the issue in IntelliJ with the information provided above in the hope that they correctly set WM_TAKE_FOCUS.
Any suggestions?

Sorry - can only test the patches. I don't know anything about wlroots :|. A patch would be fantastic - idea is my daily tool

Sorry - can only test the patches. I don't know anything about wlroots :|. A patch would be fantastic - idea is my daily tool

An update from the JetBrains team seems to indicate that it is a Sway issue: https://youtrack.jetbrains.com/issue/JBR-3015

An update from the JetBrains team seems to indicate that it is a Sway issue: https://youtrack.jetbrains.com/issue/JBR-3015

Is this fixed with https://github.com/swaywm/wlroots/pull/2605?

Is this fixed with https://github.com/swaywm/wlroots/pull/2605?

It’s fixed along with: https://github.com/swaywm/sway/pull/5958

It’s fixed along with: https://github.com/swaywm/sway/pull/5958

Closing because wlroots now exposes a helper to allow compositors to properly fix this issue.

Closing because wlroots now exposes a helper to allow compositors to properly fix this issue.


"CreateSession still waiting for response from worker" using Multi-threaded Algorithms

Hey
I am trying to launch the example presets using the -n args for multithreading.
For instance, using the following command
python coach.py -p CartPole_PG -n 2 -v

I get these errors
2018-03-09 13:21:46.936783: E tensorflow/stream_executor/cuda/cuda_driver.cc:406] failed call to cuInit: CUDA_ERROR_NO_DEVICE
2018-03-09 13:21:46.936836: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:158] retrieving CUDA diagnostic information for host: naboo21
2018-03-09 13:21:46.936853: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:165] hostname: naboo21
2018-03-09 13:21:46.936903: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:189] libcuda reported version is: 387.26.0
2018-03-09 13:21:46.936942: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:369] driver version file contents: """NVRM version: NVIDIA UNIX x86_64 Kernel Modu

And then these infinitely
2018-03-09 13:21:58.067204: I tensorflow/core/distributed_runtime/master.cc:221] CreateSession still waiting for response from worker: /job:ps/replica:0/task:0
2018-03-09 13:21:58.067340: I tensorflow/core/distributed_runtime/master.cc:221] CreateSession still waiting for response from worker: /job:worker/replica:0/task:1
2018-03-09 13:21:58.067353: I tensorflow/core/distributed_runtime/master.cc:221] CreateSession still waiting for response from worker: /job:worker/replica:0/task:2
2018-03-09 13:21:58.114851: I tensorflow/core/distributed_runtime/master.cc:221] CreateSession still waiting for response from worker: /job:ps/replica:0/task:0

However, when I run the single threaded agent using
python coach.py -p CartPole_PG -v

It works well, a GPU is found.
When I run directly the parallel_actor.py via
python3 ./parallel_actor.py --ps_hosts=localhost:34576 --worker_hosts=localhost:49588,localhost:38853,localhost:38037 --job_name=worker --load_json=./experiments/09_03_2018-13_01/run_dict_worker2.json

It founds also well a GPU
It seems like the subprocess parallel_actor.py does not have the visibility of GPUs
Any ideas ?

I uninstalled tensorflow-gpu and installed tensorflow to use only CPUs
The error failed call to cuInit: CUDA_ERROR_NO_DEVICE is gone, but still I have the lines CreateSession still waiting for response from worker printing forever

Okay my bad, after export GRPC_VERBOSITY=DEBUG I discover that it was my proxy.
Solved!

Good catch, I was just about to post that a proxy has caused this problem for me as well. It might be good if we better handled this situation.


docs: Infino example fix

Fixed a broken link in the integrations/providers/infino.mdx
Fixed a title in the integration/collbacks/infino.ipynb example
Updated text format in this example.

@baskaryan Thanks!


error intalling from linux_install_full.sh

According to documentation in order to install in linux machine it is necessary the following execution:
curl -fsSL https://h2o-release.s3.amazonaws.com/h2ogpt/linux_install_full.sh | bash

And it is failed with the following error:
...
...
...
+ sudo unzip -o chromedriver-linux64.zip
Archive:  chromedriver-linux64.zip
  inflating: chromedriver-linux64/LICENSE.chromedriver
  inflating: chromedriver-linux64/chromedriver
+ sudo mv chromedriver-linux64/chromedriver /usr/bin/chromedriver
+ sudo chown root:root /usr/bin/chromedriver
+ sudo chmod +x /usr/bin/chromedriver
+ pip install optimum==1.18.0 -c reqs_optional/reqs_constraints.txt
Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cu121
Collecting optimum==1.18.0
  Using cached optimum-1.18.0-py3-none-any.whl.metadata (18 kB)
Requirement already satisfied: coloredlogs in /root/miniconda3/envs/h2ogpt/lib/python3.10/site-packages (from optimum==1.18.0) (15.0.1)
Requirement already satisfied: sympy in /root/miniconda3/envs/h2ogpt/lib/python3.10/site-packages (from optimum==1.18.0) (1.12)
INFO: pip is looking at multiple versions of optimum to determine which version is compatible with other requirements. This could take a while.
ERROR: Cannot install optimum==1.18.0 because these package versions have conflicting dependencies.

The conflict is caused by:
    optimum 1.18.0 depends on transformers<4.40.0 and >=4.26.0
    The user requested (constraint) transformers==4.40.0

To fix this you could try to:
1. loosen the range of package versions you've specified
2. remove package versions to allow pip attempt to solve the dependency conflict

ERROR: ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/topics/dependency-resolution/#dealing-with-dependency-conflicts

No, it is the same environment I have been using for long time

You should create a new one that would work

Looks like you are using an older h2ogpt hash.  Please use latest where that was fixed. i.e. this line:
pip install optimum==1.18.0 -c reqs_optional/reqs_constraints.txt

hasn't existed for a few days and was only temporary.
It now looks like:
pip install optimum==1.17.1 -c reqs_optional/reqs_constraints.txt


add bucket to test summary, so you can determine which logfile to look at

So, now the test-summary will print you the bucket:
* Test "shell_server_aql_2"
    [SUCCESS] tests/js/server/aql/aql-array-access.js
    [SUCCESS] tests/js/server/aql/aql-bind.js
    [SUCCESS] tests/js/server/aql/aql-complex.js
    [SUCCESS] tests/js/server/aql/aql-distance.js

Tests ok.


Community

add information to README.md on how to run the Django application

Looks good! merging.


iHaveAnExistingWithWhere now throws errors when no rows are affected.

Workaround is to wrap your feature context in a try/catch, however throwing an error when something is not affected seems a bit overzealous...?

Have an existing infers that one is looking for something that already exists, if it doesnt it throws an exception. This behavior has been like this from day 1.
What are you trying to achieve?
What you could do is catch the specific type of exception which is https://github.com/forceedge01/behat-sql-extension/blob/behat/2.5.x/src/Context/Exceptions/NoRowsAffectedException.php

I think this was fixed, I just hadn't done a pull. Hence closed.


feat: file-based inputs, cleaner ux overall, full testing framework, plus lots of other goodies

<!--------------------------------------------------------------------------
👉 STEP 1: Welcome! Thanks for taking the time to submit a pull request!

## Summary

This is a really big update with tons of improvements, including file-based input support, cleaner ux for create, write, read, and most other sub-commands, better treatment of defaults and optional configs, and most importantly, a full test suite.

## Details

This was a pretty big overhaul of the current system. Includes lots of new testing infrastructure. Note that the tests _aren't _ setup to run in CI yet. This is because they take way too long right now, with the node startup etc. For now, these tests will need to be run locally before pushing major changes.

## How it was tested

I created a whole testing suite, it was a lot of work and took a long time. Testing cli tools is kinda hard?

## Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my feature works
- [x] New and existing unit tests pass locally with my changes
- [x] Any dependent changes have been merged and published in downstream modules

Consider this WIP until I can submit my comments. Computer died before I could hit send.

Ok, @joewagner and @sanderpick if you have some bandwidth, please take a look.

whoa, tons of work in here 🥇 will tal over the weekend!

Hey @sanderpick and or @joewagner can you guys take a look. This one is getting a bit old now.

Hey @sanderpick and or @joewagner can you guys take a look. This one is getting a bit old now.

Definitely!  Sorry about that.  I didn't see or capture this and it slipped through the cracks.

Hey @joewagner this is ready for a final review. Check it out, the full test suite is running in CI in under 2 minutes!

Hey @joewagner this is ready for a final review. Check it out, the full test suite is running in CI in under 2 minutes!

Excellent!  I'll take another look today.

That c8 issue seems to happen to be sometimes as well. I'm not sure what the cause is? Will have to investigate, but seems ok for now because it is nice to have that coverage report. I'll add a test and a coverage script so that we can do one locally without requiring the other.


Help Solid users to on-board Web Monetization as a consumer

Renaming this to be more like a problem description and less like a solution description:

Splitting this out becoming a creator, this issue will from now on just be about becoming a consumer.

Currently, you would need:

a Coil account
a Coil membership
a Coil browser extension

We could create an alternative browser extension, but that's a big project.
Another option is if Solid OS would send peer-to-peer IOUs to content creators when Alice views Bob's pod.
So then any Solid OS based app would not require the user to have the browser extension installed.
The money sent would be network credits (IOUs) that you can't cash out without help from the money sender; settling would be manual. We could use the existing code from https://github.com/ledgerloops/snap-solid.


Bi-Directional Homematic support

Hi,
I would like to see bi-directional communication between Home Assistant and Homeatic devices in the future.
Besides the already supported thermostats by Homematic, there are lots of devices that send event-messages to registered logic-providers to perform actions depending on the event-message content.
Examples:

Sensors report if a door / window is open or closed
Switches report if / which button is pressed
Rollershutters report the current level while working

There are far more device for which this is true. And as of now, the available thermostat just polls the CCU / Homegear for that information. Which is ok for data with low volatility like temperature. But especially the sensors for doors / windows and switches depend on instant feedback.
That being said, I've put together a (standalone) module that showcases how bi-directional communication for Homematic devices can be implemented. It is a proof of concept which just creates log-messages for the events. For usability I primarily focused on easy interaction with devices, so it's not tailored for integration in HA.
I've had a look at the HA code, but could not grasp on how I could integrate my work in a way that makes sense and suits the architechture of HA.
Hence I ask if someone could either guide me on how to integrate my code myself, or rework my code so it can be implemented.
Roughly explained it works like this:

Create a XML RPC server providing certain methods the CCU / Homegear expects and sends eventmessages to (this is my main issue since I don't want to mess up the threading HA is doing)
Create the XML RPC proxy that talks to CCU / Homegear
Handle the devices the CCU / Homegear advertises (e.g. save them for quicker HA startup - essentially this gets us auto-discovery, so later on we would just need to map the device-address to a name to use in HA)
Create the interactive entities (similar to the existing thermostat) which use the already existing proxy-object the server created
Make the XML RPC server pass event-messages to the entities' update()-method  so we can send the correct, device-specific information to the HA event-system.

Once bi-directional support is given within HA I would then proceed on implementing further device types (I have access to at least 15 other types of devices besides the 3 existing ones).
You can view my work here: pyhomematic
PS: The thermostat I have implemented would fix the Homematic-specific issue #709, not taking into account HA would generally need support for such modes.

Here are some steps to get started:

Make pyhomematic a true library to interact 2-way with homematic devices, including allowing other scripts to register for callbacks etc. Abstract the XMLRPC API away and offer a nice API for other scripts, offer objects that represent a thermostat, a switch etc.
Create a Home Assistant component called homematic that instantiates a pyhomematic server. It will check with pyhomematic what kind of platforms are supported and will fire discovery events for those platforms. This is exactly how Wink support works.
Add a platform for each component that Homematic supports. Light, switch, thermostat etc.

I would love to join in. Can offer way more motivation than experience. I have a working Homematic XML RPC Server implemented in Java but thats about all.
Is there a good example for a "true library" with a similar setup as required for the Homemtaic component in Home Assistant?

I have restructured my module to be a library already since I've created this issue. It's available through pip as well. So actually the only parts missing right now, as far as I can tell, are proper components and the platforms. Due to lack of time I haven't started with that.
Have a look at my referenced repository. The readme as well as the provided example demonstrate how to library can / should be used. All that's needed to get started is installing it via pip and figuring out an efficient implementation.

I did some very first steps to integrate the pyhomematic into home-assist (not a working component yet):
https://github.com/jazzaj/home-assistant-homematic/blob/master/homematic.py
I have a good understanding how to use the Entity class to create devices but I have trouble to get a grip on the setup of the devices. Maybe @balloob can give me a hint:

Where should I implement the actual creation of devices?
Do I need to keep track myself on the discovered devices (similar to the TellstickRegistry class in the tellstick implementation?
Am I on the right track with regards to callbacks?

Thanks for any hint...

I had a quick look.

You can't use the library without using a remote server, which you currently have set to None. The remote will always be user-defined. And the local address as well, because you can't pass <IP_ADDRESS> when initting at the CCU / Homegear. The local port could be hardcoded, but maybe someone is using that for some other service running on that machine, so I wouldn't suggest that either.
The library does not provide methods like get_rollershutters. Instead you iterate over homeatic.devices. Each device has a .TYPE attribute which is something like "HM-CC-RT-DN", which tells you what type of device you are dealing with. So basically something like:

for device in homematic.devices:
    if device.TYPE == "HM-CC-RT-DN":
        # setup Thermostat
    elif ...

As for your questions:

As far as I understand that should be done while iterating over the found devices (the code I've mentioned above). Setting up would create the platform-entities. Those entities need to be able to register their own callbacks so they update themselves as soon as an event occurs.
I'm not sure what you mean. The library detects all devices known by the CCU / Homegear. If new devices get paired, the list of devices get updated. Homeassistant won't notice that unless you add a system-callback (seen in the example.py I believe) which listens for such changes and creates the new entities. Altough I don't think hat's important in the beginning. If new devices get paired, restarting Homeassistant would take care of that.
Don't know about that. At least for stopping everything it looks ok.

My suggestion: Use the library straight from the commandline to get to know how it is behaving. Try to get it working there, and if it runs like that, try to recreate that with the component.

Thanks for your comments. I worked through them and made some progress. The code is working:

Server is starting and is auto detecting platforms (not devices). This seems to work via Netdisco (but I'm not sure)
You can manually create HMSwitch devices

Still many things to do:

Only one platform added (Switch)
Addiding more parameters to the configuration of switches (currently it is key only)
Status changes are not yet visible in hass/changes of status does not reach the ccu (issues with pyhomematic)
...

Guidance regarding auto-detection of devices is highly welcome. Mainly I lack an idea if a) I want this and b) how to trigger the creation of devices

The auto detection of devices is what I am unsure about as well. Since homematic doesn't name its devices, they have to be configured anyways. So the easy way would actually be to just do it like with the already existing thermostat platform, but instead of it creating its own xml rpc proxy, it could use the one provided by pyhomematic. Additionally, the platform would add its update-method as a callback to the corresponding device within pyhomematic.
That way the homematic component wouldn't do more than just initializing the local server to receive events, and all the platforms would just leverage the callback-functionality provided by pyhomematic.

Meanwhile I have implemented an auto-detection option (will upload soon). Users can use the configuration file to turn this feature on or off. As you have pointed out by default there is no access to the device names and without names the auto detection part seems useless.
At least for the Homematic CCU (not sure about Homegear) there is the option to set names but they are not accessible via the XML-RPC interface. The alternative would be for all CCUs that have the XML-API Addon installed (http://www.homematic-inside.de/software/addons/item/xmlapi). With devicelist.cgi you could obtain a list off all device names.
What do you think? Do you see an option to integrate this into pyhomematic? Only for CCUs with the add-on installed the names could be added to the devices.
PS: I have left some pull requests and issues in the pyhomematic git...
PSS: I'm confident to have by Monday an alpha version to integrate lights, switches and rollershutter into home assisstant.

Since I don't know about XML-API addon for Homegear as well, I suggest to initially focus on manual device/platform configuration. If that's done and working, name-resolving could be a feature within pyhomematic. So you could create the issue for that in my repo, since that's not directly related to home-assistant. I do indeed think that would be nice feature. Although I don't own a CCU, so I'd need some additional information about how all that works. But again, that's part of the pyhomematic-issue, not HA.
I'll have a look at the pull-requests. Thanks for mentioning. Didn't notice that.

Closing this issue as feature requests should be posted in the forum.


AsyncArray  support

Hi, @shimat
generally we can get better performance when making DNN inferences asynchronously
It seems there's no asyncarray implementation in OpenCvSharp
Is there some plan for this feature?
AsyncArray
forwardAsync
DNN async demo

I came to similar problem
It should be great if OpenCvSharp supports this feature