/etc/protocols.csv

https://github.com/bitcoincashorg/blob/master/etc/protocols.csv
$ bitcoin-cli getblockhash 0
000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
$ bitcoin-cli getblock
"000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" 1
{
"hash": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f",
"confirmations": 544077,
"strippedsize": 285,
"size": 285,
"weight": 1140,
"height": 0,
"version": 1,
"versionHex": "00000001",
"merkleroot": "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b",
"tx": [
"4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"
],
"time":<PHONE_NUMBER>,
"mediantime":<PHONE_NUMBER>,
"nonce":<PHONE_NUMBER>,
"bits": "1d00ffff",
"difficulty": 1,
"chainwork": "0000000000000000000000000000000000000000000000000000000100010001",
"nTx": 1,
"nextblockhash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048"
}
index:startHeight:0f
.z.ts:{[]
Hash:.bitcoind.getblockhash[index][result];   if[not 0n~Hash;     -1(string .z.p)," Processing Block: ",string[index];     processBlock[Hash];     index+:1     ];   } processBlock:{[Hash]   Block:.bitcoind.getblock[Hash;(enlist verbosity)!(enlist 2)];
}
q)meta trade

c
t f a

date
d

time
t

sym
s

price
f

size
i

stop
b

cond
c

excc
c

q)meta quote

c
c f a

date
d

time
t

sym
s

bid
f

ask
f

bsize
i

asize
i

mode
c

ex
c

q)t:select tradecount:count i by date from trade
q)q:select quotecount:count i by date from quote
q)t lj q

date
tradecount quotecount

2010.12.01
4940021    24713334

2010.12.02
4940300    24714460

2010.12.03
4940482    24708283

...

2010.12.30
4982917    24933374

2010.12.31
4994626    25000981

//set the date to be requested

d:first date

//sym before date (non-optimal order)
select from trade where sym=`IBM, date=d
//date before sym (optimal order)
select from trade where date=d, sym=`IBM
|   sym before date  |   date before sym
dates in |  time         size |  time         size

database
(ms)          (b)
(ms)          (b)

 1   |   470   75,499,920 |    78   75,499,984
 5   |   487   75,878,400 |    78   75,499,984
10   |   931   75,880,624 |    78   75,499,984
15   | 1,209   75,882,912 |    78   75,499,984
20   | 1,438   75,885,072 |    78   75,499,984

//make dictionary containing groups of distinct syms
//drawn from our sym universe
syms:n!{(neg x)?sym}each n: 1 10 100 500
//make call for open prices for ranges of security drawn from the dictionary
select open:first price by sym from trade
where date=first date, sym in syms 1
select open:first price by sym from trade
where date=first date, sym in syms 500
//create ohlc table with a number of different day-level aggregations,
//and save to disk using dpft function.
//perform across each date in the db
{ohlc::0!select
open:first price,
high:max price,
low:min price,
close: last price,
vwap: size wavg price by sym from trade where date= x;
.Q.dpft[`:.;x;`sym;`ohlc]; } each date
//This will create one row per security/date pair.
q)select count i by date from ohlc

date
x

2010.12.01
500

2010.12.02
500

...

2010.12.30
500

2010.12.31
500

//reload db
q)\l .
//repeat the same requests, but target our new aggregation table.
q)select open by sym from ohlc where date=first date, sym in syms 1
...
q)select open by sym from ohlc where date=first date, sym in syms 500
|      from trade     |      from ohlc

sym values
time (ms)  size (b)
time (ms)  size (b)

    1  |      1      265,056 |      0        2,896
   10  |     13    2,100,224 |      0        3,504
  100  |     53   16,781,680 |      0       12,400
  500  |    125  134,228,736 |      0       48,752

//perform the same query over widening date windows
//restart kdb+ and flush os cache between each request
select size wavg price by sym from trade where date in 5#date
...
select size wavg price by sym from trade where date in 20#date
dates in query  time (ms)
   5         1,863
  10         3,152
  15         4,596
  20         7,597

//add p attribute using setattrcol from dbmaint.q //NOTE: setattrcol does not adjust the order of the data.  //Our data is already segmented by sym. setattrcol[:.;trade;sym;`p]
\l .
//set the date to be requested
d:first date
//sym before date (non-optimal order)
select from trade where sym=`IBM, date=d
//date before sym (optimal order)
select from trade where date=d, sym=`IBM
|       no attribute (from example)  |         parted attribute set
| sym before date  | date before sym | sym before date | date before sym
dates |  time      size  | time      size  |  time     size  |  time   size

in db
(ms)       (b)
(ms)       (b)
(ms)      (b)
(ms)    (b)

1
470 75,499,920
78  75,499,984
3    445,024
0    445,088

5
487 75,878,400
78  75,499,984
24    889,936
0    445,088

10
931 75,880,624
78  75,499,984
26    892,816
0    445,088

15
1,209 75,882,912
78  75,499,984
64    896,336
0    445,088

20
1,438 75,885,072
78  75,499,984
177    898,576
0    445,088

//create time we will use for test requests

t:first 09:30:00.000+1?07:00:00.000

//create temporary table using one or more days of saved data
// – vary from 1 through 10 on each iteration
rtquote:{
time xasc select from quote where date in x#date   }1  rtquote:{   time xasc select from quote where date in x#date
}10
//make requests against rtquote for IBM data.
//Repeat as the rtquote table grows
select from rtquote where time=t, sym=`IBM
//repeat process with s removed from time vector rtquote:update #time from{
time xasc select from quote where date in x#date   }1 rtquote:update #time from{
`time xasc select from quote where date in x#date
}10
|    no attribute   | sorted attribute
rows in     | time         size | time        size

table
(ms)          (b)
(ms)         (b)

25,000,000
49   33,554,944
0       1,248

50,000,000
102   67,109,392
0       1,248

75,000,000
154  134,218,288
0       1,248

100,000,000
213  134,218,288
0       1,248

125,000,000
247  134,218,288
0       1,248

150,000,000
299  268,436,016
0       1,248

200,000,000
406  268,436,016
0       1,248

225,000,000
523  268,436,016
0       1,248

250,000,000
568  268,436,016
0       1,248

//function to create example last value cache keyed table with x rows

//also creates global s to retrieve query value later

mktbl:{([sym:s::(neg x)?`7];lprice:0.01*x?10000)}

//make example table, vary through the values on each iteration
tbl:mktbl 100
...
tbl:mktbl 10000000
//create test value
req:first 1?s
//select value
select lprice from tbl where sym=req
//directly index to retrieve value and create table
enlist tbl[req]
//repeat with u# applied to sym column tbl:update u#sym from mktbl 100
...
tbl:update u#sym from mktbl 10000000            |           no attribute         |    unique attribute set             |      select      |    index    |    select    |    index            rows in | time        size | time   size | time    size | time   size       table | (ms)         (b) | (ms)    (b) | (ms)     (b) | (ms)    (b)  ------------|------------------|-------------|--------------|------------         100 |   0        1,040 |   0     528 |   0    1,040 |   0     528       1,000 |   0        1,536 |   0     528 |   0    1,040 |   0     528      10,000 |   0       16,896 |   0     528 |   0    1,040 |   0     528     100,000 |   0      131,584 |   0     528 |   0    1,040 |   0     528   1,000,000 |   2    1,049,088 |   0     528 |   0    1,040 |   0     528  10,000,000 |  23   16,777,728 |   7     528 |   0    1,040 |   0     528 100,000,000 | 233  134,218,240 |  96     528 |   0    1,040 |   0     528//function to create example last value cache keyed table with x rows  //create global s to retrieve query value later  mktbl:{([sym:s::(neg x)?7];lprice:0.01*x?10000)}
//create fixed size example table
tbl:mktbl 10000000
//make dictionary containing groups of distinct syms
//drawn from our sym universe
syms:as!{(neg x)?s}each as:1 10 100 1000 10000 100000
//select value
select lprice from tbl where sym in syms 1
...
select lprice from tbl where sym in syms 100000
//directly index to retrieve value and create table
tbl each syms[1]
...
tbl each syms[100000]
//repeat with u# applied to sym q)tbl:mktbl 10000000 q)update u#sym from `tbl        |           no attribute              |      unique attribute set
|      select     |       index       |     select    |     index
sym | time       size |    time      size | time     size | time      size

values
(ms)        (b)
(ms)       (b)
(ms)      (b)
(ms)       (b)

  1 |  22  16,777,776 |       0       608 |  0      1,344 |   0        608
 10 |  77  83,886,624 |      60     1,392 |  0      1,344 |   0      1,392
100 |  77  83,886,624 |     462     9,872 |  0      6,288 |   0      9,872

1,000 |  77  83,886,624 |   7,454    88,976 |  0     53,008 |   0     88,976
10,000 |  78  83,886,624 |  48,822 1,033,616 |  7    648,208 |   7  1,033,616
100,000 |  85  83,886,624 | 484,486 9,546,128 | 90  5,821,968 |  82  9,546,128//create rt quote table with g#sym. Vary through 1-10 dates rtquote:update g#sym, #time from    time xasc select from quote where date in 1#date
...
rtquote:update g#sym, #time from
`time xasc select from quote where date in 10#date
//select all IBM data from throughout the table
select from rtquote where sym=`IBM
//repeat the process for data without g#sym rtquote:update #time from
time xasc select from quote where date in 1#date ... rtquote:update #time from
`time xasc select from quote where date in 10#date            |      no attribute   |  grouped attribute
rows in |  time          size |  time          size
table |  (ms)           (b) |  (ms)           (b)
25,000,000 |   119   301,990,304 |     8     2,228,848
50,000,000 |   243   603,980,192 |    10     4,457,072
75,000,000 |   326 1,207,959,968 |    14     8,913,520
100,000,000 |   472 1,207,959,968 |    20     8,913,520
125,000,000 |   582 1,207,959,968 |    26     8,913,520
150,000,000 |   711 2,415,919,520 |    30    17,826,416
175,000,000 |   834 2,415,919,520 |    36    17,826,416
200,000,000 |   931 2,415,919,520 |    40    17,826,416
225,000,000 | 1,049 2,415,919,520 |    46    17,826,416
250,000,000 | 1,167 2,415,919,520 |    50    17,826,416//create simulated realtime trade table sorted on time
q)rttrade:update #time from time xasc select from trade where date=first date
q)\ts select last price by sym from rttrade
24 33561280
//create simulated realtime trade table sorted on time
q)rttrade:update g#sym, #time from `time xasc select from trade where date=first date
q)\ts select last price by sym from rttrade
15 33559232
//create simulated realtime trade table sorted on time
q)rttrade:update g#sym, #time from `time xasc select from trade where date=first date
//use group on the sym column to get the indices for each sym and find
//last value in each vector. Use these index values to directly index
//into the price vector, then create a table with the results
q)\ts {1!([]sym:key x;price::value x)} rttrade[price] last each group[rttradesym]
0 15072//make dictionary containing groups of distinct syms
//drawn from our sym universe
syms:n!{(neg x)?sym}each n:1 10 100 500
//create rt quote table with g#sym rtquote:update g#sym, #time from    time xasc select from quote where date=first date
//comparing the performance of different query forms
//as we increase the number of securities requested
select first bid by sym from rtquote where sym in syms 1
...
select first bid by sym from rtquote where sym in syms 500
raze{
select first bid by sym from rtquote where sym=x
} each syms 1
...
q)raze{
select first bid by sym from rtquote where sym=x
} each syms 500      |           no attribute               |         unique attribute
|     select      |        index       |      select       |    index
sym | time       size | time          size | time        size  | time   size

vals
(ms)        (b)
(ms)           (b)
(ms)         (b)
(ms)    (b)

1 | 132 301,990,624 |    133 301,991,104 |    1      787,408 |   1 787,904

10 | 197 402,653,920 |  1,319 301,993,792 |   29   12,583,696 |  13 790,592
100 | 309 402,653,920 | 13,204 302,202,608 |  417  201,327,824 | 125 817,408
500 | 444 536,879,984 | 27,484  33,965,504 | 3,634 536,879,984 | 616 935,680

$ bitcoin-cli getblockhash 0
000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
$ bitcoin-cli getblock
"000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f" 1
{
"hash": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f",
"confirmations": 544077,
"strippedsize": 285,
"size": 285,
"weight": 1140,
"height": 0,
"version": 1,
"versionHex": "00000001",
"merkleroot": "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b",
"tx": [
"4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"
],
"time":<PHONE_NUMBER>,
"mediantime":<PHONE_NUMBER>,
"nonce":<PHONE_NUMBER>,
"bits": "1d00ffff",
"difficulty": 1,
"chainwork": "0000000000000000000000000000000000000000000000000000000100010001",
"nTx": 1,
"nextblockhash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048"
}
index:startHeight:0f
.z.ts:{[]
Hash:.bitcoind.getblockhash[index][result];   if[not 0n~Hash;     -1(string .z.p)," Processing Block: ",string[index];     processBlock[Hash];     index+:1     ];   } processBlock:{[Hash]   Block:.bitcoind.getblock[Hash;(enlist verbosity)!(enlist 2)];
}
q)meta trade

c
t f a

date
d

time
t

sym
s

price
f

size
i

stop
b

cond
c

excc
c

q)meta quote

c
c f a

date
d

time
t

sym
s

bid
f

ask
f

bsize
i

asize
i

mode
c

ex
c

q)t:select tradecount:count i by date from trade
q)q:select quotecount:count i by date from quote
q)t lj q

date
tradecount quotecount

2010.12.01
4940021    24713334

2010.12.02
4940300    24714460

2010.12.03
4940482    24708283

...

2010.12.30
4982917    24933374

2010.12.31
4994626    25000981

//set the date to be requested

d:first date

//sym before date (non-optimal order)
select from trade where sym=`IBM, date=d
//date before sym (optimal order)
select from trade where date=d, sym=`IBM
|   sym before date  |   date before sym
dates in |  time         size |  time         size

database
(ms)          (b)
(ms)          (b)

 1   |   470   75,499,920 |    78   75,499,984
 5   |   487   75,878,400 |    78   75,499,984
10   |   931   75,880,624 |    78   75,499,984
15   | 1,209   75,882,912 |    78   75,499,984
20   | 1,438   75,885,072 |    78   75,499,984

//make dictionary containing groups of distinct syms
//drawn from our sym universe
syms:n!{(neg x)?sym}each n: 1 10 100 500
//make call for open prices for ranges of security drawn from the dictionary
select open:first price by sym from trade
where date=first date, sym in syms 1
select open:first price by sym from trade
where date=first date, sym in syms 500
//create ohlc table with a number of different day-level aggregations,
//and save to disk using dpft function.
//perform across each date in the db
{ohlc::0!select
open:first price,
high:max price,
low:min price,
close: last price,
vwap: size wavg price by sym from trade where date= x;
.Q.dpft[`:.;x;`sym;`ohlc]; } each date
//This will create one row per security/date pair.
q)select count i by date from ohlc

date
x

2010.12.01
500

2010.12.02
500

...

2010.12.30
500

2010.12.31
500

//reload db
q)\l .
//repeat the same requests, but target our new aggregation table.
q)select open by sym from ohlc where date=first date, sym in syms 1
...
q)select open by sym from ohlc where date=first date, sym in syms 500
|      from trade     |      from ohlc

sym values
time (ms)  size (b)
time (ms)  size (b)

    1  |      1      265,056 |      0        2,896
   10  |     13    2,100,224 |      0        3,504
  100  |     53   16,781,680 |      0       12,400
  500  |    125  134,228,736 |      0       48,752

//perform the same query over widening date windows
//restart kdb+ and flush os cache between each request
select size wavg price by sym from trade where date in 5#date
...
select size wavg price by sym from trade where date in 20#date
dates in query  time (ms)
   5         1,863
  10         3,152
  15         4,596
  20         7,597

//add p attribute using setattrcol from dbmaint.q //NOTE: setattrcol does not adjust the order of the data.  //Our data is already segmented by sym. setattrcol[:.;trade;sym;`p]
\l .
//set the date to be requested
d:first date
//sym before date (non-optimal order)
select from trade where sym=`IBM, date=d
//date before sym (optimal order)
select from trade where date=d, sym=`IBM
|       no attribute (from example)  |         parted attribute set
| sym before date  | date before sym | sym before date | date before sym
dates |  time      size  | time      size  |  time     size  |  time   size

in db
(ms)       (b)
(ms)       (b)
(ms)      (b)
(ms)    (b)

1
470 75,499,920
78  75,499,984
3    445,024
0    445,088

5
487 75,878,400
78  75,499,984
24    889,936
0    445,088

10
931 75,880,624
78  75,499,984
26    892,816
0    445,088

15
1,209 75,882,912
78  75,499,984
64    896,336
0    445,088

20
1,438 75,885,072
78  75,499,984
177    898,576
0    445,088

//create time we will use for test requests

t:first 09:30:00.000+1?07:00:00.000

//create temporary table using one or more days of saved data
// – vary from 1 through 10 on each iteration
rtquote:{
time xasc select from quote where date in x#date   }1  rtquote:{   time xasc select from quote where date in x#date
}10
//make requests against rtquote for IBM data.
//Repeat as the rtquote table grows
select from rtquote where time=t, sym=`IBM
//repeat process with s removed from time vector rtquote:update #time from{
time xasc select from quote where date in x#date   }1 rtquote:update #time from{
`time xasc select from quote where date in x#date
}10
|    no attribute   | sorted attribute
rows in     | time         size | time        size

table
(ms)          (b)
(ms)         (b)

25,000,000
49   33,554,944
0       1,248

50,000,000
102   67,109,392
0       1,248

75,000,000
154  134,218,288
0       1,248

100,000,000
213  134,218,288
0       1,248

125,000,000
247  134,218,288
0       1,248

150,000,000
299  268,436,016
0       1,248

200,000,000
406  268,436,016
0       1,248

225,000,000
523  268,436,016
0       1,248

250,000,000
568  268,436,016
0       1,248

//function to create example last value cache keyed table with x rows

//also creates global s to retrieve query value later

mktbl:{([sym:s::(neg x)?`7];lprice:0.01*x?10000)}

//make example table, vary through the values on each iteration
tbl:mktbl 100
...
tbl:mktbl 10000000
//create test value
req:first 1?s
//select value
select lprice from tbl where sym=req
//directly index to retrieve value and create table
enlist tbl[req]
//repeat with u# applied to sym column tbl:update u#sym from mktbl 100
...
tbl:update u#sym from mktbl 10000000            |           no attribute         |    unique attribute set             |      select      |    index    |    select    |    index            rows in | time        size | time   size | time    size | time   size       table | (ms)         (b) | (ms)    (b) | (ms)     (b) | (ms)    (b)  ------------|------------------|-------------|--------------|------------         100 |   0        1,040 |   0     528 |   0    1,040 |   0     528       1,000 |   0        1,536 |   0     528 |   0    1,040 |   0     528      10,000 |   0       16,896 |   0     528 |   0    1,040 |   0     528     100,000 |   0      131,584 |   0     528 |   0    1,040 |   0     528   1,000,000 |   2    1,049,088 |   0     528 |   0    1,040 |   0     528  10,000,000 |  23   16,777,728 |   7     528 |   0    1,040 |   0     528 100,000,000 | 233  134,218,240 |  96     528 |   0    1,040 |   0     528//function to create example last value cache keyed table with x rows  //create global s to retrieve query value later  mktbl:{([sym:s::(neg x)?7];lprice:0.01*x?10000)}
//create fixed size example table
tbl:mktbl 10000000
//make dictionary containing groups of distinct syms
//drawn from our sym universe
syms:as!{(neg x)?s}each as:1 10 100 1000 10000 100000
//select value
select lprice from tbl where sym in syms 1
...
select lprice from tbl where sym in syms 100000
//directly index to retrieve value and create table
tbl each syms[1]
...
tbl each syms[100000]
//repeat with u# applied to sym q)tbl:mktbl 10000000 q)update u#sym from `tbl        |           no attribute              |      unique attribute set
|      select     |       index       |     select    |     index
sym | time       size |    time      size | time     size | time      size

values
(ms)        (b)
(ms)       (b)
(ms)      (b)
(ms)       (b)

  1 |  22  16,777,776 |       0       608 |  0      1,344 |   0        608
 10 |  77  83,886,624 |      60     1,392 |  0      1,344 |   0      1,392
100 |  77  83,886,624 |     462     9,872 |  0      6,288 |   0      9,872

1,000 |  77  83,886,624 |   7,454    88,976 |  0     53,008 |   0     88,976
10,000 |  78  83,886,624 |  48,822 1,033,616 |  7    648,208 |   7  1,033,616
100,000 |  85  83,886,624 | 484,486 9,546,128 | 90  5,821,968 |  82  9,546,128//create rt quote table with g#sym. Vary through 1-10 dates rtquote:update g#sym, #time from    time xasc select from quote where date in 1#date
...
rtquote:update g#sym, #time from
`time xasc select from quote where date in 10#date
//select all IBM data from throughout the table
select from rtquote where sym=`IBM
//repeat the process for data without g#sym rtquote:update #time from
time xasc select from quote where date in 1#date ... rtquote:update #time from
`time xasc select from quote where date in 10#date            |      no attribute   |  grouped attribute
rows in |  time          size |  time          size
table |  (ms)           (b) |  (ms)           (b)
25,000,000 |   119   301,990,304 |     8     2,228,848
50,000,000 |   243   603,980,192 |    10     4,457,072
75,000,000 |   326 1,207,959,968 |    14     8,913,520
100,000,000 |   472 1,207,959,968 |    20     8,913,520
125,000,000 |   582 1,207,959,968 |    26     8,913,520
150,000,000 |   711 2,415,919,520 |    30    17,826,416
175,000,000 |   834 2,415,919,520 |    36    17,826,416
200,000,000 |   931 2,415,919,520 |    40    17,826,416
225,000,000 | 1,049 2,415,919,520 |    46    17,826,416
250,000,000 | 1,167 2,415,919,520 |    50    17,826,416//create simulated realtime trade table sorted on time
q)rttrade:update #time from time xasc select from trade where date=first date
q)\ts select last price by sym from rttrade
24 33561280
//create simulated realtime trade table sorted on time
q)rttrade:update g#sym, #time from `time xasc select from trade where date=first date
q)\ts select last price by sym from rttrade
15 33559232
//create simulated realtime trade table sorted on time
q)rttrade:update g#sym, #time from `time xasc select from trade where date=first date
//use group on the sym column to get the indices for each sym and find
//last value in each vector. Use these index values to directly index
//into the price vector, then create a table with the results
q)\ts {1!([]sym:key x;price::value x)} rttrade[price] last each group[rttradesym]
0 15072//make dictionary containing groups of distinct syms
//drawn from our sym universe
syms:n!{(neg x)?sym}each n:1 10 100 500
//create rt quote table with g#sym rtquote:update g#sym, #time from    time xasc select from quote where date=first date
//comparing the performance of different query forms
//as we increase the number of securities requested
select first bid by sym from rtquote where sym in syms 1
...
select first bid by sym from rtquote where sym in syms 500
raze{
select first bid by sym from rtquote where sym=x
} each syms 1
...
q)raze{
select first bid by sym from rtquote where sym=x
} each syms 500      |           no attribute               |         unique attribute
|     select      |        index       |      select       |    index
sym | time       size | time          size | time        size  | time   size

vals
(ms)        (b)
(ms)           (b)
(ms)         (b)
(ms)    (b)

1 | 132 301,990,624 |    133 301,991,104 |    1      787,408 |   1 787,904

10 | 197 402,653,920 |  1,319 301,993,792 |   29   12,583,696 |  13 790,592
100 | 309 402,653,920 | 13,204 302,202,608 |  417  201,327,824 | 125 817,408
500 | 444 536,879,984 | 27,484  33,965,504 | 3,634 536,879,984 | 616 935,680


Add right click to change databases

Add an ability to change database context via right click option instead of requiring a new query

Since version 1.7.30, the query file will bind database, you change database when switch query file.

But if I have a query file with scripts in it and I want to run the same scripts on different databases that I am connected to,  would need to have separate files for each database or I would have to fully qualify the database names and change them between executions.  It would be easier if I could just right click on a server and say "Select DB" and then it would run in that context.  Then pick the next database, and run the same script in the same file.  Rather than have multiple instances of the same scripts, just in different files to maintain the DB context.  Could that be done?

It's a good idea to add switching database context on the menu, I'll add this feature.

I add this feature since version 1.8.0.

@cweijan I tried using this feature but when I right click on a database it doesn't have a "Select DB" option.  I verified I am on 1.9.1 so it should include this feature.  Can you clarify how this should work?


Showcase code is not working

In the main README.md file, seems like the code isn't working properly in the CodePen.
Motion Graphics for the Web
Bubble Layout

Hi @PricelessCode 👋
Thanks for pointing this out, unfortunately we don't have hand on @legomushroom codepens.
Despite the fact you access all the codepen codebase, those snippets have been built years ago.. so assembling part of code with the new version could lead to unrecoverable errors / issues.
I suggest that you start your animation from scratch or use a different "working" codepen.
We are really sorry about that.

I remember @Sandstedt previously answered about broken codepens? 🤔

@PricelessCode If I remember correctly, those pens where really hard to convert. But if you check the tutorials and the documentation site on https://mojs.github.io/, all those examples should work just fine.

Thanks @Sandstedt 🙏 , I remember now.
Additional infos: at the time of writing this codepen, Oleg was using a minified version of mojs that was hosted on his private cloud, dropbox or something, and since this version is not reachable anymore, the codepen fails to run properly. I am seeing that in some cases there are bug syntax or missing code, and in this case we can do nothing.
I am closing this issue since, from my point of view, those pens are currently unrecoverable 🐛


NARS Report - Integration with Enterprise Data Warehouse

The NARS is a massive, really all encompassing, annual report that we send to the federal government as a part of our admin services for running their program. It includes all inputs and all outputs (assessment, award, withdrawal and a number of others) for every single application that comes in within a program year. Changes to the data and formatting of this report is (I think) included within the scope of Program Year changes, and comes in periodically (not necessarily every year). This is the foundational report from which I believe a number of other business reports are generated, including the reports that go to institutions.
The report pulls data from SFAS and SAIL but is generated by EDW.
Jason – FYI given your upcoming work with EDW on integration, this is something that will likely come up as a key deliverable driving the integration work (e.g. a “critical” report that they generate).

@HRAGANBC @CarlyCotton @Joshua-Lakusta  What Information to we have to provide the Feds in this report. Can you please turn this into a User Story- As FED Partner User.. I need this information annually  so that I can .....

Closing as this will be covered as part of the EDW integration. EDW produces the NARS report


Update Setup-Your-Mac-via-Dialog.bash

Adding a check for the Dock running.  This allows the script to pause here and wait for the user to be fully logged in if the script is started at the Enrollment Complete trigger.

Please also use echo_logger in place of echo.


How do you extract blob from response?

Hello!
First of all, I love this library. Very nice.
I have probably just a simple question. I didn't find it in the Documentation. And I tried different syntaxes but never worked.
How do you extract a Blob from an Eden response?
I.e. if you have an endpoint returning an Image body:
          // In FETCH:
          const response = await fetch(`/readfile/${encodeURIComponent(path)}`); // TODO: This has the .blob(); that idk how to use in Eden.
          if (response.ok) {
            const blob = await response.blob();
            setImage(blob);
          }
          // In EDEN:
          // const response = await server.readfile[encodeURIComponent(path)].get();
          // ??

For context, setImage is used like this in React
  const [image, setImage] = useState<Blob | null>(null);
  {image && <img src={URL.createObjectURL(image)} alt="Loaded from server" />}

By default Eden would try to parse the response body and put it to response.data. It is not hundred percent accurate, for some cases we can use onResponse in eden treaty config to intercept responses and parse it the way we want to.
https://elysiajs.com/eden/treaty/config.html#onresponse
In my case I am returning FormData from elysia endpoint, but Eden only parses the form fields using get. The field named image can contain one or more images so I intercept the response and unwrap the File/Blob using getAll:
export const treaty = treaty2<App>(this.domain.replace('http://', '').replace('https://', ''), {
  async onResponse(response) {
    const contentType = response.headers.get('Content-Type')
    if (contentType?.includes('multipart/form-data')) {
      const formData = await response.formData()
      if (formData.has('image')) {
        return formData.getAll('image')
      }
    }
  }
})

Usage in client is like this:
const treatyResponse = await apiTreaty.image.convert.post({
  image: formData.getAll('image') as unknown as File | FileList
}, {
  query: {
    format: format.value as typeof ImageFormatSchema.static
  }
})

console.log(treatyResponse.data)


Made formatting of the JS questions uniform

Checklist:

[x] I have read freeCodeCamp's contribution guidelines.
[x] My pull request has a descriptive title (not a vague title like Update index.md)

Closes #XXXXX

Some of the questions for JS were all on one line but most were in the following format.
Question:
  "The question"

They are all like the above format now.

HI @not-a-ethan !
Thank you for your contribution.
The formatting is done automatically by the Prettier which means some of the questions will be on one line while others won't.
I am going to close this PR 👍


RegulatoryRequirement should be a repeated string

This would make it easy to list multiple regulatory requirements that a model should follow.
https://github.com/cylynx/verifyml/blob/main/verifyml/model_card_toolkit/proto/model_card.proto#L65

Fixed in #58


ddynamic_reconfigure: 0.1.7-0 in 'kinetic/distribution.yaml' [bloom]

Increasing version of package(s) in repository ddynamic_reconfigure to 0.1.7-0:

upstream repository: https://github.com/pal-robotics/ddynamic_reconfigure.git
release repository: https://github.com/pal-gbp/ddynamic_reconfigure.git
distro file: kinetic/distribution.yaml
bloom version: 0.6.9
previous version for package: null

ddynamic_reconfigure
* Merge branch 'fix-gmock-dependency' into 'erbium-devel'
  Rename gmock dependency for public release
  See merge request control/ddynamic_reconfigure!11
* Rename gmock dependency for public release
* Contributors: Victor Lopez

Holding for sync


Add signaling to properly handle SIG* interrupts.

Currently, when we want to close the running origami server we have to do it forcefully which leaves a traceback. Adding a SIGTERM handler will help us properly close running server by first stopping the running Tornado server and then exiting gracefully.

Since this a package, I don't think it is a good idea to implement signaling inside it. I had something else in mind when I created the issue. Closing this for now.
Though if anyone has a strong reason to support this we can implement this.


Added 18 quotes

Wonderful stuff, thank you :) I hope to deploy this in the next couple of days, I just need to review #3 this evening.

Ok, nice ;)


prompt for an extrapoint when there is no need.

As you can see in the image both have exactly four points, but software prompts for an extra point why?
NOTE: Note intended as repost/duplicate my other post has the whole host of other issues.

Uhm.......
HELLO FROM THE OTHER SIIIIIIIIIIIDE, I'm still wondering why an extra Pooooooint.

Can you post the 2 pathData strings that you are using?
i.e. The ones that you see in here:

I thought it's was a rouge point too. so I double checked, the svg files. there wasn't any!
to make sure I recreated both icons, flattened it and exported it and finally checked the svg file for an extra point. please take a look at the files.
mpv.zip
From : M 12 2 L 12 14 C 10.343 14 9 12.657 9 11 L 9 5 C 9 3.343 10.343 2 12 2 Z
To      : M 6 5 L 10 5 L 10 19 L 6 19 Z

Well I'll just sing that same Old Soooooooong, hitting my head against the waaaaaaaaaall.
sorry just getting my adele on, I know you must busy but, any thoughts on the issue.

Hi, any thoughts? also happy new year

who is there?
It's me again

still here

another year, another hi


Prevent Electron-based applications from being detected as node.js

If we are in an Electron-based application, the browser process should have node.js Zen Nature, while other ones (renderer) should be treated as if we are in a browser

lgtm.

If it claims to be node it should work with the node apis that are being used. Why would it not work?

If it claims to be node it should work with the node apis that are being used. Why would it not work?

I don't have the original crash trace, but it would crash on startup here: https://github.com/petkaantonov/bluebird/blob/31d02b46a750151ebba072a8ec455f818c1d0844/src/async.js#L19 with schedule being null.
This is probably because https://github.com/petkaantonov/bluebird/blob/634af0e27ff4faab62c6c5bfd105527abcf8b06e/src/schedule.js#L7 doesn't really grok things that are node but also have MutationObservers.
Philosophically, in the render process, we should try to think of it as Chrome as much as possible, this usually results in The Right Thing happening, rather than embracing node.js and trying to constantly patch away the "Oh, node.js! No DOM here!" type of decisions.

Its impossible for schedule to be null

Its impossible for schedule to be null

¯_(ツ)_/¯
Here's the minified stack trace, this crashes Electron 100% of the time for us.
TypeError: Cannot read property 'isStatic' of undefined
  at new Async (https://A URL/js/libs/bluebird-2.9.12.v1434403486.js:69:17)
  at Object.2../queue.js (https://A URL/js/libs/bluebird-2.9.12.v1434403486.js:157:18)
  at s (https://A URL/js/libs/bluebird-2.9.12.v1434403486.js:29:579)
  at https://A URL/js/libs/bluebird-2.9.12.v1434403486.js:29:630
  at module.exports (https://A URL/js/libs/bluebird-2.9.12.v1434403486.js:2119:13)
  at Object.4../promise.js (https://A URL/js/libs/bluebird-2.9.12.v1434403486.js:244:39)

The latest version is 2.9.30, not 2.9.12.

@petkaantonov :cool:

What does that mean? You are testing with a very old version while making a PR to master.

@petkaantonov We're just gonna maintain a patched version then, thanks for having a look!

Paul, that makes no sense, what's preventing you from using 2.30?
On Fri, Jul 3, 2015 at 9:37 PM, Paul Betts<EMAIL_ADDRESS>wrote:

Closed #658 https://github.com/petkaantonov/bluebird/pull/658.
—
Reply to this email directly or view it on GitHub
https://github.com/petkaantonov/bluebird/pull/658#event-347571232.


After npm install, there's no module in Android studio

And then include it manually, get this:

What went wrong:
Task 'GenerateNodeNativeAssetsListsarmeabi-v7a' not found in project ':nodejs-mobile-react-native'.

Env:
"nodejs-mobile-react-native": "^16.17.5",
"react-native": "0.71.3",
Android Studio Electric Eel | 2022.1.1 Patch 1

Forgotten abifilter config


18776 add diamond icon to konturio default icons

https://kontur.fibery.io/Tasks/My-tasks-3575#Task/Add-diamond-icon-to-konturio-default-icons-18776

Summary by CodeRabbit

New Features

Introduced a new Diamond24 icon component for rendering a diamond-shaped icon.
Added a memoized version of the Diamond24 component for improved performance.

Documentation

Updated the README to include instructions for setting the Figma token on Windows.

Please add chore(ticketnumber) to PR name


bug exists in the ArtifactServiceApi.get_artifact_file  for sdks/python

Pre-requisites

[X] I have double-checked my configuration
[X] I can confirm the issues exists when I tested with :latest
[ ] I'd like to contribute the fix myself (see contributing guide)

What happened/what you expected to happen?
I want to get the pod log file through the get_artifact_file interface.
api_instance.get_artifact_file(namespace=namespace, id_discriminator=id_discriminator,
                                                          id=id, node_id=node_id, artifact_name=artifact_name,
                                                          artifact_discriminator=artifact_discriminator)

The parameter validation setting of this interface is incorrect. When I used id_discriminator="workflows" or "archived-workflows", I received an error( argo_workflows.exceptions.ApiValueError: Invalid value for id_discriminator (archived-workflows), must be one of ['workflow', 'archived-workflows '])
But exectly, the value for id_discriminator should be workflows or archived-workflows , since I can get the result through the initial interface when I use idDiscriminato=workflows or archived-workflows
curl http://<IP_ADDRESS>:2746/artifact-files/{namespace}/{idDiscriminator}/{id}/{nodeId}/{artifactDiscriminator}/{artifactName}.

So the solution is:
Modifying the code of line 81 & 82 in artifact_service_api.py
'allowed_values': {
    ('id_discriminator',): {

        "WORKFLOW": "workflow",## should be workflows
        "ARCHIVED-WORKFLOWS_": "archived-workflows " ## should be archived-workflows， remove spaces
     },

Version
latest( argo-workflows==6.4.5)
Paste a small workflow that reproduces the issue. We must be able to run the workflow; don't enter a workflows that uses private images.
import time
import argo_workflows
from argo_workflows.api import artifact_service_api
from argo_workflows.model.grpc_gateway_runtime_error import GrpcGatewayRuntimeError
from pprint import pprint
# Defining the host is optional and defaults to http://localhost:2746
# See configuration.py for a list of all supported configuration parameters.
configuration = argo_workflows.Configuration(
    host = "http://localhost:2746"
)

# The client must configure the authentication and authorization parameters
# in accordance with the API server security policy.
# Examples for each auth method are provided below, use the example that
# satisfies your auth use case.

# Configure API key authorization: BearerToken
configuration.api_key['BearerToken'] = 'YOUR_API_KEY'

# Uncomment below to setup prefix (e.g. Bearer) for API key, if needed
# configuration.api_key_prefix['BearerToken'] = 'Bearer'

# Enter a context with an instance of the API client
with argo_workflows.ApiClient(configuration) as api_client:
    # Create an instance of the API class
    api_instance = artifact_service_api.ArtifactServiceApi(api_client)
    namespace = "namespace_example" # str |
    id_discriminator = "workflows" # str |
    id = "id_example" # str |
    node_id = "nodeId_example" # str |
    artifact_name = "artifactName_example" # str |

    # example passing only required values which don't have defaults set
    try:
        # Get an artifact.
        api_response = api_instance.get_artifact_file(namespace, id_discriminator, id, node_id, artifact_name, )
        pprint(api_response)
    except argo_workflows.ApiException as e:
        print("Exception when calling ArtifactServiceApi->get_artifact_file: %s\n" % e)

Logs from the workflow controller
api_response = api_instance.get_artifact_file(namespace=namespace, id_discriminator=id_discriminator,
  File "venv/lib/python3.8/site-packages/argo_workflows/api/artifact_service_api.py", line 490, in get_artifact_file
    return self.get_artifact_file_endpoint.call_with_http_info(**kwargs)
  File "venv/lib/python3.8/site-packages/argo_workflows/api_client.py", line 830, in call_with_http_info
    self.__validate_inputs(kwargs)
  File "venv/lib/python3.8/site-packages/argo_workflows/api_client.py", line 705, in __validate_inputs
    check_allowed_values(
  File "venv/lib/python3.8/site-packages/argo_workflows/model_utils.py", line 872, in check_allowed_values
    raise ApiValueError(
argo_workflows.exceptions.ApiValueError: Invalid value for `id_discriminator` (archived-workflows), must be one of ['workflow', 'archived-workflows ']

Logs from in your workflow's wait container
api_response = api_instance.get_artifact_file(namespace=namespace, id_discriminator=id_discriminator,
  File "venv/lib/python3.8/site-packages/argo_workflows/api/artifact_service_api.py", line 490, in get_artifact_file
    return self.get_artifact_file_endpoint.call_with_http_info(**kwargs)
  File "venv/lib/python3.8/site-packages/argo_workflows/api_client.py", line 830, in call_with_http_info
    self.__validate_inputs(kwargs)
  File "venv/lib/python3.8/site-packages/argo_workflows/api_client.py", line 705, in __validate_inputs
    check_allowed_values(
  File "venv/lib/python3.8/site-packages/argo_workflows/model_utils.py", line 872, in check_allowed_values
    raise ApiValueError(
argo_workflows.exceptions.ApiValueError: Invalid value for `id_discriminator` (archived-workflows), must be one of ['workflow', 'archived-workflows ']

cc @flaviuvadan


Error reading the trained model with h5py

Hi,
This is an odd error that is occurring to me on Linux Ubuntu. When I tried to run the predictor I get this error:
 Running EqTransformer  0.1.56  *** Loading the model ... Traceback (most recent call last):   File "run_OVtrans.py", line 67, in <module>     main()   File "run_OVtrans.py", line 57, in main     OVT.makePredictor(trained_model='EqT_model.h5')   File "/home/esteban/Work/EQtrans/C.Pacific/OV_transformer.py", line 94, in makePredictor     number_of_cpus=10,   File "/home/esteban/anaconda3/envs/eqt/lib/python3.7/site-packages/EQTransformer/core/predictor.py", line 227, in predictor     'f1': f1   File "/home/esteban/anaconda3/envs/eqt/lib/python3.7/site-packages/keras/engine/saving.py", line 492, in load_wrapper     return load_function(*args, **kwargs)   File "/home/esteban/anaconda3/envs/eqt/lib/python3.7/site-packages/keras/engine/saving.py", line 583, in load_model     with H5Dict(filepath, mode='r') as h5dict:   File "/home/esteban/anaconda3/envs/eqt/lib/python3.7/site-packages/keras/utils/io_utils.py", line 191, in __init__     self.data = h5py.File(path, mode=mode)   File "/home/esteban/anaconda3/envs/eqt/lib/python3.7/site-packages/h5py/_hl/files.py", line 408, in __init__     swmr=swmr)   File "/home/esteban/anaconda3/envs/eqt/lib/python3.7/site-packages/h5py/_hl/files.py", line 173, in make_fid     fid = h5f.open(name, flags, fapl=fapl)   File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper   File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper   File "h5py/h5f.pyx", line 88, in h5py.h5f.open OSError: Unable to open file (file signature not found)
If I try to read the h5 file individually, I get the same OSErorr:
`In [3]: h5py.File("EqT_model.h5", "r")
OSError                                   Traceback (most recent call last)
 in ()
----> 1 h5py.File("EqT_model.h5", "r")
/home/esteban/anaconda3/envs/eqt/lib/python3.7/site-packages/h5py/_hl/files.py in init(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, **kwds)
406                 fid = make_fid(name, mode, userblock_size,
407                                fapl, fcpl=make_fcpl(track_order=track_order),
--> 408                                swmr=swmr)
409
410             if isinstance(libver, tuple):
/home/esteban/anaconda3/envs/eqt/lib/python3.7/site-packages/h5py/_hl/files.py in make_fid(name, mode, userblock_size, fapl, fcpl, swmr)
171         if swmr and swmr_support:
172             flags |= h5f.ACC_SWMR_READ
--> 173         fid = h5f.open(name, flags, fapl=fapl)
174     elif mode == 'r+':
175         fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
h5py/_objects.pyx in h5py._objects.with_phil.wrapper()
h5py/_objects.pyx in h5py._objects.with_phil.wrapper()
h5py/h5f.pyx in h5py.h5f.open()
OSError: Unable to open file (file signature not found)`
What's weird, is that, it only shows up when I execute it on a Linux server. On my macbook pro, predictor works just fine.
I tried updating and downgrading the version of the h5py through anaconda, but didn't work.

did you try downgrading your TensorFlow or Keras version as well? Sometimes similar things happen that will be fixed by only reinstalling the TensorFLow.

Will try this a let you know.


fix: pass valid SQL to SM

SUMMARY

When creating a virtual dataset we pass the unprocessed SQL to security manager. If the SQL contains Jinja it might not be valid and might fail to parse, which causes the security manager to allow the query to run (see https://github.com/apache/superset/pull/26748).
This PR fixes it by processing the templated SQL before passing it to the security manager. See the discussion in https://github.com/apache/superset/pull/26476.
BEFORE/AFTER SCREENSHOTS OR ANIMATED GIF

N/A
TESTING INSTRUCTIONS

ADDITIONAL INFORMATION

[ ] Has associated issue:
[ ] Required feature flags:
[ ] Changes UI
[ ] Includes DB Migration (follow approval process in SIP-59)

[ ] Migration is atomic, supports rollback & is backwards-compatible
[ ] Confirm DB migration upgrade and downgrade tested
[ ] Runtime estimates and downtime expectations provided

[ ] Introduces new feature or API
[ ] Removes existing feature or API

Thanks @betodealmeida for authoring this fix. Thinking a bit about this over the weekend, I think this—a two phase approach—is likely the right technique.
Interestingly previously using sqlparse (3.0.3) the following queries returned different results,
>>> from superset.sql_parse import ParsedQuery

>>> ParsedQuery("SELECT * FROM foo WHERE ds = '{{ presto.latest_partition('foo') }}'").tables
{Table(table='foo', schema=None, catalog=None)}

>>> ParsedQuery("SELECT '{{ presto.latest_partition('foo') }}'").tables
set()

i.e., Superset only identified the foo table from the FROM clause and didn't identify the table from within the Jinja macro.
Finally, @betodealmeida regarding your comment,

When creating a virtual dataset we pass the unprocessed SQL to security manager.

This could also be a physical dataset which contains a freeform SQL filter containing a Jinja macro.

Regarding https://github.com/apache/superset/pull/27464#discussion_r1520265947 and https://github.com/apache/superset/pull/27464#discussion_r1520393408, I was thinking maybe an alternative approach would be a multi-phase approach for the ParsedQuery table extraction where:

Phase I: Identify all the Jinja macros—hopefully by leveraging Jinja functionality—and extract any table names present, i.e., we would need to parse (likely via SQLGlot) the macros

>>> from sqlglot import parse
>>> parse("trino.latest_partition('foo')")
[Dot(
  this=Identifier(this=trino, quoted=False),
  expression=Anonymous(
    this=latest_partition,
    expressions=[
      Literal(this=foo, is_string=True)]))]

Phase II: Replace the Jinja macros with some SQL placeholder* and then parse the full query to extract any remaining table names.

The extracted set of tables is then the union of phase I and phase II.
* I wonder if NULL would work for all scenarios. We don't need to generate SQL which is actually executable, but merely SQL which SQLGlot will be able to successfully parse.

@john-bodley why not simply translate the macro (by executing queries, if necessary) in order to have the actual query that's going to run, so we can check for all the tables used? I think it's the best way to prevent a tight coupling between Jinja templates and SQL parsing, and I'm worried that with the approach you're proposing we might have errors in the future when people introduce new macros and the logic to extract tables misses them.
Is your concern strictly performance (ie, running a query that might be expensive only for the security manager to block the main query)? Or are you worried that it might leak metadata, eg, with a query like this, where the user doesn't have access to table bar:
SELECT '{{ latest_partition('bar') }}';

Is a user knowing the latest partition of a table they don't have access to a problem? One solution to this problem would be checking the macro queries against the security manager as well, so the process would be:

Render all macros
When macros like latest_parititon render, they run their queries against the security manager before executing them
Run the rendered query against the security manager
Run query

Though it again requires new macros to adopt this pattern (of doing step 2) if we want to keep Superset secure.

@betodealmeida I raised the issue with simply rendering all macros here. The challenge is the [hive|presto|trino].latest[_sub]_partition macros likely will resolve to Hive, Presto, or Trino queries which need to be executed, however the user may not have access to the underlying table(s)—hence my chicken-and-egg statement. I don't believe it's viable to first run a precursor query in order to then resolve which tables are associated with the query and whether said user has access to them.
The logic I outlined in https://github.com/apache/superset/pull/27470 doesn't render all macros, which @michael-s-molina, @tobyma, et. al have correctly pointed out could be problematic—especially if there's SQL injection which likely would come from the url_param macro.
Regarding your four steps I think they would need to be tweaked—in essence a hybrid of the logic in this PR and https://github.com/apache/superset/pull/27470 as the latest_[_sub]_parititon macros need special  treatment. Specifically reordering/tweaking steps 1 and 2 and tweaking steps 3 and 4:

Via Jinja extract the table referenced in the latest_[_sub]_parititon macro and replace the contents with some benign query.
Render all remaining macros.
Extract the tables from the rendered query and combine with 1.
Verify that via the security manager that the user has access to said tables.
Run query.

https://github.com/apache/superset/pull/27470 currently defined steps 1, 3, 4, and 5. The only missing part is 2, rendering the SQL query to resolve any remaining macros.

For reference here's another place in code (for right or wrong) which renders the Jinja template (for validation purposes), yet faces the same chicken-and-egg situation where any latest[_sub]_partition macros are executed prior to checking if the user has access.


Remove create store name from run all

The run_all.sh script is primarily for out automated tests and we will reuse the same old bucket for that. I've left it around for people to use to generate a name and fixed the problem where it would override the name of the bucket you set in the env.

/lgtm


GCC fails because argument list created by nix is too long

Quick summary

nix blows up number of -L flags passed to GCC
GCC passes all -L flags to its subprogram cc1 via an environment variable
If that's longer than 128 KB, then everything crashes with E2BIG

Details
So once again, I'm building some Haskell with lots of library dependencies.
Trying to upgrade this build to 18.03, the nix-build of the Haskell package newly fails with
Setup: Missing dependencies on foreign libraries:
* Missing C libraries: glog

This error message is wrong (cabal doesn't correctly propagate the argument list too long error shown below; I filed a bug about it being misleading at https://github.com/haskell/cabal/issues/5355); the dependency exists.
The problem is that because in some eventual gcc invocation the amount of arguments passed to GCC turns out to be very long.
strace confirms:
strace -fye execve -s 100000000 -v runhaskell Setup.hs build 2>&1 | grep E2BIG

execve("cc1", [arguments here], [env vars here, "COLLECT_GCC_OPTIONS='-fno-stack-protector -L... 150KB of -L flags here'"] = -1 E2BIG (Argument list too long)

E2BIG (Argument list too long), in this case because COLLECT_GCC_OPTIONS is longer than 128 KB (32 * 4 KB pages, see here, and a repro script  I made here).
What is the COLLECT_GCC_OPTIONS environment variable? It is an environment variable set by gcc before calling out to cc1, over which it communicates flags to cc1. Most (if not all?) flags given to gcc will make it into this variable. So it can grow very big (easily larger than the 128 KB limit, especially on nix).
Note that even flags given in a "response file" via gcc @myresponsefile.rsp (which was designed to pass GCC flags via a file instead of command line args to circumvent command line arg limits) will be put into COLLECT_GCC_OPTIONS by gcc itself to communicate them to cc1 (I have just confirmed that with a small example on my Ubuntu 16.04). So using @myresponsefile.rsp is not a workaround. (Yes, this seems to defeat the purpose of response files, but I suspect those were originally made to circumvent a much smaller limit of command line argument on Windows, where the limit is well below the 128 KB limit for environment variable lengths on Linux).

Aside: nix inflates the number of -L flags by the fact that each -L option to gcc is present multiple times, but those duplicates make only for factor 4x or so; even if they were deduplicated, I'd already be at half of MAX_ARG_STRLEN with my medium-size Haskell project; so if I added a couple more dependencies to my Haskell project (all recursive nix Haskell dependencies make it as -L options into the gcc command line), I'd quickly exceed that limit again even without duplication.

Problems to fix

Deduplicating the -L flags passed to GCC will help the issue by a small constant factor and make a couple more projects compile, but won't help with projects with many dependencies.
The fundamental issue seems to be a GCC problem (its way of passing arbitrarily sized information via environment variables to a direct child program doesn't work), so technically it's not nix's department. But we need to do something about it, because otherwise we can't build large Haskell projects (on nix or otherwise).

Steps to reproduce

Build a Haskell project with lots of dependencies on nixpkgs 18.03.

update: and lots of native C dependencies, and lots of executable sections in the cabal file, see comment below

Environment

on top of nixpkgs commit a0b977bdb461a756047adb1dcbb70d7c106507da; tested on both NixOS and Ubuntu 16.04

Build a Haskell project with lots of dependencies on nixpkgs 18.03.

How many dependencies are we talking here? I've worked with quite a few without error.

CCing people that were in the past involved in dealing with large amount of args to gcc (#26554, #26974, #27609, #27657):
@domenkozar @edolstra @orivej @copumpkin @Ericson2314 @ryantrinkle

Could we use GCC spec files instead of command line arguments? This wouldn't work with Clang, but creating a Clang wrapper for supporting spec files is something that has sounded useful for a while now.

Deduplicating the -L flags passed to GCC will help the issue by a small constant factor and make a couple more projects compile, but won't help with projects with many dependencies.

I am relatively certain that this involves inserting an ordNub into this line: https://github.com/haskell/cabal/blob/db05f8dd42bf28bfe9afa7992f3ca51e0f1af0c1/Cabal/Distribution/Simple/Configure.hs#L1684
just as in the line below that. I have confirmed that this line produces the duplicates.

Could we use GCC spec files instead of command line arguments?

@ElvishJerricco I don't know that. It may well be that gcc still passes options to cc1 via the env var if we do it, but I'm not familiar with them.
Do you know how to use them? If yes, could you make a small example that we can strace to check?

I can see the duplicated -L paths already in echo $NIX_LDFLAGS in nix-shell.
The nixpkgs manual says

Bintools Wrapper's setup hook causes any lib and lib64 subdirectories to be added to NIX_LDFLAGS.

but I haven't quite found yet where it's done and whether we can dedupe it there.

I found there's another part to it (https://github.com/haskell/cabal/pull/5356#issuecomment-393880674):
Independet of Haskell depencencies, for system dependencies (so, stuff that makes it in via no-Haskell-package-buildInputs), Cabal adds a duplicated -L flag for each executable, test-suite and benchmark in the .cabal file.
My case has 11 of those, so that also contributes to some blow-up.

Great cabal fixes! We should set strictDeps = true; in generic builder to further deduplicate things.

@Ericson2314 I think the strictDeps = true; fix does not work for stack. We'll need the cabal fix to get that working.

So is there a workaround/trick how I can build a stack project now? Can I downgrade the version of something?

@i-am-the-slime Building stack against a version of the Cabal library that has this patch https://github.com/haskell/cabal/pull/5356 should fix it.

My cabal PR has been merged which should provide a temporary mitigation: https://github.com/haskell/cabal/pull/5356
For the long run we still have to fix GCC upstream to not use the length-limited COLLECT_GCC_OPTIONS environment variable for passing things around.

@Mistuke I saw your comment and a mentioned fix on https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86030
What confuses me is that everthing there talks about -L flags being passed via the command line while what we observe here is that they are being passed between gcc and cc1 via an environment variable COLLECT_GCC_OPTIONS (which also breaks due to the length limit on env vars).
Do you know if this should be fixed by that upstream patch?

@nh2 probably, the code filters out the -L very early on. So what comes out of do_spec_1 will already be shortened so COLLECT_GCC_OPTIONS should use it. It does honor response files so it shouldn't have re-expanded the options yet.
If you want confirmation I can test using a GCC 9 build on Monday.

@Mistuke Thanks for double-checking that for us. Will you follow up with upstream GCC on your issue, or should I do that?

I will re-open the ticket on Monday, but it would be helpful if you could
then comment as well, especially mention that it's a problem on Linux too
as I think that carries more weight than a Windows issue :)
On Thu, Jun 28, 2018 at 12:57 PM, Niklas Hambüchen<EMAIL_ADDRESS>
wrote:

@Mistuke https://github.com/Mistuke Thanks for double-checking that for
us. Will you follow up with upstream GCC on your issue, or should I do that?
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
https://github.com/NixOS/nixpkgs/issues/41340#issuecomment-401010178,
or mute the thread
https://github.com/notifications/unsubscribe-auth/ABH3KY7zKNKCFQrVUzdiNuCD7ac-3fWMks5uBMS2gaJpZM4UVz4b
.

@Mistuke Sure, just let me know when I shall comment!

I have a similar issue on macOS:
Linking Setup ...
clang-5.0: warning: argument unused during compilation: '-nopie' [-Wunused-command-line-argument]
/nix/store/ckq71kkymh1ji2b44xn80wmr7fmi6wr5-clang-wrapper-5.0.2/bin/cc: line 183: /nix/store/bcl9zj60h52p47dy85s326mdrqx52417-clang-5.0.2/bin/clang: Argument list too long
`cc' failed in phase `Linker'. (Exit code: 126)

@domenkozar Same here. IIRC the maximum environment size is like one eighth of that on standard Linux, so it's a lot easier to hit.

@srhb what nixpkgs commit? I'm using 5974bb7c9c6a6fd4968516ebc6efa5370323fc8d
What's not clear to me is why this worked on GHC 8.2 and not so recent nixpkgs.

@domenkozar I wasn't aware that this was ever not a problem in the haskell infra, I always hit it for packages with a sufficiently large number of dependencies. fwiw currently head of nixos-18.03. I didn't build darwin long before that however.

For posterity:

angerman | The underlying issue is the following: we have $out/lib for each haskell library we build. However that folder is mostly useless for anything but haskell.
angerman | However the cc-wrapper logic, puts any $out/lib folder of the dependencies into the NIX_CLFAGS.
angerman | Now you end up with $pkg/lib for each haskell dependency, as well as $pkg/lib/ghc8.4.../ that GHC forwards to the compiler/linker.
angerman | and that overruns your limit.
angerman | So a qhick fix is to just mutilate the haskell pkg builder, to stop it from generating lib folders.
angerman | The proper fix would be to use response-files in the cc-wrapper, so that we pass @args-file to clang or gcc instead of passing the flags on the command line.
domenkozar | does clang support that?
angerman | domenkozar yes, for quite a few years already, the commit was landed in clang in 2014. See https://reviews.llvm.org/D4897

@nh2 I've re-opened the issue, https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86030 it seems however that in part it's a configuration thing. when GCC is build with --with-gnu-ld will treat the response files correctly. (this behaviour seems to be undocumented.).
So it seems that only the environment variables stuff may still be an issue.  But who knows.. maybe there's another undocumented flag..

Yes, we should use a separate output for that :)

Given that strictDeps fix not imminent, should we go with Angerman solution?

I've dug deeper.

So I've dug deeper into the issue and it has been progressingly getting worse with cross-compilation fixes. I've been running following command to count how many time a dependency shows itself as a linker argument:
nix-shell -A haskellPackages.memory --run "env |grep NIX_LDFLAGS|grep -o -w basement |wc -w " .

2a5688574c5c5d9797f4b010325f799910b42af6 (18.03 release at Jul 8): 2 times
a8e867f9905 (13 Jun): 3 times
1511a45ba65e502e210306969f4a001241911f40 (15 Jun) 6 times
1511a45ba65 (Jul 9): 6 times, 9 times in NIX_BUILD_LDFLAGS

More work on strictDeps: https://github.com/NixOS/cabal2nix/pull/358

@nh2 I've re-opened the issue, https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86030 it seems however that in part it's a configuration thing. when GCC is build with --with-gnu-ld will treat the response files correctly. (this behaviour seems to be undocumented.).
So it seems that only the environment variables stuff may still be an issue. But who knows.. maybe there's another undocumented flag..

@Mistuke Uh OK, interesting. Is the --with-gnu-ld something that we can use in general or does that put some restrictions on us? If we can use it, then we probably need only the env var upstream fix indeed.

@nh2 as far as I know it's something that has no other effects but allowing the response files. I think it's because at implementation time only ld supported this. So it's safe to turn on always as long as you either only use ld or all linkers you use support response files.

So I've bisected and https://github.com/NixOS/nixpkgs/compare/114a9b625386e3ca4e142dce6ce8bcfcabea8fe3...469fd8983276d851c19827cd9e78a89dd53a5914 is where the first time LDFLAGS start to be duplicated in nixpkgs history.
@Ericson2314 could you explain why that's happening?

So it's safe to turn on always as long as you either only use ld or all linkers you use support response files.

@Mistuke makes sense; two questions:

Do you know if gold and lld support it?
Do you know why they made this a compile-time option instead of a run-time one?

$ git co 4d2b7638173afb4c0f0c12a43654bcff5c9500ab
$ nix-shell -A libxml2 --run "env |grep LDFLAGS" . | grep --color python
NIX_LDFLAGS=-rpath /nix/store/f04m1im3cwj9rgcys31h80ic3i2n8fl3-libxml2-2.9.7/lib64 -rpath /nix/store/f04m1im3cwj9rgcys31h80ic3i2n8fl3-libxml2-2.9.7/lib  -L/nix/store/6yb5rvr6rvgvx8ylpchwz808djfw07rb-python-2.7.14/lib -L/nix/store/r2icacayl0prn5n65206x0bqx4idarpm-zlib-1.2.11-dev/lib -L/nix/store/la2xxf4grdyf37a94ak28m7a61yjqzr9-zlib-1.2.11/lib -L/nix/store/6yb5rvr6rvgvx8ylpchwz808djfw07rb-python-2.7.14/lib -L/nix/store/r2icacayl0prn5n65206x0bqx4idarpm-zlib-1.2.11-dev/lib -L/nix/store/la2xxf4grdyf37a94ak28m7a61yjqzr9-zlib-1.2.11/lib

$ git co 54d01b0e97f50c3fe87a41ebbbef36bb4ddc1e7b
$ nix-shell -A libxml2 --run "env |grep LDFLAGS" . | grep --color python
NIX_LDFLAGS=-rpath /nix/store/pjdj1mnqlyqnwmy4wcvrn3ipl2jhln39-libxml2-2.9.7/lib64 -rpath /nix/store/pjdj1mnqlyqnwmy4wcvrn3ipl2jhln39-libxml2-2.9.7/lib  -L/nix/store/hr461qqci171p6k53wwz52ifyp2h9g9l-python-2.7.14/lib -L/nix/store/ggxa0sw8h1y0yjvfb7agi7vniyax70dn-zlib-1.2.11-dev/lib -L/nix/store/4zs260xcpyzf9h252lw2qny8wwpm97wm-zlib-1.2.11/lib

I explained to domen on IRC. But let me recap here. https://github.com/NixOS/nixpkgs/commit/469fd8983276d851c19827cd9e78a89dd53a5914 was the source of the issue. It, for compatability purposes only, runs env hooks multiple times so incorrectly categorized deps still end up working. This was done based on the crossConfig environment variable, a convenient bash-level way to see whether we were cross compiling or not. https://github.com/NixOS/nixpkgs/commit/330ca731e88ec015181c43d92ae8f7c77cf0226a Then removed crossConfig, making a new strictDeps for this issue.
All this is to say my strictDeps PR isn't some sort of band-aid, but a fix of the the original issue. Indeed I wish to write an RFC to just get rid of the strictDeps flag by enabling it everywhere.
The final issue is that strictDeps, in its strict deps, breaks some Haskell packages. But, per Cabal's own plan, those packages are deemed broken. It is my opinion then that we should tollerate a higher number of overrides and/or more broken packages than usual, but the build failures are not Nix's fault, but the package's. In practical terms, with by the end of the year Nix and Cabal not building the package for the same easily-fixable reason, the package authors should be compelled to fix their packages so the number of overrides/broken packages should quickly decrease.

@nh2 I don't know how to get a version of stack build with that cabal patch. Do you happen to have one? Or some nix definition that builds it?

@nh2 Sorry I don't know the state of support for gold and lld, gold I wouldn't be terribly surprised if they didn't support it. I think lld might but couldn't find anything conclusive in the manual.
As to why it's a compile time check, I think a runtime check would be quite expensive to perform. Particularly on platforms where this is really required like windows. But realistically speaking... Someone just probably cut a corner here.. It could have been at least ast a configure time check, The same question was asked upstream in a different thread before with no answer.

@i-am-the-slime Wait, you're seeing this issue with stack already, without nix? On what platform are you?
You will have to build stack from source with the cabal dependency pointed at a commit that includes my patch. I can help you do so if that really is what you need.

OK so this issue has got me again now.
My cabal fix for https://github.com/haskell/cabal/pull/5356 isn't enough. With a larger project, I now see that

the ghc invocation by cabal is deduped fine
the cc invocation by ghc is deduped fine
the gcc invocation by cc contains all -L flags duplicated 10x

this is the sum of NIX_LDFLAGS duplicates (4x) and NIX_BUILD_LDFLAGS (6x)

the collect invocation by gcc also and that one crashes with E2BIG

This is on commit 2c07921cff84dfb0b9e0f6c2d10ee2bfee6a85ac and when I build with musl (but I think that is unrelated) for https://github.com/nh2/static-haskell-nix/ with nix on Ubuntu 16.04.

I am not sure if this should be clear to me via https://github.com/NixOS/nixpkgs/issues/41340#issuecomment-405272346, but does anybody understand why the duplicates are in NIX_LDFLAGS and NIX_BUILD_LDFLAGS?
I see flags like strictDeps flying around but I don't really know what that does so it would be great if somebody could explain this to me.

@nh2 see #43559 for some discussion of the injection of lirbary search dirs by nix for packages with $out/lib folders.

@i-am-the-slime Do you need the solution for NixOS or other distros?
For other distros
For other distros, the easiest is to clone stack from git, check out the release tag you want, and change the extra-deps entry for Cabal in stack.yaml to point at your fork of cabal at the same release tag but with the changes cherry-picked.
Here is a concrete example of that, picking my cabal fix on top of Cabal <IP_ADDRESS>:
diff --git a/stack.yaml b/stack.yaml
index e6696b14..4c921f08 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -18,7 +18,10 @@ flags:
     supported-build: true
 extra-deps:
 - rio-<IP_ADDRESS>@rev:0
-- Cabal-<IP_ADDRESS>@rev:0
+- git: git@github.com:nh2/cabal.git
+  commit: 7cb409fe7433833a3a8aa4b38a5fb3c2e01a5e5d
+  subdirs:
+  - Cabal
 - hpack-0.28.2
 - http-api-data-<IP_ADDRESS>@rev:0

For NixOS
You may be able to directly use code I wrote for static-haskell-nix: https://github.com/nh2/static-haskell-nix/blob/ef283274ce193f713082591dd462f4bd3fb4dd1f/survey/default.nix#L96
Build instructions here.
If you don't want to build a static version of stack, you can probably also just use
stack = useFixedCabal super.stack;

with useFixedCabal coming from here.

Hope that helps!

I don't know if anything will be interested in this (other than @i-am-the-slime), but I was able to use GHC-8.0.2 to work around this.
Here is a way to run the code.  It is using commit a4fcfdb24:
$ git clone git@github.com:cdepillabout/termonad.git
$ cd termonad
$ git checkout 8d371be175c3bb40c2204fefe538462ff9a5186b
$ stack --nix build

This is using GHC-8.0.2 from the nixpkgs for NixOS-18.03.
In case anyone is interested, GHC-8.2.2 also didn't work for any version of nixpkgs I tested.

Testing at https://hydra.nixos.org/jobset/nixpkgs/staging-next - it should finish tomorrow unless something important comes to Hydra queue meanwhile. So far it looks very promising.

To back this off:
$ on staging-next
$ nix-shell -A haskellPackages.stack --run "env | grep NIX_LDFLAGS | wc -m"
637

$ on master
$ nix-shell -A haskellPackages.stack --run "env | grep NIX_LDFLAGS | wc -m"
26505

Thanks to @dezgeg

Thanks a lot @domenkozar and @dezgeg!
I'd love to test this when hyrda finishes building it.
How can I figure out when Hydra has finished building it?  When I go to https://hydra.nixos.org/jobset/nixpkgs/staging-next, I see a list of evaluations, but I can't tell which are finished and which still haven't been built.  Also, I'm not sure which ones contain the commit from @dezgeg you linked to above.
Is there a guide to using Hydra you would recommend?

@cdepillabout it's the last two at this very moment on that list.
PR is open now: https://github.com/NixOS/nixpkgs/pull/44009
Hydra testing is done on linux, but it has quite a few packages left for arch/darwin, so probably merge is imminent for tomorrow.

Merged to master!

But the underlying problem of duplicated -L flags is still there, it's just that we workaround the problems happening with Haskell. It's being hacked around here for instance: https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/graphics/inkscape/default.nix#L24

@domenkozar I guess you could split off it into a separate issue, with say the findings from https://github.com/NixOS/nixpkgs/issues/41340#issuecomment-403453874 and any other findings you have (I haven't followed this problem as closely as you).

Yes, the underlying issue of setup-hooks being triggered 6 times per dependency is still there, but I think "argument list too long" is "fixed" for now (as in, the limit isn't reached).
$ on master
$ nix-shell -A inkscape --run "env | grep NIX_LDFLAGS | wc -m"
4672

$ on (HEAD detached at f68920176c2)
$ nix-shell -A inkscape --run "env | grep NIX_LDFLAGS | wc -m"
9112

@domenkozar I'm still having a problem when trying to build with stack's nix support, using buildStackProject.
Here is my project.  Here is the shell.nix I am using.
Here is what happens when I try to build this project on NixOS:
$ stack build --nix
...
haskell-gi-0.21.3: configure
haskell-gi-0.21.3: build
Progress: 1/12
--  While building custom Setup.hs for package haskell-gi-0.21.3 using:
      /home/illabout/.stack/setup-exe-cache/x86_64-linux-nix/Cabal-simple_mPHDZzAJ_<IP_ADDRESS>_ghc-8.4.3 --builddir=.stack-work/dist/x86_64-linux-nix/Cabal-<IP_ADDRESS> build --ghc-options " -ddump-hi -ddump-to-file -fdiagnostics-color=always"
    Process exited with code: ExitFailure 1
    Logs have been written to: /home/illabout/git/termonad/.stack-work/logs/haskell-gi-0.21.3.log

    Configuring haskell-gi-0.21.3...
    Preprocessing library for haskell-gi-0.21.3..
    gcc: error trying to exec '/nix/store/hsw4smim26xm732xxi4xnpx5s1az81ld-gcc-7.3.0/libexec/gcc/x86_64-unknown-linux-gnu/7.3.0/cc1': execv: Argument list too long
    compiling .stack-work/dist/x86_64-linux-nix/Cabal-<IP_ADDRESS>/build/Data/GI/CodeGen/GType_hsc_make.c failed (exit code 1)
    command was: /nix/store/frqm8gncayv83s3hrv7hpldp865gbspc-gcc-wrapper-7.3.0/bin/cc -c .stack-work/dist/x86_64-linux-nix/Cabal-<IP_ADDRESS>/build/Data/GI/CodeGen/GType_hsc_make.c -o .stack-work/dist/x86_64-linux-nix/Cabal-<IP_ADDRESS>/build/Data/GI/CodeGen/GType_hsc_make.o -fno-stack-protector ...

Is this a known problem?  Is buildStackProject still not supported?

First of all—and I am embarrassed that I don't remember saying it before—Sorry for all the headache my cross changes have caused with this. Second, yes please do open an issue door the underlying problem. I will refer to it in my upcomming strictDeps everywhere RFC.

@domenkozar Thanks for trying to build it on your machine.  That is good to hear that it is working for you.
I was able to get it working on my machine as well.
I was running stack build --nix using the system-wide-installed stack.  I thought this would re-exec the stack defined in buildStackProject, and use that to actually build the project.  This turned out to not be the case.
Instead, I just ran nix-shell with the shell.nix file linked above, and then from there I ran stack build.  That appears to work.
Thanks!

I'm also hitting the error 'cc' failed in phase 'Linker' on mac. However, I get it without using anything stack related, I can consistently get the error with something like cabal install hyper-haskell-server.
Both cabal-install-<IP_ADDRESS> and ghc-8.4.3 were recently installed from the unstable channel.

@cdepillabout What is the maximum command line length on the version of OSX you're using?
I cannot reliably find information about this on the Internet, but https://apple.stackexchange.com/questions/254010/is-there-a-way-around-error-command-line-too-long suggests it's 130592 and your command line in question is 137k long.
@matthewbauer's fix is about environment variables, which unfortunately don't appear in your output, so we can't exactly tell. Perhaps with some equivalent of strace on OSX it could be shown what the actual environment variables are during the exec() call, so we can be sure that it's really the command line arguments limit, not long environment variables?
In any case, the command line looks reasonable to me, I don't think it can be shortened.
I think we really have to fix this issue in upstream compilers (like I said in https://github.com/NixOS/nixpkgs/issues/41340#issuecomment-399465674 for gcc, I didn't know clang has a similar issue), as we can only get so far with hacky workarounds trying to reduce the command line length. Modern programs link many libraries, and compilers must support that, there is no way around it.

I've been researching this issue for the past few days, because I've been trying to get the LibreOffice package to build on Darwin, and it was hitting this issue too.
In both that case, and the termonad issue posted by @cdepillabout, the issue has been that the arguments passed to the linker by clang have been too long. This probably isn't something we can fix – I suspect the reason we're hitting it and discovering all these things is that the extra linker flags added by cc-wrapper are just enough to push the linker invocation over the edge.
I fixed the issue in LibreOffice here (https://github.com/LibreOffice/core/commit/ab9d95e6073d84a0dbabf1a4e704b8468afe7bff) by using -Wl,-filelist to pass object files to link to the linker, through clang, as a file rather than an argument list. Here's an example of it being fixed in Thunderbird when they independently ran into the same issue.
These fixes are specific to particular build systems, though, so if we want a proper solution, I agree that the thing to do is fix clang. There is a bug open in clang for this exact issue, and it seems to be generally accepted to be a good idea, but there hasn't been any activity on it for five years.

Once we have that fixed, we can move on to using response files when we invoke the compiler, and that should be pretty straightforward. But there's not much point doing that until the -filelist thing is implemented. Since I've been testing it anyway, though, it should be something like this:
diff --git a/pkgs/build-support/cc-wrapper/cc-wrapper.sh b/pkgs/build-support/cc-wrapper/cc-wrapper.sh
index 8003fe1d8f3..4665362ad80 100644
--- a/pkgs/build-support/cc-wrapper/cc-wrapper.sh
+++ b/pkgs/build-support/cc-wrapper/cc-wrapper.sh
@@ -180,7 +180,10 @@ fi

 PATH="$path_backup"
 # Old bash workaround, see above.
-exec @prog@ \
-    ${extraBefore+"${extraBefore[@]}"} \
-    ${params+"${params[@]}"} \
-    ${extraAfter+"${extraAfter[@]}"}
+responseFile=$(mktemp)
+printf "%q " \
+  ${extraBefore+"${extraBefore[@]}"} \
+  ${params+"${params[@]}"} \
+  ${extraAfter+"${extraAfter[@]}"} \
+  > "$responseFile"
+exec @prog@ "@$responseFile"

The problem with the clang bug is that on Darwin clang eventually calls system ld64, which doesn’t support a response file. Unfortunately, -filelist is not the same as the response file, as you can’t give options in it, you can only give input files. This issue here is caused mostly by the growing number of libraries, which translates into the growing number of options; by making clang pass inputs to ld via -filelist we do not really solve the problem, although we slightly improve on it, of course. I guess, this is the reason for the lack of activity.
To be honest, I don’t think anyone expects Apple to implement this in ld64. One viable strategy would be to patch ld64 ourselves, then patch clang ourselves, tell it to use our ld instead of the system one, and then maintain a patched linker and a patched C compiler 🙄. The good news is that the clang patch might be upstreamable, if hidden behind some sort of --with-gnu-ld, similar to GCC, but maybe with a more indicative name. The bad news is that a patched linker is still meh 🤷♂. (Although, maybe, just maybe, if we send this patch to Apple...)

@vcunat vcunat closed this in 9172c1e 4 hours ago

@vcunat @matthewbauer Should the above really have closed this issue?
avoid running the same hook twice sounds to me like at most a factor of 2x can be gained, which would mean the problem isn't really solved in general.
So it seems to me the real solution remains fixing GCC and other compilers to not turn response file contents into environment variables.
Reopening until somebody confirms otherwise.

So it seems to me the real solution remains fixing GCC and other compilers to not turn response file contents into environment variables.

This. So. Much. This.

Oops sorry. Yeah that commit might fix the issue but was mostly just to avoid duplication that happens when strictDeps = false.

I've made a repro (for this) in Python on my Ubuntu 16.04 to confirm the 128KB limit for me:
import os
import sys

# num_chars = 128000 # works
# num_chars = (2**16) # works
# num_chars = (2**17) # fails with E2BIG (Argument list too long)
# num_chars = (2**17 - 3) # works
num_chars = (2**17 - 2) # fails with E2BIG (Argument list too long)

sys.stdout.write("trying with " + str(num_chars) + " chars\n")

os.execve('/bin/true', [], {
  'x': 'a' * num_chars
})

This shows
trying with 131069 chars

and
trying with 131070 chars
OSError: [Errno 7] Argument list too long

The c2hs variant of this issue is captured in this ticket: https://github.com/haskell/hsc2hs/issues/22

Hrm maybe we need to teach cabal to call c2hs with a response file too, sigh.

(note that cabal actually calls hsc2hs with response files when possible: https://github.com/haskell/cabal/issues/3122)

I found two more points to this problem (hitting it again today):

NIX_LDFLAGS already contains duplicated -L flags, sneaking them past Cabal directly to the linker. This means no fixes in Cabal can help that.

I think the whole NIX_*FLAGS variables are just bad hacks and should be avoided wherever possible. These hacks cause these problems. Instead, flags should be passed directly to the build systems, as I propose for a similar variable in #79303. I think in nixpkgs we must work to remove these hacks.

If you use clang as e.g. the Haskell -pgmc compiler (or at least that seems to make the difference for me), then additionally the same variables will be also in NIX_TARGET_LDFLAGS, which will also make its way into COLLECT_GCC_OPTIONS (in addition to NIX_LDFLAGS).

For the second question, can anybody explain to me what NIX_TARGET_LDFLAGS is designed for? Its uses in the code have zero comments, it's just a pile of bash script without any explanations.

NIX_LDFLAGS already contains duplicated -L flags, sneaking them past Cabal directly to the linker.

I've developed a hacky workaround for this, which that deduplicates NIX_*LDFLAGS variables straight in you preConfigure.
https://gist.github.com/nh2/fc01e3652b55e36a6a03f4295c3e8388

I'm also running into this problem, with gst-plugins-bad on darwin: https://hydra.nixos.org/build/123830272
Sanity check compile stderr:
/nix/store/jhfr9mqrjjc1nfjrnjwb8sl889aj8lj9-clang-wrapper-7.1.0/bin/clang: line 194: /nix/store/mgh4h96hhxgd1szm9zfy57pijh931vmx-clang-7.1.0/bin/clang: Argument list too long

-----

meson.build:1:0: ERROR: Compiler clang can not compile programs.

I don't know if that is the problem, but PKG_CONFIG_PATH contains 142 items and 10706 characters.
Every item is duplicated once, sort -u | wc -l gives back 71 items.
NIX_CFLAGS_COMPILE_FOR_BUILD is 30330 characters and everything is included three times.
I have not tried the script above yet, but I'm not sure if it will help, since it seems to filter other things.

I'm also running into this problem, with gst_all_1.gst-plugins-bad on darwin: https://hydra.nixos.org/build/123830272
Sanity check compile stderr:
/nix/store/jhfr9mqrjjc1nfjrnjwb8sl889aj8lj9-clang-wrapper-7.1.0/bin/clang: line 194: /nix/store/mgh4h96hhxgd1szm9zfy57pijh931vmx-clang-7.1.0/bin/clang: Argument list too long

-----

meson.build:1:0: ERROR: Compiler clang can not compile programs.

I don't know if that is the problem, but PKG_CONFIG_PATH contains 142 items and 10706 characters.
Every item is duplicated once, sort -u | wc -l gives back 71 items.
NIX_CFLAGS_COMPILE_FOR_BUILD is 30330 characters and everything is included three times.
I have not tried the script above yet, but I'm not sure if it will help, since it seems to filter other things.
Edit: After trying the build in a nix-shell with NIX_DBG=1, I see that the total number of arguments that are passed is 1594, with a total of 77316 characters. I don't know what the limit is on mac.
The final argument list sent to clang has 5 copies of everything, so sort -u gives 210 arguments, instead of 1003 arguments (after merging paired arguments like -iframework /nix/... into single arguments).
Edit 2: The limit on my mac seems to be 262117 characters, so I don't know why it failed. I may be missing something.

I'm running into exactly the same problem, do you have any progress here?

I only skimmed this discussion, but wouldn't it be possible to write something that hardlinks/symlinks nix store packages using minimum hashes? i.e., for /nix/store/2p2mh4rl8lrlpgkc1lss50csd3gk0l97-home-manager  and /nix/store/848yljqxq9zfkfa2x8d93h23ka1mqmkg-home-manager we could have 2-home-manager and 8-home-manager, respectively? This shouldn't hurt the usual lookups by hash and perhaps could exist without ingering in Nix itself.
I am running into this issue from a  chatty haskell-gi build

I also bumped into the same error on OSX. Unfortunately it seems that ARG_MAX "values cannot be changed in Mac OS X kernel."
However, like mentioned in previous posts, the number of arguments or the size of the arguments didn't seem to match the limit. I realised with this post https://unix.stackexchange.com/questions/604066/macosx-number-of-arguments-on-command-line that in fact the environment counts towards this limit. So a large env | wc is an aggravating factor.
In my case, my environment was bloated because while I was using direnv I also used nix develop which is not needed but added another 150K to the env.
Strangely enough, just as I was exploring that it seemed possible to change some system limits using the tcsh shell (?) I reran my build in this shell and it worked...

As relevant as it gets

I believe that Big Sur did something to alleviate this issue, but I haven't tracked down particular documentation as to what. I can verify that packages we had failing to link before, started doing so properly once users upgraded to Big Sur.

Could we do something to detect when we get many copies of the same argument in an argument list and/or try to deduplicate it? The instance of this I ran into had everything included 5 times, when only one instance would have been sufficient.
We would of course need to be clever to not have tons of false positives, but : separated lists like PATH should be fairly safe to deduplicate.

try to deduplicate it

The best way to deduplicate flags is to not generate them as duplicates in the first place, like I did to some extent in Cabal in e.g. https://github.com/haskell/cabal/pull/5356
It's my understanding that deduplicating -L path flags could be done even before the GCC invocation without impact, while passing duplicated -l flags might have actual real-world use cases (but -l flags are typically short).
I also think the overall best way to fix this would be to do a fundraising with companies that suffer from this, so that we can have somebody fix the plumbing of arguments inside GCC to not have this limit. I suspect this could be done within a few tens of hours by somebody reasonably acquainted with the GCC code base.

This issue has been mentioned on NixOS Discourse. There might be relevant details there:
https://discourse.nixos.org/t/rust-lld-execv-argument-list-too-long/18842/5

I'm getting this issue when i run shake in nix-shell but not in nix-shell --pure.
I'm using https://github.com/tweag/haskell-stack-nix-example to build our build system, which is written in shake. And then running shake after that. I can't share the project because it isn't public.
On mac nix-shell --pure won't work because there some dependencies are missing in nixpkgs.
Can we somehow only pass the needed -L path flags to gcc?

This may also help things: https://github.com/NixOS/nixpkgs/pull/213831

Also, for ghc specifically this might also help (on macOS): https://gitlab.haskell.org/ghc/ghc/-/merge_requests/9877
The same change could be done for Linux, too, in theory

We've verified internally that the combination of #213831 and and https://gitlab.haskell.org/ghc/ghc/-/merge_requests/9897 fixes Haskell builds on macOS to no longer exceed ARG_MAX by consistently using response files throughout the chain (ghc, cc-wrapper, and ld-wrapper).
Fixing Linux builds still might require a tiny bit more work (since ld-wrapper currently only enables response files for cctools, which is only used on macOS).
This obviates the need to deduplicate or compress command-line arguments, but that's still a good thing to do anyway.

Unfortunately, limiting argv is not enough to avoid E2BIG on linux. The kernel checks if there is enough stack space for both argv and environ before execing anything: https://github.com/torvalds/linux/blob/4eb2bd24756e0c8e254de8931ba7ee4346e75bbc/fs/exec.c#L509

With GHC 9.4.5 on NixOS 24.05, it seems I am still encountering the issue when trying to build xmobar:
/ghc-bignum-1.3/include -I/nix/store/k2nxkk1glg5blqjpak6incqyjyhgdnlm-elfutils-0.191-dev/include -I/nix/store/1xcnwyx7pdmdq66xdz20fbl2q0arjzrl-libffi-3.4.6-dev/include -I/nix/store/lnp12bydmfwjf19wbnw0xzynq1xx86ad-ghc-9.6.4/lib/ghc-9.6.4/lib/../lib/x86_64-linux-ghc-9.6.4/rts-1.0.2/include -I/nix/store/lnp12bydmfwjf19wbnw0xzynq1xx86ad-ghc-9.6.4/include/
xmobar> error: gcc: fatal error: cannot execute ‘/nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/libexec/gcc/x86_64-unknown-linux-gnu/13.2.0/cc1’: execv: Argument list too long
xmobar> compilation terminated.

Quite possibly, I'm going about this the wrong way - I have detailed instructions and all the relevant files at https://github.com/bbarker/dotxmonad/tree/0fdc2c9f4efb2895350474c2ceb7784cbf1c919f - any suggestions would be welcome.

I'm encountering it on NixOS 24.05 with the Haskell opencv package; Cabal outputs:
Error: Setup: Missing dependencies on foreign libraries:
* Missing (or bad) C libraries: stdc++, opencv_gapi, opencv_stitching,
opencv_alphamat, opencv_aruco, opencv_bgsegm, opencv_bioinspired,
opencv_ccalib, opencv_dnn_objdetect, opencv_dnn_superres, opencv_dpm,
opencv_face, opencv_freetype, opencv_fuzzy, opencv_hdf, opencv_hfs,
opencv_img_hash, opencv_intensity_transform, opencv_line_descriptor,
opencv_mcc, opencv_quality, opencv_rapid, opencv_reg, opencv_rgbd,
opencv_saliency, opencv_stereo, opencv_structured_light,
opencv_phase_unwrapping, opencv_superres, opencv_optflow,
opencv_surface_matching, opencv_tracking, opencv_highgui, opencv_datasets,
opencv_text, opencv_plot, opencv_videostab, opencv_videoio,
opencv_xfeatures2d, opencv_shape, opencv_ml, opencv_ximgproc, opencv_video,
opencv_xobjdetect, opencv_objdetect, opencv_calib3d, opencv_imgcodecs,
opencv_features2d, opencv_dnn, opencv_flann, opencv_xphoto, opencv_photo,
opencv_imgproc, opencv_core

because the g++ invocations that probe for the libraries produce E2BIG (Argument list too long) (only visible in strace due to https://github.com/haskell/cabal/issues/5355).

This is most (in my mind) a gcc bug. Most everything that gcc drives gets invoked with a response file. There is one exception: collect2. It gets invoked with an env var that is multiple megabytes in size by the time it gets there. Please see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86030 and maybe push on upstream a bit if you can.
I think the problem is a hard one to solve but would make a lot of people's lives a lot better.
There is a workaround - put your problematic modules in separate packages altogether. That's what we ended up doing at $WORK.

@jsoo1 Yes, definitely a GCC bug.

A workaround for https://github.com/NixOS/nixpkgs/issues/41340#issuecomment-2171931982 that is to set
__propagatePkgConfigDepends = false;

which is from
https://github.com/NixOS/nixpkgs/blob/63b875e099c14114b0dfdb21401a04de62903222/pkgs/development/haskell-modules/generic-builder.nix#L123-L132
That existing is to some extent my fault:

https://github.com/haskell/cabal/issues/8455

A workaround for #41340 (comment) that is to set
__propagatePkgConfigDepends = false;

Any idea how to do this with haskell-flake?

This issue has been mentioned on NixOS Discourse. There might be relevant details there:
https://discourse.nixos.org/t/trouble-building-xmobar-argument-list-too-long/46897/3


New error handling logic

This is how I used to do error handling:
https://github.com/go-openwechat/owc-insight/blob/3463e407dddce8b0187ec2bc9ec77d031540ef31/main.go#L94-L107
	bot.GetMessageErrorHandler = func(err error) {
		t := time.Now()
		if t.Sub(lastError) < 30*time.Minute {
			count++
		} else {
			count = 1
		}
		// 如果发生了三次错误,那么直接退出
		if count > 3 {
			abortOn("Too many errors", err)
		}
		logIf(0, "catch-and-skip", "count", count, "err", err)
		lastError = time.Now()
	}

But the error-resuming is not working any more, after the recent error handling logic change.
I saw in my log:
TS=0731T22:42:57 Msg=catch-and-skip count=1 err="Post \"https://web.wechat.com/cgi-bin/mmwebwx-bin/webwxsync?pass_ticket=Dg%252BlKiL53lu3OWQoX%252BbrpGPBCS%252FEz6YdHeW431WCvfU%253D&sid=fzHyectwb4p0PXWd&skey=%40crypt_60175222_bca13b8cf30a7748a9a71edf111ea98d\": EOF"

And that is the last entry in my log. Nothing else logged afterwards, yet I know lots happened afterwards.
So, what's the new error handling logic supposed to be? Thx.

I'll start from scratch again.


Getting Error while creating APK

Versions

Python: 3.6
OS: Ubuntu 18.04
Kivy: 1.10.1
Kivy installation method:

Description
// REPLACE ME: What are you trying to get done, what has happened, what went wrong, and what did you expect?
Code and Logs
Command failed: /usr/bin/python3 -m pythonforandroid.toolchain apk --debug --bootstrap=sdl2 --dist_name myapp --name 'My Application' --version 0.1 --package org.test.myapp --android_api 19 --minsdk 9 --private /home/developer/Desktop/App/.buildozer/android/app --orientation portrait --window --copy-libs --arch armeabi-v7a --color=always --storage-dir=/home/developer/Desktop/App/.buildozer/android/platform/build

Buildozer failed to execute the last command
The error might be hidden in the log above this error
Please read the full log, and search for it before
raising an issue with buildozer itself.
In case of a bug report, please add a full log with log_level = 2

Sorry you reported to the wrong project, you probably wanted python-for-android or buildozer.
However you don't provide enough log details to identify the bug.
If what you wanted was not a bug report but you looked for help, it's better to use Discord chat.


Deprecated metrics mixin drops too many time series

In k8s 1.14, the following metrics were deprecated:

kubelet_runtime_operations
kubelet_runtime_operations_latency_microseconds
kubelet_runtime_operations_errors

They were renamed to fit naming conventions:

kubelet_runtime_operations_total
kubelet_runtime_operations_duration_seconds
kubelet_runtime_operations_errors_total

However, the mixin that drops deprecated time series (dropping-deprecated-metrics-relabelings.libsonnet) is a little too aggressive:
  // Drop all kubelet metrics which are deprecated in kubernetes.
  {
    sourceLabels: ['__name__'],
    regex: 'kubelet_(pod_worker_latency_microseconds|pod_start_latency_microseconds|cgroup_manager_latency_microseconds|pod_worker_start_latency_microseconds|pleg_relist_latency_microseconds|pleg_relist_interval_microseconds|runtime_operations|runtime_operations_latency_microseconds|runtime_operations_errors|eviction_stats_age_microseconds|device_plugin_registration_count|device_plugin_alloc_latency_microseconds|network_plugin_operations_latency_microseconds).*',
    action: 'drop',
  },

The trailing .* in the regex is causing the new time series to be dropped along with the old ones.
This breaks the Kubernetes / Kubelet dashboard -- every graph reports No data.

/cc @LiliC @krasi-georgiev

Thanks for reporting the issue, will work on the fix!


no extension detected on linux firefox

On linux / firefox it can't seem to detect an extension. I have metamask, parity signer and polkadot-js extensions installed so 2 of them ought to show up.
(I know this is a niche platform...)

Fixed in #73


Pp 5357 consume authorise success event

Removes EventType enum in favour of SalientEventType. There is now no validation of event type on reception of event, so emitters can send any event type they want. The SalientEventType enum will be used to translated a known set of events to a known set of transaction states. This is done via the EventDigest, which now stores the 'latestSalientEventType'.
Introduces handling of multiple different events concerning a single external resource id. This comprises:

Changing TransactionService to map from latest salient event to transaction state
Changing TransactionDao to upsert rather than insert transactions, using ON CONFLICT DO UPDATE …
Introduce event count to avoid stale updates of transaction - transaction will only be updated if the changes are built of more events than current record
introduces new transaction state 'SUBMITTED', and maps it to AUTHORISATION SUCCESS event

I think that by "Salient" event you mean one which causes a state change.
I wonder if it might be more self-evident if we just called it "StateChangingEvent"?

I think that by "Salient" event you mean one which causes a state change.
I wonder if it might be more self-evident if we just called it "StateChangingEvent"?

Heh, maybe. I think we've been talking about salient events for so long I lost sight of the fact it's a bit of a quirky name

also something to keep in mind.. the unique constraint will fail when deploying if there are dups (in test env)

I'd consider renaming SalientEvent to StateChangingEvent but not a blocker to merging


CONFLUENT: Change default dir paths from /var/lib to /tmp

This reverts the changes done in: https://github.com/confluentinc/kafka/commit/e5a6ed3471efa501e3ea96b2302447a10d43b137
Committer Checklist (excluded from commit message)

[ ] Verify design and implementation
[ ] Verify test coverage and CI build status
[ ] Verify documentation (including upgrade notes)

Thanks for the review. Merging to trunk and 2.3.