Official layer norm example fails with big absolute differences

Hey team,
I'm trying to get this fork of triton working on MI250X GPUs and I'm running into some test failures for the official layer norm example. I'm on a commit from earlier today (fb3f2d6feb8021f07ec8b5cabf1db04567f608d6).
Here's the exact code I'm running (a simplified version of the official example):
from typing import Optional, Tuple

import torch
import triton
import triton.language as tl

@triton.jit
def _layer_norm_fwd_fused(
    X,  # pointer to the input
    Y,  # pointer to the output
    W,  # pointer to the weights
    B,  # pointer to the biases
    Mean,  # pointer to the mean
    Rstd,  # pointer to the 1/std
    stride,  # how much to increase the pointer when moving by 1 row
    N,  # number of columns in X
    eps,  # epsilon to avoid division by zero
    BLOCK_SIZE: tl.constexpr,
):
    # Map the program id to the row of X and Y it should compute.
    row = tl.program_id(0)
    Y += row * stride
    X += row * stride
    # Compute mean
    mean = 0
    _mean = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
    for off in range(0, N, BLOCK_SIZE):
        cols = off + tl.arange(0, BLOCK_SIZE)
        a = tl.load(X + cols, mask=cols < N, other=0.0).to(tl.float32)
        _mean += a
    mean = tl.sum(_mean, axis=0) / N
    # Compute variance
    _var = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
    for off in range(0, N, BLOCK_SIZE):
        cols = off + tl.arange(0, BLOCK_SIZE)
        x = tl.load(X + cols, mask=cols < N, other=0.0).to(tl.float32)
        x = tl.where(cols < N, x - mean, 0.0)
        _var += x * x
    var = tl.sum(_var, axis=0) / N
    rstd = 1 / tl.sqrt(var + eps)
    # Write mean / rstd
    tl.store(Mean + row, mean)
    tl.store(Rstd + row, rstd)
    # Normalize and apply linear transformation
    for off in range(0, N, BLOCK_SIZE):
        cols = off + tl.arange(0, BLOCK_SIZE)
        mask = cols < N
        x = tl.load(X + cols, mask=mask, other=0.0).to(tl.float32)
        x_hat = (x - mean) * rstd
        w = tl.load(W + cols, mask=mask)
        b = tl.load(B + cols, mask=mask)
        y = x_hat * w + b
        # Write output
        tl.store(Y + cols, y, mask=mask)

@triton.jit
def _layer_norm_bwd_dx_fused(
    DX,  # pointer to the input gradient
    DY,  # pointer to the output gradient
    DW,  # pointer to the partial sum of weights gradient
    DB,  # pointer to the partial sum of biases gradient
    X,  # pointer to the input
    W,  # pointer to the weights
    B,  # pointer to the biases
    Mean,  # pointer to the mean
    Rstd,  # pointer to the 1/std
    Lock,  # pointer to the lock
    stride,  # how much to increase the pointer when moving by 1 row
    N,  # number of columns in X
    GROUP_SIZE_M: tl.constexpr,
    BLOCK_SIZE_N: tl.constexpr,
):
    # Map the program id to the elements of X, DX, and DY it should compute.
    row = tl.program_id(0)
    cols = tl.arange(0, BLOCK_SIZE_N)
    mask = cols < N
    X += row * stride
    DY += row * stride
    DX += row * stride
    # Offset locks and weights/biases gradient pointer for parallel reduction
    lock_id = row % GROUP_SIZE_M
    Lock += lock_id
    Count = Lock + GROUP_SIZE_M
    DW = DW + lock_id * N + cols
    DB = DB + lock_id * N + cols
    # Load data to SRAM
    x = tl.load(X + cols, mask=mask, other=0).to(tl.float32)
    dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32)
    w = tl.load(W + cols, mask=mask).to(tl.float32)
    mean = tl.load(Mean + row)
    rstd = tl.load(Rstd + row)
    # Compute dx
    xhat = (x - mean) * rstd
    wdy = w * dy
    xhat = tl.where(mask, xhat, 0.0)
    wdy = tl.where(mask, wdy, 0.0)
    c1 = tl.sum(xhat * wdy, axis=0) / N
    c2 = tl.sum(wdy, axis=0) / N
    dx = (wdy - (xhat * c1 + c2)) * rstd
    # Write dx
    tl.store(DX + cols, dx, mask=mask)
    # Accumulate partial sums for dw/db
    partial_dw = (dy * xhat).to(w.dtype)
    partial_db = (dy).to(w.dtype)
    while tl.atomic_cas(Lock, 0, 1) == 1:
        pass
    count = tl.load(Count)
    # First store doesn't accumulate
    if count == 0:
        tl.atomic_xchg(Count, 1)
    else:
        partial_dw += tl.load(DW, mask=mask)
        partial_db += tl.load(DB, mask=mask)
    tl.store(DW, partial_dw, mask=mask)
    tl.store(DB, partial_db, mask=mask)
    # Release the lock
    tl.atomic_xchg(Lock, 0)

@triton.jit
def _layer_norm_bwd_dwdb(
    DW,  # pointer to the partial sum of weights gradient
    DB,  # pointer to the partial sum of biases gradient
    FINAL_DW,  # pointer to the weights gradient
    FINAL_DB,  # pointer to the biases gradient
    M,  # GROUP_SIZE_M
    N,  # number of columns
    BLOCK_SIZE_M: tl.constexpr,
    BLOCK_SIZE_N: tl.constexpr,
):
    # Map the program id to the elements of DW and DB it should compute.
    pid = tl.program_id(0)
    cols = pid * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
    dw = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
    db = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
    # Iterate through the rows of DW and DB to sum the partial sums.
    for i in range(0, M, BLOCK_SIZE_M):
        rows = i + tl.arange(0, BLOCK_SIZE_M)
        mask = (rows[:, None] < M) & (cols[None, :] < N)
        offs = rows[:, None] * N + cols[None, :]
        dw += tl.load(DW + offs, mask=mask, other=0.0)
        db += tl.load(DB + offs, mask=mask, other=0.0)
    # Write the final sum to the output.
    sum_dw = tl.sum(dw, axis=0)
    sum_db = tl.sum(db, axis=0)
    tl.store(FINAL_DW + cols, sum_dw, mask=cols < N)
    tl.store(FINAL_DB + cols, sum_db, mask=cols < N)

class _LayerNormAffineF(torch.autograd.Function):
    @staticmethod
    def forward(
        ctx,
        x: torch.Tensor,
        normalized_shape: Tuple[int, ...],
        weight: Optional[torch.Tensor],
        bias: Optional[torch.Tensor],
        eps: float,
    ):
        assert len(normalized_shape) == 1  # TODO: handle general case
        # allocate output
        y = torch.empty_like(x)
        # reshape input data into 2D tensor
        x_arg = x.reshape(-1, x.shape[-1])
        M, N = x_arg.shape
        mean = torch.empty((M,), dtype=torch.float32, device="cuda")
        rstd = torch.empty((M,), dtype=torch.float32, device="cuda")
        # Less than 64KB per feature: enqueue fused kernel
        MAX_FUSED_SIZE = 65536 // x.element_size()
        BLOCK_SIZE = min(MAX_FUSED_SIZE, triton.next_power_of_2(N))
        if N > BLOCK_SIZE:
            raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.")
        # heuristics for number of warps
        num_warps = min(max(BLOCK_SIZE // 256, 1), 8)
        # enqueue kernel
        _layer_norm_fwd_fused[(M,)](
            x_arg,
            y,
            weight,
            bias,
            mean,
            rstd,
            x_arg.stride(0),
            N,
            eps,
            BLOCK_SIZE=BLOCK_SIZE,
            num_warps=num_warps,
        )
        ctx.save_for_backward(x, weight, bias, mean, rstd)
        ctx.BLOCK_SIZE = BLOCK_SIZE
        ctx.num_warps = num_warps
        ctx.eps = eps
        return y

    @staticmethod
    def backward(ctx, dy):
        x, w, b, m, v = ctx.saved_tensors
        # heuristics for amount of parallel reduction stream for DW/DB
        N = w.shape[0]
        GROUP_SIZE_M = 64
        if N <= 8192:
            GROUP_SIZE_M = 96
        if N <= 4096:
            GROUP_SIZE_M = 128
        if N <= 1024:
            GROUP_SIZE_M = 256
        # allocate output
        locks = torch.zeros(2 * GROUP_SIZE_M, dtype=torch.int32, device="cuda")
        _dw = torch.empty((GROUP_SIZE_M, w.shape[0]), dtype=x.dtype, device=w.device)
        _db = torch.empty((GROUP_SIZE_M, w.shape[0]), dtype=x.dtype, device=w.device)
        dw = torch.empty((w.shape[0],), dtype=w.dtype, device=w.device)
        db = torch.empty((w.shape[0],), dtype=w.dtype, device=w.device)
        dx = torch.empty_like(dy)
        # enqueue kernel using forward pass heuristics
        # also compute partial sums for DW and DB
        x_arg = x.reshape(-1, x.shape[-1])
        M, N = x_arg.shape
        _layer_norm_bwd_dx_fused[(M,)](
            dx,
            dy,
            _dw,
            _db,
            x,
            w,
            b,
            m,
            v,
            locks,
            x_arg.stride(0),
            N,
            BLOCK_SIZE_N=ctx.BLOCK_SIZE,
            GROUP_SIZE_M=GROUP_SIZE_M,
            num_warps=ctx.num_warps,
        )
        grid = lambda meta: [triton.cdiv(N, meta["BLOCK_SIZE_N"])]
        # accumulate partial sums in separate kernel
        _layer_norm_bwd_dwdb[grid](
            _dw,
            _db,
            dw,
            db,
            GROUP_SIZE_M,
            N,
            BLOCK_SIZE_M=32,
            BLOCK_SIZE_N=128,
        )
        return dx, None, dw, db, None

def layer_norm(
    x: torch.Tensor,
    normalized_shape: Tuple[int, ...],
    weight: torch.Tensor,
    bias: torch.Tensor,
    eps: float = 1e-5,
) -> torch.Tensor:
    return _LayerNormAffineF.apply(x, normalized_shape, weight, bias, eps)

def test_layer_norm_with_affine(M, N, dtype, eps=1e-5, device="cuda"):
    # create data
    x_shape = (M, N)
    w_shape = (x_shape[-1],)
    weight = torch.rand(w_shape, dtype=dtype, device=device, requires_grad=True)
    bias = torch.rand(w_shape, dtype=dtype, device=device, requires_grad=True)
    x = -2.3 + 0.5 * torch.randn(x_shape, dtype=dtype, device=device)
    dy = 0.1 * torch.randn_like(x)
    x.requires_grad_(True)
    # forward pass
    y_tri = layer_norm(x, w_shape, weight, bias, eps)
    y_ref = torch.nn.functional.layer_norm(x, w_shape, weight, bias, eps).to(dtype)
    # backward pass (triton)
    y_tri.backward(dy, retain_graph=True)
    dx_tri, dw_tri, db_tri = [_.grad.clone() for _ in [x, weight, bias]]
    x.grad, weight.grad, bias.grad = None, None, None
    # backward pass (torch)
    y_ref.backward(dy, retain_graph=True)
    dx_ref, dw_ref, db_ref = [_.grad.clone() for _ in [x, weight, bias]]
    # compare
    torch.testing.assert_close(y_tri, y_ref, atol=1e-2, rtol=0)
    torch.testing.assert_close(dx_tri, dx_ref, atol=1e-2, rtol=0)
    torch.testing.assert_close(db_tri, db_ref, atol=1e-2, rtol=0)
    torch.testing.assert_close(dw_tri, dw_ref, atol=1e-2, rtol=0)

if __name__ == "__main__":
    # Test.
    print("Testing layer norm with elementwise affine...", end="")
    test_layer_norm_with_affine(1151, 8192, torch.float16)
    print("\N{check mark}")

The output is:
Testing layer norm with elementwise affine...Traceback (most recent call last):
  File "/users/pewalsh/LLM/scratch/05_layer_norm.py", line 291, in <module>
    test_layer_norm_with_affine(1151, 8192, torch.float16)
  File "/users/pewalsh/LLM/scratch/05_layer_norm.py", line 284, in test_layer_norm_with_affine
    torch.testing.assert_close(db_tri, db_ref, atol=1e-2, rtol=0)
  File "/usr/local/lib/python3.10/dist-packages/torch/testing/_comparison.py", line 1511, in assert_close
    raise error_metas[0].to_error(msg)
AssertionError: Tensor-likes are not close!

Mismatched elements: 8173 / 8192 (99.8%)
Greatest absolute difference: 11.775390625 at index (5869,) (up to 0.01 allowed)
Greatest relative difference: 24855.63945578231 at index (808,) (up to 0 allowed)

@scxiao Can you give it a try?

@scxiao Can you give it a try?

OK, will take a look.

I'm still trying to get this running. Sometimes it works, sometimes it doesn't. 80% of the time it just fails at the compile stage with this:
File "/users/pewalsh/.local/lib/python3.10/site-packages/triton/compiler/compiler.py", line 604, in __getattribute__
  self._init_handles()
File "/users/pewalsh/.local/lib/python3.10/site-packages/triton/compiler/compiler.py", line 595, in _init_handles
  mod, func, n_regs, n_spills = fn_load_binary(self.metadata["name"], self.asm[bin_path], self.shared, device)
SystemError: <built-in function load_binary> returned NULL without setting an exception

Any idea what that's about?

Thanks @scxiao, it would be difficult to reproduce the compile error since this only occurs on a server where we're not able to run Docker. Could the compile error be from a ROCm version mismatch? Is there a way to get more information from the error message?

The load binary function finally goes to the function at: https://github.com/ROCmSoftwarePlatform/triton/blob/a8574be74d68abfc476c34be58a4dff16b8c2888/python/triton/runtime/backends/hip.c#L45. Maybe you can print some information from this function.
Also, to get more compilation info, you can enable the three env vars,
https://github.com/ROCmSoftwarePlatform/triton/blob/a8574be74d68abfc476c34be58a4dff16b8c2888/scripts/amd/build.sh#L8-L10
then run your program, a lot of IRs will be printed, you can dump them and post here, then we will get a better idea of which pass is wrong.

Hi @epwalsh, for the original incorrect results, could you please try the latest docker image: rocm/pytorch:rocm6.0_ubuntu20.04_py3.9_pytorch_2.1.1? I tried it, now it can generate correct results.

The issue is gone, so close this issue.


Uncaught TypeError: Cannot read property Error via PIXI

Launched examples. Everything works (except weird position, not scaling into some widgets), but when I swap formats of HelloDragonBones (lines 34/35) to JSON, that are commented there (I assume as an example), I get this error:
HelloDragonBones.js:42 Uncaught TypeError: Cannot read property 'resource/mecha_1002_101d_show/mecha_1002_101d_show_ske.json' of undefined     at HelloDragonBones._onStart (HelloDragonBones.js:42)     at e.<anonymous> (BaseDemo.js:56)     at e.a.emit (pixi.min.js:8)     at pixi.min.js:8     at t.value (pixi.min.js:8)     at e._onComplete (pixi.min.js:8)     at pixi.min.js:8     at s (pixi.min.js:8)     at e.os.use (pixi.min.js:8)     at pixi.min.js:8
Needed that format (JSON), because all of my assets are in it.
Also all demo projects are outdated with depreciated functions from Pixi previous versions.

found a fix... looks like a syntax error:
there was an 's' missing in "_pixiResources":
//factory.parseDragonBonesData(this._pixiResource["resource/mecha_1002_101d_show/mecha_1002_101d_show_ske.json"].data);


README: describe startup script actions

We should explain how the startup script works and what it does (+ why)

Closing since issue has been resolved


Does not build with GHC 8.0.2

Trying to build with the Stack lts-8.16 release fails with the following error:
hssqlppp-0.6.1: build (lib + test)
Log files have been written to: /home/ayo/hs/hssqlppp/.stack-work/logs/
Progress: 1/4
--  While building package hssqlppp-0.6.1 using:
      /home/ayo/.stack/setup-exe-cache/x86_64-linux/Cabal-simple_mPHDZzAJ_<IP_ADDRESS>_ghc-8.0.2 --builddir=.stack-work/dist/x86_64-linux/Cabal-<IP_ADDRESS> build lib:hssqlppp test:Tests --ghc-options " -ddump-hi -ddump-to-file"
    Process exited with code: ExitFailure 1
    Logs have been written to: /home/ayo/hs/hssqlppp/.stack-work/logs/hssqlppp-0.6.1.log

    Configuring hssqlppp-0.6.1...
    Preprocessing library hssqlppp-0.6.1...
    Preprocessing test suite 'Tests' for hssqlppp-0.6.1...
    [ 1 of 84] Compiling Database.HsSqlPpp.Utils.Here ( tests/Database/HsSqlPpp/Utils/Here.lhs, .stack-work/dist/x86_64-linux/Cabal-<IP_ADDRESS>/build/Tests/Tests-tmp/Database/HsSqlPpp/Utils/Here.o )
    [ 2 of 84] Compiling Database.HsSqlPpp.Utils.GroomUtils ( tests/Database/HsSqlPpp/Utils/GroomUtils.lhs, .stack-work/dist/x86_64-linux/Cabal-<IP_ADDRESS>/build/Tests/Tests-tmp/Database/HsSqlPpp/Utils/GroomUtils.o )

    /home/ayo/hs/hssqlppp/hssqlppp/tests/Database/HsSqlPpp/Utils/GroomUtils.lhs:22:24: error:
        • Expecting one more argument to ‘Exp’
          Expected a type, but ‘Exp’ has kind ‘* -> *’
        • In the type signature:
            groomF :: Show a => (Exp -> Exp) -> a -> String

    /home/ayo/hs/hssqlppp/hssqlppp/tests/Database/HsSqlPpp/Utils/GroomUtils.lhs:22:31: error:
        • Expecting one more argument to ‘Exp’
          Expected a type, but ‘Exp’ has kind ‘* -> *’
        • In the type signature:
            groomF :: Show a => (Exp -> Exp) -> a -> String
make: *** [Makefile:40: test] Error 1

Thanks for the report, I uploaded a fixed version, 0.6.1. I think it was an issue with an updated version of haskell-src-exts.

Latest version builds fine. Thanks.


FaucetStringOf*Test tests send Aruba pipeline definition to OVS/mininet

I am attempting to run the FaucetStringOf*Test tests against a physical Aruba switch, but the test reports that an OFPErrorMsg was found in the debug log during tearDown().
ERROR: test_tagged (__main__.FaucetStackStringOfDPTaggedTest)
All tagged hosts in stack topology can reach each other.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/autotester/faucet/tests/faucet_mininet_test_base.py", line 223, in tearDown
    msg=('debug log has OFPErrorMsgs: %s' % debug_log))
AssertionError: debug log has OFPErrorMsgs

When I look at the debug log, the test is attempting to push a pipeline definition to another DPID and gets an error reported. I'm assuming that the test is attempting to stack a mininet switch with the physical Aruba switch. I think the catch here is that the mininet DPID doesn't support pipeline definition (see ArubaValve.switch_features() in valve.py). Perhaps the request is being sent because there's an Aruba switch involved in the setup?
I’ve attached the hw_switch_config.yaml file from my test setup, as well as the resulting ofchannel log where OFPErrorMsg was found (second line). Note that the DPID reported in the log is different from the DPID configured for the switch. I also don’t see this error message taken in a tcpdump of the ‘em2’ (control-plane) interface.
issue.zip

For what it's worth, at least a hack fix to this is to pass "Open vSwitch" rather than self.hardware to the call to self.get_config() in FaucetStringOfDPTest.build_net(..). In my case, the test case failed when using self.hardware ('Aruba') and passed when using "Open vSwitch".

The multiswitch tests don't yet have a way to configure which switch in the test stack, you want the hardware switch to be. For the moment we now skip string tests when hardware based testing is enabled. There should be an additional way to configure different switches in a multi switch stack.
https://github.com/REANNZ/faucet/commit/986da8dd1b1e34595bf738afcfe5bff114a0c4cb for now - I added a TODO also to add that config infrastructure.


SqlAuthenticationProviderManager.ctor throws an Exception when called from Unmanaged Code

Hello,
we tried migrating from System.Data.SQLClient to Microsoft.Data.SQLClient, and everything seems to work fine except when calling the Code from COM Interop,
the class SqlAuthenticationProviderManager tries to to call System.Configuration.ConfigurationManager.GetSection with "SqlAuthenticationProviders" as Parameter,
however, ConfigurationManager.GetSection will throw an Exception since it always uses System.Reflection.Assembly.GetEntryAssembly().Location, which is null when called from COM Interop.
While the code is in a try catch block, only ConfigurationErrorsException's  are caught but the ClientConfigPath class throws a PlatformNotSupportedException if the EntryAssembly is null.
System.TypeInitializationException
HResult=0x80131534
Message=The type initializer for 'Microsoft.Data.SqlClient.SqlAuthenticationProviderManager' threw an exception.
Source=Microsoft.Data.SqlClient
StackTrace:
at Microsoft.Data.SqlClient.SqlInternalConnectionTds..ctor(DbConnectionPoolIdentity identity, SqlConnectionString connectionOptions, SqlCredential credential, Object providerInfo, String newPassword, SecureString newSecurePassword, Boolean redirectedUserInstance, SqlConnectionString userConnectionOptions, SessionData reconnectSessionData, Boolean applyTransientFaultHandling, String accessToken, DbConnectionPool pool, SqlAuthenticationProviderManager sqlAuthProviderManager)
at Microsoft.Data.SqlClient.SqlConnectionFactory.CreateConnection(DbConnectionOptions options, DbConnectionPoolKey poolKey, Object poolGroupProviderInfo, DbConnectionPool pool, DbConnection owningConnection, DbConnectionOptions userOptions)
at Microsoft.Data.ProviderBase.DbConnectionFactory.CreatePooledConnection(DbConnectionPool pool, DbConnection owningObject, DbConnectionOptions options, DbConnectionPoolKey poolKey, DbConnectionOptions userOptions)
at Microsoft.Data.ProviderBase.DbConnectionPool.CreateObject(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection)
at Microsoft.Data.ProviderBase.DbConnectionPool.UserCreateRequest(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection)
at Microsoft.Data.ProviderBase.DbConnectionPool.TryGetConnection(DbConnection owningObject, UInt32 waitForMultipleObjectsTimeout, Boolean allowCreate, Boolean onlyOneCheckConnection, DbConnectionOptions userOptions, DbConnectionInternal& connection)
at Microsoft.Data.ProviderBase.DbConnectionPool.TryGetConnection(DbConnection owningObject, TaskCompletionSource1 retry, DbConnectionOptions userOptions, DbConnectionInternal& connection)    at Microsoft.Data.ProviderBase.DbConnectionFactory.TryGetConnection(DbConnection owningConnection, TaskCompletionSource1 retry, DbConnectionOptions userOptions, DbConnectionInternal oldConnection, DbConnectionInternal& connection)
at Microsoft.Data.ProviderBase.DbConnectionInternal.TryOpenConnectionInternal(DbConnection outerConnection, DbConnectionFactory connectionFactory, TaskCompletionSource1 retry, DbConnectionOptions userOptions)    at Microsoft.Data.ProviderBase.DbConnectionClosed.TryOpenConnection(DbConnection outerConnection, DbConnectionFactory connectionFactory, TaskCompletionSource1 retry, DbConnectionOptions userOptions)
at Microsoft.Data.SqlClient.SqlConnection.TryOpen(TaskCompletionSource1 retry)    at Microsoft.Data.SqlClient.SqlConnection.Open()    at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.OpenDbConnection(Boolean errorsExpected)    at Microsoft.EntityFrameworkCore.Storage.RelationalConnection.Open(Boolean errorsExpected)    at Microsoft.EntityFrameworkCore.Storage.RelationalCommand.ExecuteReader(RelationalCommandParameterObject parameterObject)    at Microsoft.EntityFrameworkCore.Query.Internal.QueryingEnumerable1.Enumerator.InitializeReader(DbContext _, Boolean result)
at Microsoft.EntityFrameworkCore.SqlServer.Storage.Internal.SqlServerExecutionStrategy.Execute[TState,TResult](TState state, Func3 operation, Func3 verifySucceeded)
at Microsoft.EntityFrameworkCore.Query.Internal.QueryingEnumerable1.Enumerator.MoveNext()    at System.Linq.Enumerable.SingleOrDefault[TSource](IEnumerable1 source)
at Microsoft.EntityFrameworkCore.Query.Internal.QueryCompiler.Execute[TResult](Expression query)
at Microsoft.EntityFrameworkCore.Query.Internal.EntityQueryProvider.Execute[TResult](Expression expression)
at System.Linq.Queryable.FirstOrDefault[TSource](IQueryable`1 source)
at Nippon.Persistance.Controller.MFPController.GetMFP(String mfp) in E:\Nippon.NET\7. Persistance Layer\Nippon.Persistance\Controller\MFPController.cs:line 29
at Nippon.Common.Interop.Base.VB6InteropWindow..ctor() in E:\Nippon.NET\2. Presentation Layer\WPF\Nippon.Common.Interop\Base\VB6InteropWindow.cs:line 34
at Nippon.Controls.Views.Bestand..ctor() in E:\Nippon.NET\2. Presentation Layer\WPF\Nippon.Common.Interop\Views\Bestand\Bestand.xaml.cs:line 38
This exception was originally thrown at this call stack:
System.Configuration.ClientConfigPaths.ClientConfigPaths(string, bool)
System.Configuration.ClientConfigPaths.GetPaths(string, bool)
System.Configuration.ClientConfigurationHost.ConfigPaths.get()
System.Configuration.ClientConfigurationHost.GetStreamName(string)
System.Configuration.ClientConfigurationHost.IsAppConfigHttp.get()
System.Configuration.Internal.DelegatingConfigHost.IsAppConfigHttp.get()
System.Configuration.ClientConfigurationSystem.ClientConfigurationSystem()
System.Configuration.ConfigurationManager.EnsureConfigurationSystem()
Inner Exception 1:
InvalidOperationException: Failed to read the config section for authentication providers.
Inner Exception 2:
ConfigurationErrorsException: Configuration system failed to initialize
Inner Exception 3:
PlatformNotSupportedException: Operation is not supported on this platform.
To reproduce
Make sure System.Reflection.Assembly.GetEntryAssembly() is null by calling from any non .NET Code, open any Microsoft.Data.SqlClient.SqlConnection with .Open() causes an InvalidOperationException
Expected behavior
The behavior can stay the same however there should be some sort of new  SqlServerDbContextOptionsBuilder option to use an AlternateConfigurationExe when calling from unmanaged Code
Further technical details
Microsoft.Data.SqlClient version: 1.1.1
.NET target: Core 3.1
SQL Server version: SQL Server 2019

Hi @Kylon92
Could you please provide us a repro app to reproduce the problem?

Hi,
i made a Testproject to demonstrate the Issue, you can find it at https://github.com/Kylon92/issue500test
While making it i noticed that C# COM Consumer to C# Com Server does not trigger the issue as System.Reflection.Assembly.GetEntryAssembly() is filled.
However when you test it with VBA from Excel (ComConsumerVBA.xlsm) or with VB6 is will crash as soon as you try to open a Microsoft.Data.SQLClient.SQLConnection.
I have included a VBA Testcase and i recommend using it as the VB6 Testproject requires to run a setup(VB6 Consumer/dist/setup.exe) to install missing vb6 runtime DLLs

I believe PR #573 will also fix this issue, @Kylon92 could you give it a try and let us know?

@cheenamalhotra I tried it and now it works, thank you.


initializing stats takes over a minute when starting jmmserver

When first loading JMMServer it cycles through its operations till it gets to initializing stats then I have to wait over a minute for it to complete. From my log file this one takes the longest.
=> StatsCache.InitStats GetAllGroups (Contracts) in 62134.8 ms
Todays log file is here
http://s000.tinyupload.com/?file_id=00240151331242114394
Thanks

this is far from uncommon, mine takes about 3min and 50sec.
the more series and groups you have the slower the start will be, aswell as how many tasks you have on startup.
its an media server, it suppose to be running in the background at all time and not just opened when you use it.
1-5min startup doesnt hurt it TBH as its an server. if it was JMM desktop in the other hand i would stress it for a speedup.

StatCache Initialization was already slow, and on the top i make it slower caching Plex metadata, currently I cannot find a better solution right now without changing the way JMMServer works or loosing Plex speed.

ok thanks for the replies

Hi @xsystematic, one last question.
Roughly how many series do you have?
Other things that will affect start up speed is the number of group filters you have.
And also the number of users you have -> You can delete any users you don't plan on using.

Around 170 or so. I have managed to get it down to about 16 seconds on initializing stats. I did this by changing my processors overclock settings to optimized (from disabled) and changing a few other BIOS settings.


Categorial to numeric

FirstLevelFeaturesProcessor renamed to ToNumericFeatureTransformer
SecondLevelFeaturesProcessor renamed to HistoryBasedFeaturesProcessor
HistoryBasedFeaturesProcessor split on two different feature processors, LogStatFeaturesProcessor and ConditionalPopularityProcessor.
added creating additional binary columns showing where nulls were replaced after joining generated features in LogStatFeaturesProcessor and ConditionalPopularityProcessor
processing is divided into staged (protected static methods) in LogStatFeaturesProcessor
HistoryBasedFeaturesProcessor, LogStatFeaturesProcessor and ConditionalPopularityProcessor are moved from scenario module to upper-level
two timestamp-related features added to LogStatFeaturesProcessor (history length in days and day difference between last log date and last interaction with the user/item.

closes #94
closes #93


Selections of changes within the undo/redo histories must be updated on remote change

ATM these changes get out of date when applying external changes.

If I change the selection within a transaction using args.selection = ..., on undo, the selection doesn't change. Is this issue related to that? Should I open a new one? Please read for a brief explanation.
eg. Inside a transaction that converts heading h-a to paragraph p-b,

first create p-b,
transfer annotations,
transfer selection,
delete h-a

Now, if I undo,

h-a is recreated,
annotations are transferred,
p-b is deleted.

However, selections are not transferred back to h-a. This causes the Tool.getCommandState() methods to go haywire.
Thanks!

@schatten Basically I think that is related, yes. There is more to it, because we don't have a proper notion of transferring coordinates. I.e. such cases need to be considered OT wise.
I will make an integration test out of your description.

Thx for reporting!


Fix: RuntimeError: 'weight' must be 2-D issue

Fix #669
Problem description
In ZeRO3, issues like #669 are caused by running deepspeed.initialize on only one of the two models passed to DPO_Trainer.
Most users use .from_pretrained to get the weight of the model, and inside from_pretrained is the code below.
    if is_deepspeed_zero3_enabled():
        import deepspeed

        logger.info("Detected DeepSpeed ZeRO-3: activating zero.init() for this model")
        init_contexts = [deepspeed.zero.Init(config_dict_or_path=deepspeed_config())] + init_contexts
    elif load_in_8bit or low_cpu_mem_usage:
        init_contexts.append(init_empty_weights())

    with ContextManagers(init_contexts):
        model = cls(config, *model_args, **model_kwargs)

Models that don't use ZeRO3 don't matter,
But most of the heavier models like LLM are run using ZeRO3, so from_pretrained, is_deepspeed_zero3_enabled() becomes True, it have to use the if statement.
But since ZeRO3 is parameter partitioning, does this mean that it need to use the _partition_param in deepspeed.zero.Init to divide the parameters in one layer by the number of GPUs?
At this time, the partitioned parameter is put into a .ds_tensor, and parameter.data(or weight, bias), where the original parameter was, is filled with a zero tensor of size 0.
So far, so good
The problem is that we need to do a deepspeed.initialize when training to load the partitioned parameters in .ds_tensor,
But ref_model didn't do deepspeed.initialize, so it can't load the .ds_tensor.
This causes an error like RuntimeError: 'weight' must be {n}-D because ref_model's parameter.data does not contain weight, as shown in the issue.
So to solve the issue, it need to deepspeed.initialize ref_model as well as model.
However, since prepare_model does not have a deepspeed wrapper,
so it need to add the process of wrapping the model with deepspeed using _prepare_deepspeed to the init.

@lvwerra Can you review this PR if you don't mind?

Thanks for the PR. A quick question: does deep speed allow you to initialize multiple models? I seem to have run into some related issue.

The docs for this PR live here. All of your documentation changes will be reflected on that endpoint.

here a example code
from typing import Dict

import torch
import torch._dynamo
from datasets import Dataset, load_dataset
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    HfArgumentParser,
    TrainingArguments,
)
from trl import DPOTrainer

def extract_anthropic_prompt(prompt_and_response):
    """Extract the anthropic prompt from a prompt and response pair."""
    search_term = "\n\nAssistant:"
    search_term_idx = prompt_and_response.rfind(search_term)
    assert search_term_idx != -1, f"Prompt and response does not contain '{search_term}'"
    return prompt_and_response[: search_term_idx + len(search_term)]

def get_hh(
    split: str, sanity_check: bool = False, silent: bool = False, cache_dir: str = None
) -> Dataset:
    """Load the Anthropic Helpful-Harmless dataset from Hugging Face and convert it to the necessary format.

    The dataset is converted to a dictionary with the following structure:
    {
        'prompt': List[str],
        'chosen': List[str],
        'rejected': List[str],
    }

    Prompts should be structured as follows:
      \n\nHuman: <prompt>\n\nAssistant:
    Multiple turns are allowed, but the prompt should always start with \n\nHuman: and end with \n\nAssistant:.
    """
    dataset = load_dataset("Anthropic/hh-rlhf", split=split, cache_dir=cache_dir)
    if sanity_check:
        dataset = dataset.select(range(min(len(dataset), 1000)))

    def split_prompt_and_responses(sample) -> Dict[str, str]:
        prompt = extract_anthropic_prompt(sample["chosen"])
        return {
            "prompt": prompt,
            "chosen": sample["chosen"][len(prompt) :],
            "rejected": sample["rejected"][len(prompt) :],
        }

    return dataset.map(split_prompt_and_responses)

def main():
    training_args = TrainingArguments(
        per_device_train_batch_size=1,
        max_steps=10,
        remove_unused_columns=False,
        gradient_accumulation_steps=1,
        learning_rate=0.00001,
        evaluation_strategy="no",
        save_strategy="no",
        logging_strategy="steps",
        logging_steps=1,
        output_dir="./",
        report_to="none",
    )

    model_name = "circulus/Llama-2-7b-orca-v1"
    tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        device_map="auto",
        torch_dtype=torch.bfloat16,
    )
    model_ref = AutoModelForCausalLM.from_pretrained(
        model_name,
        torch_dtype=torch.float16,
        trust_remote_code=True,
    )

    tokenizer.bos_token = "<s>"
    tokenizer.eos_token = "</s>"
    tokenizer.pad_token = tokenizer.eos_token

    with training_args.main_process_first():
        train_dataset = get_hh("train", sanity_check=True)
        eval_dataset = get_hh("test", sanity_check=True)

    dpo_trainer = DPOTrainer(
        model,
        model_ref,
        args=training_args,
        beta=0.1,
        train_dataset=train_dataset,
        eval_dataset=eval_dataset,
        tokenizer=tokenizer,
        max_length=2048,
        max_prompt_length=128,
    )

    # 6. train
    dpo_trainer.train()
    if training_args.local_rank == 0:
        model.save_pretrained(training_args.output_dir)
        tokenizer.save_pretrained(training_args.output_dir)

if __name__ == "__main__":
    main()

I think the test doesn't pass if the ref_model is None, and I need to do a little more work on that.

we found the cause of the crash on test:
if we didn't specify the path to the config file for deepspeed in the TrainingArgument,
it would be replaced with None, which would be true when compared to
self.args.deepspeed != "", causing an error like AttributeError: 'NoneType' object has no attribute 'deepspeed_config'.
So I changed self.args.deepspeed != "" to self.is_deepspeed_enabled

Has this been addressed for PPOTrainer as well? I am getting this issue when trying to use ZeRO3, but inserting the lines
    ppo_trainer.ref_model = ppo_trainer.accelerator._prepare_deepspeed(ppo_trainer.ref_model)[0]
    ppo_trainer.ref_model.eval()

leads to the following error:
AssertionError: You can't use same Accelerator() instance with multiple models when using DeepSpeed

@andrew-zm-ml yes, we added full ZeRO-{1,2,3} integration for PPOTrainer in #758 which takes care of the limitations around only having 1 model per Accelerator()


Feedback for Other vendors page

In this article
Maintaining a strong community
What if something or someone offends you?
What happens if someone violates GitHub's policies?
Appeal and Reinstatement
Legal Notices

In this article
Maintaining a strong community
What if something or someone offends you?
What happens if someone violates GitHub's policies?
Appeal and Reinstatement
Legal Notices

In this article
Maintaining a strong community
What if something or someone offends you?
What happens if someone violates GitHub's policies?
Appeal and Reinstatement
Legal Notices

In this article
Maintaining a strong community
What if something or someone offends you?
What happens if someone violates GitHub's policies?
Appeal and Reinstatement
Legal Notices


jekyll 3.8.5 | Error:  Permission denied @ dir_s_mkdir - /src/jekyll/_site

Description
Using the "latest" Docker image of Jekyll with a very minimal Dockerfile:
FROM jekyll/jekyll
COPY . /src/jekyll
RUN chown -R jekyll /src/jekyll

ENTRYPOINT ["jekyll", "serve"]

WORKDIR /src/jekyll

EXPOSE 4000

CMD ["jekyll", "serve", "--verbose"]

This WAS working until a few days ago, so it must be something recent that was changed in the latest Docker image. I see that the [latest Docker image update is from 4 hours ago](Last update: 4 hours ago).
A temporary solution is to fix the version inside the Dockerfile: FROM jekyll/jekyll:3.8.4
Running Docker Compose under Ubuntu 18.04.
Output
api_docs_1  | ruby 2.6.2p47 (2019-03-13 revision 67232) [x86_64-linux-musl]
api_docs_1  |   Logging at level: debug
api_docs_1  | Configuration file: /src/jekyll/_config.yml
api_docs_1  |        Deprecation: The 'pygments' configuration option has been renamed to 'highlighter'. Please update your config file accordingly. The allowed values are 'rouge', 'pygments' or null.
api_docs_1  |   Logging at level: debug
api_docs_1  |             Source: /src/jekyll
api_docs_1  |        Destination: /src/jekyll/_site
api_docs_1  |  Incremental build: disabled. Enable with --incremental
api_docs_1  |       Generating...
api_docs_1  |            Reading: _posts/2000-01-01-authentication-01.md
api_docs_1  |            Reading: _posts/2000-02-01-audiobooks-index.md
api_docs_1  |            Reading: _posts/2000-01-02-authentication-02.md
api_docs_1  |            Reading: _posts/2000-02-03-audiobooks-download.md
api_docs_1  |            Reading: _posts/2000-02-02-audiobooks-view.md
api_docs_1  |            Reading: _posts/2000-02-03-audiobooks-favorite.md
api_docs_1  |            Reading: _posts/2012-12-28-response-status-codes.md
api_docs_1  |          Rendering: _posts/2000-01-01-authentication-01.md
api_docs_1  |   Pre-Render Hooks: _posts/2000-01-01-authentication-01.md
api_docs_1  |   Rendering Markup: _posts/2000-01-01-authentication-01.md
api_docs_1  |   Rendering Layout: _posts/2000-01-01-authentication-01.md
api_docs_1  |          Rendering: _posts/2000-01-02-authentication-02.md
api_docs_1  |   Pre-Render Hooks: _posts/2000-01-02-authentication-02.md
api_docs_1  |   Rendering Markup: _posts/2000-01-02-authentication-02.md
api_docs_1  |   Rendering Layout: _posts/2000-01-02-authentication-02.md
api_docs_1  |          Rendering: _posts/2000-02-01-audiobooks-index.md
api_docs_1  |   Pre-Render Hooks: _posts/2000-02-01-audiobooks-index.md
api_docs_1  |   Rendering Markup: _posts/2000-02-01-audiobooks-index.md
api_docs_1  |   Rendering Layout: _posts/2000-02-01-audiobooks-index.md
api_docs_1  |          Rendering: _posts/2000-02-02-audiobooks-view.md
api_docs_1  |   Pre-Render Hooks: _posts/2000-02-02-audiobooks-view.md
api_docs_1  |   Rendering Markup: _posts/2000-02-02-audiobooks-view.md
api_docs_1  |   Rendering Layout: _posts/2000-02-02-audiobooks-view.md
api_docs_1  |          Rendering: _posts/2000-02-03-audiobooks-download.md
api_docs_1  |   Pre-Render Hooks: _posts/2000-02-03-audiobooks-download.md
api_docs_1  |   Rendering Markup: _posts/2000-02-03-audiobooks-download.md
api_docs_1  |   Rendering Layout: _posts/2000-02-03-audiobooks-download.md
api_docs_1  |          Rendering: _posts/2000-02-03-audiobooks-favorite.md
api_docs_1  |   Pre-Render Hooks: _posts/2000-02-03-audiobooks-favorite.md
api_docs_1  |   Rendering Markup: _posts/2000-02-03-audiobooks-favorite.md
api_docs_1  |   Rendering Layout: _posts/2000-02-03-audiobooks-favorite.md
api_docs_1  |          Rendering: _posts/2012-12-28-response-status-codes.md
api_docs_1  |   Pre-Render Hooks: _posts/2012-12-28-response-status-codes.md
api_docs_1  |   Rendering Markup: _posts/2012-12-28-response-status-codes.md
api_docs_1  |   Rendering Layout: _posts/2012-12-28-response-status-codes.md
api_docs_1  |          Rendering: index.html
api_docs_1  |   Pre-Render Hooks: index.html
api_docs_1  |   Rendering Liquid: index.html
api_docs_1  |   Rendering Markup: index.html
api_docs_1  |   Rendering Layout: index.html
api_docs_1  |      Layout source: site
api_docs_1  | jekyll 3.8.5 | Error:  Permission denied @ dir_s_mkdir - /src/jekyll/_site

Any permissions errors must originate from the underlying host (hypervisor) as we do not control your systems permissions. On a rare occasion they can originate from this image, but most of the time they do not.  Considering this image hasn't actually updated since December for 2018... I would need more information than "a random permission error suddenly popped up".
The daily builds that happen with this image apply security updates to Ruby, and Alpine... that is it.

I see your point, however, I still don't understand why, with the same absolutely identical project and the same folder permissions, I get the effect below.
This works:
FROM jekyll/jekyll:3.8.4
This doesn't work:
FROM jekyll/jekyll:latest
My immediate (logical) assumption was that something changed in the Docker image of Jekyll. Anyway, feel free to close this ticket. The workaround of a fixed version works for me. Just thought this information should be out there for future readers who might stumble upon the same issue.

Are you sure this isn't a Jekyll error?

I have no way to tell. I am fine with the closure of the issue though. So long as it doesn't impact the general public.


Azure AD user preferredLanguage is not an ISO 639-1 code

https://docs.microsoft.com/en-us/azure/active-directory-b2c/user-profile-attributes#azure-ad-user-resource-type

preferredLanguage
The preferred language for the user. Should follow ISO 639-1 Code. Example: "en-US".

ISO 639-1 is a two-letter code, e.g. "en".
"en-US" is a correct example for an Azure AD's user's preferredLanguage attribute, but it isn't an ISO 639-1 code.
What it's called, exactly, is hard to say in a pithy way.  This resource may help in describing it:-
https://docs.microsoft.com/en-us/dotnet/api/system.globalization.cultureinfo.name?view=net-5.0

Document Details
⚠ Do not edit this section. It is required for docs.microsoft.com ➟ GitHub issue linking.

ID: b30bc6cd-5d1a-6966-747e-ec03607d8967
Version Independent ID: 5e6fd653-e232-3a19-3739-7144ea587c51
Content: User profile attributes in Azure Active Directory B2C
Content Source: articles/active-directory-b2c/user-profile-attributes.md
Service: active-directory
Sub-service: b2c
GitHub Login: @msmimart
Microsoft Alias: mimart

@Andrew-Webb
Thanks for your feedback! We will investigate and update as appropriate.

@Andrew-Webb
Thanks for your feedback! We will investigate and update as appropriate.

@Andrew-Webb
Thank you for pointing this out! I've assigned this issue to the author who will investigate and update as appropriate.

@Andrew-Webb
Thank you for pointing this out! I've assigned this issue to the author who will investigate and update as appropriate.

Thanks @ Andrew-Webb for bringing this to our attention. The fix will be published later today with correct language code.
Yoel


openra: 20190314 -> 20230225

Description of changes

While technically an update, the huge change between the last version and the current one requires complete overhaul of how openra is build. Therefore, in this PR I moved the old openra derivations into a openraPackages_2019 namespace, and in the openra namespace, I introduced a completely new build, which uses buildDotnetModule (since openra is a fully dotnet app).
Everything seems to work correctly. I consider external mods out of scope of this PR, since it will require a lot more work to update them - that's why I left them in the openraPackages_2019
Currently I added 2 engine versions:

release, from the latest release tag
bleed, from the latest commit to bleed

The playtest version is pretty old, so I didn't add it for now.
This closes https://github.com/NixOS/nixpkgs/issues/96600
Things done

Built on platform(s)

[x] x86_64-linux
[ ] aarch64-linux
[ ] x86_64-darwin
[ ] aarch64-darwin

[ ] For non-Linux: Is sandbox = true set in nix.conf? (See Nix manual)
[x] Tested, as applicable:

NixOS test(s) (look inside nixos/tests)
and/or package tests
or, for functions and "core" functionality, tests in lib/tests or pkgs/test
made sure NixOS tests are linked to the relevant packages

[ ] Tested compilation of all packages that depend on this change using nix-shell -p nixpkgs-review --run "nixpkgs-review rev HEAD". Note: all changes have to be committed, also see nixpkgs-review usage
[x] Tested basic functionality of all binary files (usually in ./result/bin/)
23.05 Release Notes (or backporting 22.11 Release notes)

[ ] (Package updates) Added a release notes entry if the change is major or breaking
[ ] (Module updates) Added a release notes entry if the change is significant
[ ] (Module addition) Added a release notes entry if adding a new NixOS module

[x] Fits CONTRIBUTING.md.

I can't comment on the code itself, but can confirm that this builds and (the default ra mod) runs on my Linux machine.
Super cool!

This pull request has been mentioned on NixOS Discourse. There might be relevant details there:
https://discourse.nixos.org/t/prs-ready-for-review/3032/1966

This pull request has been mentioned on NixOS Discourse. There might be relevant details there:
https://discourse.nixos.org/t/prs-ready-for-review/3032/2253

Is the PR OK to merge in the current state? Or is there anything else that I'd need to update to make it possible to be merged?

Visually it looks good enough to me.
Anyone disagrees? If not, I will commit it in 3 days.


Update the collate_fn for PYGGraphTransformer

Motivation
For the latest version of torch_geometric 2.4.0, the Collater was refactored and require an argument dataset.
Therefore, the current implementation of PYGGraphTransformer.get_collate_fn is no longer  valid.
Pitch
The argument dataset should also be added in the get_collate_fn.
def get_collate_fn(
        self,
        dataset: Union[Dataset, Sequence[BaseData], DatasetAdapter],
        follow_batch: Optional[List[str]] = None,
        exclude_keys: Optional[List[str]] = None,
        return_pair: Optional[bool] = True,
        **kwargs,
    ):
        collator = Collater(dataset=dataset, follow_batch=follow_batch, exclude_keys=exclude_keys)
        return partial(self._collate_batch, collator=collator, return_pair=return_pair)

Additional context
No response

Thanks @zhu0619, can you make a PR ?

@zhu0619, can you document the alternative of using the pyg DataLoader object directly somewhere ?
It allows actually not providing any collate function


Update translations, improve workflow to keep updated

Translations seem to go out of date very easily, we should have process around keeping this updated and generating support for translations.
Perhaps this is a fabric task or part of something larger, like release automation.
The documentation on translation could also be updated probably.

Related to https://github.com/rtfd/readthedocs.org/issues/3329

#3329 is unrelated. It describes a feature that allows for Transifex integration for users.
Some of this is updated in #4645, there is invoke tasks for helping. Closing this in favor of #4463


[C++] Provide a method to read a string correctly

For a string (using CME's "Symbol" as an example), the generated APIs are:

const char *symbol(void): generally incorrect and unsafe -- if the encoded field contains a maximum-length string, the returned pointer will point to an unterminated string.
sbe_char_t symbol(const int index) const: this has its range checked, but it's barely useful in practice unless you really want character-by-character access.
void symbol(const int index, const sbe_char_t value): not a getter at all.  (In general, naming getters and setters the same thing is not very friendly to users.)
```int getSymbol(char *dst, const int length) const````: no more useful than any of the other APIs.
static const int symbolLength(void): returns the static length, not the actual length.

A convenience method for return std::string (using move semantics) will be evaluated for data.

Unless I am mistaken, CME  Symbol is an array of char. Which has a fixed length. So, most of what is mentioned is actually incorrect. For example, the const char * return method is quite safe when used appropriately (since what is pointed to is a fixed length).
Encoding and decoding individual elements of an array, i would hope, makes sense.
Copying the entire array to use should also make sense.
And lastly, length should make sense since by the SBE spec, it is a static length. The actual length has no meaning for an array of static length. It is how the app interprets it.
Still, as a convenience, it would be nice if arrays of char types generated std::string as well as data fields.
So, I am changing the title to more accurately reflect the "general" request of std::string interaction.


Bypass checking URLs in comments for 'max-line-length' rule

The case is we have some long URLs in comments in our code base . These URLs sometimes are quite long, so we have to use tslint:disable-next-line:max-line-length before long URLs. My feature request/suggestion is to modify max-line-length rule to allow long URLs in comments, or add an option to allow such URLs.

There was a similar request to allow long lines for import statements. IIRC the resolution was that tslint won't fix this and you should just disable the rule for that line.
Personally I'd like to add some options to the rule, too.
In the mean you can use max-len from tslint-eslint-rules

Just a suggestion: maybe it makes sense to just add an option like ignoreRegexp which will contain array of regular expressions?..

@roman-petrov, we have the same issue with our code. To give an example:
/**
 * Definition of OAuth 2.0 Scopes for Google Plus.
 * @see https://developers.google.com/resources/api-libraries/documentation/plus/v1/java/latest/com/google/api/services/plus/PlusScopes.html
 */
export default class PlusScopes {
  // Know the list of people in your circles, your age range, and language
  public static PLUS_LOGIN = 'https://www.googleapis.com/auth/plus.login';
  // Know who you are on Google
  public static PLUS_ME = 'https://www.googleapis.com/auth/plus.me';
  // View your email address
  public static USERINFO_EMAIL = 'https://www.googleapis.com/auth/userinfo.email';
  // View your basic profile info
  public static USERINFO_PROFILE = 'https://www.googleapis.com/auth/userinfo.profile';
}

The URL next to @see is too long when using "extends": "tslint:recommended" in tslint.json.
In ESLint there is a ignoreComments option which lets you bypass long comments. Would be nice if TSLint had something similar.

We'd take a PR for a rule option that exempts URLs from the rule.

@adidahiya  please also long regular expressions.

@xjose97x I'm reluctant to add regular expressions for now because of the performance impact. Can you describe your use cases for regex?

@adidahiya I have a regex for checking email addresses:
/^(([^<>()[]\.,;:\s@"]+(.[^<>()[]\.,;:\s@"]+)*)|(".+"))@(([[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}])|(([a-zA-Z-0-9]+.)+[a-zA-Z]{2,}))$/
Which makes tslint bring up the max-line-length error. I actually like that tslint rule, but it should be disabled for cases like regex and urls.

ah, sorry, I thought you were asking for a rule option that takes a regex to whitelist certain kinds of text from the max-line-length rule. exempting regex syntax itself is much simpler; we can also take a PR for that

I am using the following pattern to ignore long urls. I allow one to three ”other“ characters at the beginning to allow for listings (- http://....).
"max-line-length": [
  true,
  {
    "ignore-pattern": "^\\s*// *(.{1,3} )?(https?)://[^ ]+$",
    "limit": 80
  }
]

Similarly to @myrdd, I'm ignoring any line that contains 'http'
"max-line-length": [
  true,
  {
    "ignore-pattern": "http",
    "limit": 80
  }
]

☠ TSLint's time has come! ☠
TSLint is no longer accepting most feature requests per #4534. See typescript-eslint.io for the new, shiny way to lint your TypeScript code with ESLint. ✨
It was a pleasure open sourcing with you all!


Editing Rule - Problems with JSON and JSONata

Describe the bug
If you try to edit/create a rule with JSON or JSONata as payload, the editor is open in the background. It´s not possible to edit the data.
Screenshots

Expected behavior
The editor should open in front of the rule editor and not behind
System information (please complete the following information):

node-red 2.0.3
contrib-sun 2.1.1

The problem is fixed in the latest version<EMAIL_ADDRESS>Since this version has undergone some changes internally, I have this but still released as beta. That means you can't update it via Node-Red, but only via the command line.


Implement drag support for items in the recent files widget

Describe the solution
Allow items in the recent items widget to be dragged, so that users can easily drag items they were recently using into other applications. For example, the user could drag a document they just opened into mail to attach it.
Describe alternatives you've considered
Copy-paste maybe.
Scope

Capability
Priority

This proposal will allow users to drag items from the recent items widget into other programs
Must

Bumped #8282


[FEATURE] Set (initial) size of list, set, dict

Is there an existing issue for this?

[X] I have searched the existing issues

Is your feature request related to a problem?
It might be helpful to have the ability to specify the initial size of list, set, and dict.  A user would set the initial size of the type but wouldn't prevent them from further extension.
Describe the solution you'd like
var dozen = newList(12);

var daysOfTheWeek = newDict(7);

Describe alternatives you've considered
No response
Additional context
No response

Sounds good to me! I wonder if it's worth having a List / Dict namespace then later on we can add other methods that create these from method calls, i.e:
List.fill(...)
Dict.from([["key", "value"], [...]])

I was thinking of doing that initially but decided to keep it simple at the outset. I'm happy to do whatever you think is best. I have a working implementation but need to find all the memory I'm leaking. 😂

The joys of C right 😂 If it's not too much trouble I think the namespace would be the way to go as it saves us changing it later down the line

No trouble at all. I completely it'll be best.


Updated link to shorthands documentation

Link to shorthands documentation was still pointing to older version v0.3.x.

Thank you!


fix: disable codegen

Summary
Fix codegen by changing types to supported one as it does not support string unions
Motivation
Fix compatibility with codegen
Changes

Updated native view types

Test plan

[x] Tested locally
[ ] CI pass

@KrzysztofMoch changes looks good to me. Don't you think the codegen removal (temporary) should be done in another PR ? It would be easier to revert.
BTW, is it really still a draft ?


Fix for data stores resummarized without changes. (#8990)

Cherry-pick from https://github.com/microsoft/FluidFramework/pull/8990.
Fixes https://github.com/microsoft/FluidFramework/issues/8963.
Issue
When a container is loaded, garbage collector reads the GC blobs for data stores from the base snapshot and passes it to data stores during creation. However, it does not sort the used routes before passing it to the data stores. These used routes become referenceUsedRoutes for the data stores, i.e., used routes in the previous summary.
During summarize, the current used routes are compared with the referenceUsedRoutes to determine if the reference state of the data store changed. The used routes must be sorted so that they can be compared correctly.
Fix
Data stores sort the used routes passed by the container runtime before saving it as referenceUsedRoutes.

@tylerbutler Does this need your approval before it can go in?

No. The only requirement is that you have one review approval. The code owners are asked to review, but you shouldn't be blocked.

I am blocked, it isn't letting me merge the changes:

@agarwal-navin Weird, sorry about that. I confirmed the settings are such that this shouldn't happen, so please let me know if it does. Could be a GitHub bug.

Thanks Tyler. Will let you know if I see this again.


Using plugin without Vim package managers

Hello.
I have problem using some of the plugins similar to this. I'm configuring my work environment using Nix Package manager. In the README.md , it is stated how to use it with Lazy.nvim or Packer.
I installed this plugin using Nix, and sourced to see that requires are working.
This is my Lua script for this plugin:
alpha = require("alpha")
api = vim.api

api.nvim_create_autocmd("VimEnter", {
  callback = function()
    print("This prints, but below is not executed ?")
    alpha.setup(
      require("alpha.themes.dashboard").config
    )
  end
})

I want to show dashboard on startup, but I don't see it. If I run :Alpha, the dashboard is show. Even if I simplify Lua script for this (without VimEnter), it still doesn't work. Any help ?

Update:
When using require in Lua script, it seems like setup method does not exist...

hard to say without seeing the rest of your config. setup registers a VimEnter autocmd, which might be why that's not working. but if you say it's truly not even starting without that autocmd I'm at a loss. how does plugin management work with nix? does it compile your config for you?

I think it's something how I configure vim plugins using Nix. I have contacted maintainers of this package manager for additional help at https://github.com/nix-community/home-manager/issues/3683
I will update here once how to use your plugin with Nix package manager. This could be a good resource for Nix users.

I can confirm that plugin works with Packer correctly. So it's something related to how I setup Neovim using home-manager. Nix users can follow above linked issue.
Closing this issue. Thank you for support 🚀


Add package.xml to allow build in catkin workspace - [merged]

In GitLab by @wxmerkt on Oct 23, 2019, 18:30
Merges topic/wxm-add-package.xml -> devel
I just spent two full days debugging matrix sizes in Pinocchio that exhausted memory (#275) - in the end this was due to compilation issues in configuring different repositories locally on a single machine; every change requires a cascade of reconfiguration & compilation so tracking these things down often leads to going down a rabbit hole.
Adding this package.xml (together with similar changes I am working on) allows building all dependencies as third-party libraries in a catkin workspace. The entire Crocoddyl development workflow with eigenpy, pinocchio, example-robot-data etc. reduces to a single catkin build without any further changes to the buildsystem or CMake files in any of the projects.
As I am the one person onboarding every person at Edinburgh (same for HPP etc. last year) and the only person debugging everybody's problems with the jrl cmake buildsystem at Edinburgh, this will make my life easier and save countless hours.

In GitLab by @wxmerkt on Oct 23, 2019, 18:31
changed the description

In GitLab by @wxmerkt on Oct 23, 2019, 18:34
mentioned in merge request gepetto/example-robot-data!17

In GitLab by @cmastalli on Oct 23, 2019, 18:48
Please add my name as developer and maintainer. Use my personal email address<EMAIL_ADDRESS>

In GitLab by @wxmerkt on Oct 23, 2019, 18:52
added 1 commit
e64e9921 - package.xml: Add Carlos as maintainer and add reference to README.md for author
Compare with previous version

In GitLab by @wxmerkt on Oct 23, 2019, 18:53
resolved all threads

In GitLab by @cmastalli on Oct 23, 2019, 22:30
assigned to @cmastalli

In GitLab by @cmastalli on Oct 23, 2019, 22:37
enabled an automatic merge when the pipeline for e64e992186ba11915d6c009a2edb95e4188f80ca succeeds

In GitLab by @cmastalli on Oct 23, 2019, 22:38
Thank for making the ROS integration

In GitLab by @cmastalli on Oct 23, 2019, 22:58
merged

In GitLab by @cmastalli on Oct 23, 2019, 22:58
mentioned in commit c077a8a46ee5e4bb6da827bd1f591eb167302af8

In GitLab by @wxmerkt on Oct 24, 2019, 10:57
Thank you for addressing this quickly. This is not a real ROS integration, it just allows building in a catkin workspace (as a CMake project using the existing build system) with dependency resolution/management and that works without ROS (and simply catkin/catkin tools). :-)


Title:  Wordle is a popular online word-guessing game

Is your feature request related to a problem? Please describe.
Setup:
The game displays a grid with six rows and five columns, representing the maximum six attempts to guess the correct word.
Players enter their guesses into the grid using the provided input field.
Input and Feedback:
After submitting a guess, each letter of the word is color-coded to indicate its accuracy:
Green: The letter is in the correct position.
Yellow: The letter is in the word but in the wrong position.
Grey: The letter is not in the word at all.
This feedback helps players refine their guesses in subsequent attempts.
Winning and Losing:
The game is won if the player guesses the correct word within six attempts.
If the player fails to guess the word in six tries, the game ends, and the correct word is revealed.
Key Features
Daily Challenge: Wordle presents a new word each day, creating a shared experience among players who all attempt to guess the same word.
No Repeats: Each word is unique, preventing repetition and keeping the game fresh.
Social Sharing: Players often share their results on social media, showcasing their guessing prowess without revealing the actual word.
Describe the solution you'd like.
HTML:
Creates the basic structure of the game, including the board for displaying guesses, an input field for entering guesses, and a button for submitting guesses.
CSS:
Styles the game board and input elements for a clean and centered layout.
JavaScript:
Initializes the game board by creating a grid of tiles.
Handles user input, checking the guess against the solution.
Colors the tiles based on the correctness of each letter (green for correct, yellow for present but wrong position, grey for absent).
This implementation gives you a simple, playable version of Wordle. You can further enhance it by adding features such as a list of possible words for random solutions, better handling of duplicate letters, and more sophisticated UI elements.
Describe alternatives you've considered.
HTML:
Creates the basic structure of the game, including the board for displaying guesses, an input field for entering guesses, and a button for submitting guesses.
CSS:
Styles the game board and input elements for a clean and centered layout.
JavaScript:
Initializes the game board by creating a grid of tiles.
Handles user input, checking the guess against the solution.
Colors the tiles based on the correctness of each letter (green for correct, yellow for present but wrong position, grey for absent).
This implementation gives you a simple, playable version of Wordle. You can further enhance it by adding features such as a list of possible words for random solutions, better handling of duplicate letters, and more sophisticated UI elements.
Add any other context or screenshots about the feature request here.

@jennifereunice Thank you for providing your insight but it is not required as of now

why are you closing
On Sat, 15 Jun 2024 at 23:57, Swapnil Srivastava @.***>
wrote:

Closed #1779
https://github.com/swapnilsparsh/30DaysOfJavaScript/issues/1779 as not
planned.
—
Reply to this email directly, view it on GitHub
https://github.com/swapnilsparsh/30DaysOfJavaScript/issues/1779#event-13170824606,
or unsubscribe
https://github.com/notifications/unsubscribe-auth/AZEHKS2F6ALIJUJHNRVAIXLZHSBS7AVCNFSM6AAAAABJLTRLFCVHI2DSMVQWIX3LMV45UABCJFZXG5LFIV3GK3TUJZXXI2LGNFRWC5DJN5XDWMJTGE3TAOBSGQ3DANQ
.
You are receiving this because you were mentioned.Message ID:
<swapnilsparsh/30DaysOfJavaScript/issue/1779/issue_event/13170824606@
github.com>


Not able to test a Teams Meeting app while the app is sideloaded and not published

Hello! I'm trying to sideload a Teams meeting app and test it but when I upload the app package using the "Upload a custom app" option I don't see any of my meetings in the list and I can only select from chats/teams.
I can't add the app to a meeting the way described in this doc because the app is not published yet and it is sideloaded for testing. Is there a way to test an app for a meeting while it's sideloaded?
(I was able to test it for chats, groupchats, teams but not meetings. The option to select a meeting when I upload the app package doesn't seem to exist).
I'd appreciate any help here.

Document Details
⚠ Do not edit this section. It is required for docs.microsoft.com ➟ GitHub issue linking.

ID: b60eeacc-a71e-9644-b869-3e0a4f1506d8
Version Independent ID: 313bf83f-5325-f9f4-5780-9bc81e315c1e
Content: Create apps for teams meetings - Teams
Content Source: msteams-platform/apps-in-teams-meetings/create-apps-for-teams-meetings.md
Product: msteams
GitHub Login: @laujan
Microsoft Alias: lajanuar

Could you please try and follow these steps?

Schedule a new meeting with at least one user and send the invite.

Now edit the meeting

Click on Add New Tab

Select Manage Tab from bottom right

Click on Upload a custom app to upload your app manifest.

Let me know if this helps.

That worked! Thank you very much, Wajeed.


[REQUEST] Dict input and output of Pipeline Parallelism

Hi I found it confusing when handling multiple inputs of a layer in pipeline parallelism. The lack of flexbility make it difficult to use tuple input. Is there any plans to use dict input?
e.g.
class Layer1(nn.Module):
    ...
    def forward(self, input_ids, attention_mask):
        ...
        return {"hidden_states": h, "position_ids": pos_ids}

class Layer2(nn.Module):
    ...
    def forward(self, hidden_states, position_ids):
        ...
        return {"hidden_states": h}

Also use can set some parameters with default values in forward to use them alternatively.

maybe this helps, it's a PR that allows multi-variable inputs on pipeline's LayerSpec: https://github.com/microsoft/DeepSpeed/pull/4766


Remove accept condition because of auto accept transfer implemented

Auto accept transfer implemented and now transfee is not required to accept it first.

@cn0151
Can you review the proposed changes?
Important: When the changes are ready for publication, adding a #sign-off comment is the best way to signal that the PR is ready for the review team to merge.
#label:"aq-pr-triaged"
@MicrosoftDocs/public-repo-pr-review-team

#sign-off

@cn0151
Can you review this old PR and determine whether it needs to be closed or merged?

Thanks, @CN0151! This PR should close automatically after the changes are merged in the other PR. If it doesn't, comment #please-close.

Hi @cn0151 - Is there a new PR with these changes? Should this PR be closed?

@cn0151 - Could you link to the PR where you make the changes that will allow this PR to be closed?

I sent an email to @cn0151  today.
@MicrosoftDocs/public-repo-pr-review-team

Please close this PR

#please-close


Hot reload fails when using partial class with GeneratedRegex

Hot reload throws an error if GeneratedRegex is used.
To test, add the following class to any project and try using hot reload:
public partial class Test
{
    [System.Text.RegularExpressions.GeneratedRegex("-")]
    private static partial System.Text.RegularExpressions.Regex MyRegex();
}

You will get the following error:
error CS8795: Partial method 'Test.MyRegex()' must have an implementation part because it has accessibility modifiers.

Hi, simple mode can't handle source generators, for that to work you need to opt in for full mode: add -m Full to dotnet-maui-reactor command.
Another option is to put this function in a secondary assembly/project linked to the main project that will not be hot-reloaded.
Hope it helps!

Cool, thanks for the amazing library!


[FLINK-28112][filesystems] Fix error message when directly supported file system is not able to be handled

What is the purpose of the change
The error message is wrong when a directly supported filesystem is not able to be handled.
In this PR I've fixed it.
Brief change log

Split the error message to 2 parts, directly supported and not directly supported cases
Adapted the existing unit tests
Made the unit tests junit5 compliant

Verifying this change
Existing unit tests.
Does this pull request potentially affect one of the following parts:

Dependencies (does it add or upgrade a dependency): no
The public API, i.e., is any changed class annotated with @Public(Evolving): no
The serializers: no
The runtime per-record code paths (performance sensitive): no
Anything that affects deployment or recovery: JobManager (and its components), Checkpointing, Kubernetes/Yarn, ZooKeeper: no
The S3 file system connector: no

Documentation

Does this pull request introduce a new feature? no
If yes, how is the feature documented? not applicable

Thanks for the patch, LGTM +1 (non-binding)

cc @gyfora @mbalassi

@flinkbot run azure

@flinkbot run azure

it seems to be no problem overall, I only left some small suggestions.

@MartijnVisser I've added your suggestions, may I ask to have another look plz?


Update peserta-universitas.md

Incomplete.
Checklist:
:heavy_check_mark: Registration form
:heavy_check_mark: Project Euler Baby Steps
:heavy_check_mark: GitHub Pages with Project Euler username
:heavy_check_mark: 50 :star: GitHub repository contribution
:x: bahasa Indonesia Wikipedia user page
:heavy_check_mark: bahasa Indonesia Wikipedia content edit
:heavy_check_mark: StackOverflow account with at least one badge
:x: Besut Kode markdown Pull Request

Your Wikipedia Indonesia user page should mention "Besut Kode Universitas".
Your markdown edit:

the first column link text should be your GitHub username (Fizhu, not M.HafizhAnbiya);
the fourth column link text should be your Wikipedia Indonesia username (Hafizhanbiya, not M.HafizhAnbiya);
I just merged another PR, which caused a merge conflict with yours. Please resolve this issue by creating a new PR based on the current peserta-universitas.md.

Please submit a new pull request once more, after re-reading http://wikimedia-id.github.io/besutkode/university-modules-id.html and completing the tasks.
If you need help, ask on the Telegram chat group "Besut Kode Universitas".

Im sorry mr. Sage M.A , i have some question, i have received email from
mr. John vandenberg about incomplete terms of besut code, but the email
contains incomplete terms that i was complete it before, why can be like
that?
This is the screenshot of mr. John vandenberg message
Selasa, 18 Oktober 2016, Sage M.A<EMAIL_ADDRESS>menulis:

Incomplete.
Checklist:
✔ Registration form
✔ Project Euler Baby Steps
✔ GitHub Pages with Project Euler username
✔ 50 ⭐ GitHub repository contribution
❌ bahasa Indonesia Wikipedia user page
✔ bahasa Indonesia Wikipedia content edit
✔ StackOverflow account with at least one badge
❌ Besut Kode markdown Pull Request

Your Wikipedia Indonesia user page should mention "Besut Kode
Universitas".
Your markdown edit:

the first column link text should be your GitHub username (Fizhu,
not M.HafizhAnbiya);
the fourth column link text should be your Wikipedia Indonesia
username (Hafizhanbiya, not M.HafizhAnbiya);
I just merged another PR, which caused a merge conflict with
yours. Please resolve this issue by creating a new PR based on the current
peserta-universitas.md
https://github.com/BesutKode/BesutKode.github.io/blob/master/peserta-universitas.md.

Please submit a new pull request once more, after re-reading
http://wikimedia-id.github.io/besutkode/university-modules-id.html and
completing the tasks.
If you need help, ask on the Telegram chat group "Besut Kode Universitas
https://telegram.me/joinchat/D9IRcQqFMoIiieZe5YYb_w".
—
You are receiving this because you authored the thread.
Reply to this email directly, view it on GitHub
https://github.com/BesutKode/BesutKode.github.io/pull/172#issuecomment-254375985,
or mute the thread
https://github.com/notifications/unsubscribe-auth/AVet7IR34rTjuu6j9fJMcMyMTIDz1XNAks5q1BddgaJpZM4KY1JZ
.


[RESTEASY-2901] Clean up possibly-garbage Response before calling ExceptionHandler

(Same as #2795, but for main instead of 3.13)
Hello all,
First attempt at contributing to fix https://issues.redhat.com/browse/RESTEASY-2901
I'll wait for a feedback from someone more knowledgable of the code in question, but I'm starting to think a more adequate/elegant fix could be made around the definition of the entityStream used by ResteasyJackson2Provider, since the problem doesn't seem to happen with JSONB.
Cheers!

Thank you for the contribution. We are cleaning up older PR's that seem to be stale. If you feel this is still an issue please feel free to re-open.


Services and modules inheritance.

Hi guys, I wonder, if T3 supports any kind of services and/or modules inheritance?
For instance, I have message-list-service that renders messages list with its author nicknames. Now I want to create messages-list-service-with-photos, which additionally renders author's avatar. Is there a way to reuse message-list-service code or the aggregation and modification of generated DOM is the only way?

Modules are specifically not allowed to inherit from one another, however, you can use behaviors to have reusable code between modules.

Thank you for feedback.


ads support for query variables

This ads support for query variables.

Extracted type classes to separate files as only one class per file allowed
Added VariableType class and renderer
Added __variables field and builder
Added corresponding test

Nice work @terion-name , thanks! Will just run it up locally and add some docs then will push it out :)


Simplify code around ConfigMap creation

Just refactoring. No changes in behavior.
Signed-off-by: Tomáš Golembiovský<EMAIL_ADDRESS>

Looks good to me, nice simplification. Thanks @nyoxi


Index & Publish symbols with core 2.0

Troubleshooting
Checkout how to troubleshoot failures and collect debug logs: https://docs.microsoft.com/en-us/vsts/build-release/actions/troubleshooting
Environment

Server - TFS 2018

Agent - Private, Windows Server 2012R2, Agent 2.122.1

Issue Description

When using the Index Sources & Publish Symbols I am unable to index the pdb files from a dot net core 2.0 (targeting netstandard20). When it tries to index the source it states that it can't index the source because it's a portable library. When it executes symstore, symstore skips the pdb because it's an invalid file.
When checking enable verbose output the /o parameter is not putting out any additional information. The /o is missing from the symstore arguments. To figure out why symstore wasn't working I had to manually modify the powershell scripts on my build server.

Error logs
2017-12-29T23:59:49.8815868Z ##[debug]Evaluating condition for step: 'Publish symbols path: '
2017-12-29T23:59:49.8815868Z ##[debug]Evaluating: succeeded()
2017-12-29T23:59:49.8815868Z ##[debug]Evaluating succeeded:
2017-12-29T23:59:49.8815868Z ##[debug]=> (Boolean) True
2017-12-29T23:59:49.8815868Z ##[debug]Expanded: True
2017-12-29T23:59:49.8815868Z ##[debug]Result: True
2017-12-29T23:59:49.8815868Z ##[section]Starting: Publish symbols path:
2017-12-29T23:59:49.8815868Z ==============================================================================
2017-12-29T23:59:49.8815868Z Task         : Index Sources & Publish Symbols
2017-12-29T23:59:49.8815868Z Description  : Index your source code and publish symbols to a file share or Visual Studio Team Services Symbol Server
2017-12-29T23:59:49.8815868Z Version      : 2.0.3
2017-12-29T23:59:49.8815868Z Author       : Microsoft Corporation
2017-12-29T23:59:49.8815868Z Help         : See [more information](https://go.microsoft.com/fwlink/?LinkID=613722) on how to use this task. The Visual Studio Team Services Symbol Server is in private preview. See [these instructions](https://go.microsoft.com/fwlink/?linkid=846265) on how to participate in the private preview.
2017-12-29T23:59:49.8815868Z ==============================================================================
2017-12-29T23:59:51.0378442Z ##[debug]VstsTaskSdk 0.9.0 commit 6c48b16164b9a1c9548776ad2062dad5cd543352
2017-12-29T23:59:51.1940881Z ##[debug]Entering C:\agent\_work\_tasks\PublishSymbols_0675668a-7bba-4ccb-901d-5ad6554ca653\2.0.3\PublishSymbols.ps1.
2017-12-29T23:59:51.1940881Z ##[debug]Loading resource strings from: C:\agent\_work\_tasks\PublishSymbols_0675668a-7bba-4ccb-901d-5ad6554ca653\2.0.3\Task.json
2017-12-29T23:59:51.2409635Z ##[debug]Loaded 23 strings.
2017-12-29T23:59:51.2409635Z ##[debug]SYSTEM_CULTURE: 'en-US'
2017-12-29T23:59:51.2409635Z ##[debug]Loading resource strings from: C:\agent\_work\_tasks\PublishSymbols_0675668a-7bba-4ccb-901d-5ad6554ca653\2.0.3\Strings\resources.resjson\en-US\resources.resjson
2017-12-29T23:59:51.2878389Z ##[debug]Loaded 23 strings.
2017-12-29T23:59:51.2878389Z ##[debug]Loading module from path 'C:\agent\_work\_tasks\PublishSymbols_0675668a-7bba-4ccb-901d-5ad6554ca653\2.0.3\SymbolsCommon.psm1'.
2017-12-29T23:59:51.3203843Z ##[debug]Exporting function 'Get-PdbstrPath'.
2017-12-29T23:59:51.3203843Z ##[debug]Exporting function 'Get-DbghelpPath'.
2017-12-29T23:59:51.3203843Z ##[debug]Exporting function 'Get-SymStorePath'.
2017-12-29T23:59:51.3203843Z ##[debug]Importing function 'Get-DbghelpPath'.
2017-12-29T23:59:51.3203843Z ##[debug]Importing function 'Get-PdbstrPath'.
2017-12-29T23:59:51.3347290Z ##[debug]Importing function 'Get-SymStorePath'.
2017-12-29T23:59:51.3503394Z ##[debug]INPUT_SYMBOLSERVERTYPE: 'FileShare'
2017-12-29T23:59:51.3503394Z ##[debug]INPUT_SYMBOLSMAXIMUMWAITTIME (empty)
2017-12-29T23:59:51.3503394Z ##[debug] Defaulted to: '0'
2017-12-29T23:59:51.3503394Z ##[debug] Converted to int: 0
2017-12-29T23:59:51.3659643Z ##[debug]INPUT_DELETE (empty)
2017-12-29T23:59:51.3659643Z ##[debug] Converted to bool: False
2017-12-29T23:59:51.3659643Z ##[debug]INPUT_SYMBOLSPATH: '\\build\symbols'
2017-12-29T23:59:51.3659643Z ##[debug]INPUT_SOURCEFOLDER (empty)
2017-12-29T23:59:51.3659643Z ##[debug]BUILD_DEFINITIONNAME: 'Build1-ASP.NET Core-CI'
2017-12-29T23:59:51.3815923Z ##[debug]INPUT_SYMBOLSPRODUCT (empty)
2017-12-29T23:59:51.3815923Z ##[debug] Defaulted to: 'Build1-ASP.NET Core-CI'
2017-12-29T23:59:51.3972158Z ##[debug]BUILD_BUILDNUMBER: 'master-1.0.9'
2017-12-29T23:59:51.3972158Z ##[debug]INPUT_SYMBOLSVERSION (empty)
2017-12-29T23:59:51.3972158Z ##[debug] Defaulted to: 'master-1.0.9'
2017-12-29T23:59:51.3972158Z ##[debug]INPUT_SYMBOLSARTIFACTNAME: 'Symbols_release'
2017-12-29T23:59:51.3972158Z ##[debug]INPUT_INDEXSOURCES: 'true'
2017-12-29T23:59:51.3972158Z ##[debug] Converted to bool: True
2017-12-29T23:59:51.6784664Z ##[debug]INPUT_TREATNOTINDEXEDASWARNING: 'false'
2017-12-29T23:59:51.6784664Z ##[debug] Converted to bool: False
2017-12-29T23:59:51.6784664Z ##[debug]BUILD_SOURCESDIRECTORY: 'C:\agent\_work\1\s'
2017-12-29T23:59:51.6941070Z ##[debug]INPUT_SYMBOLSFOLDER: 'C:\agent\_work\1\s'
2017-12-29T23:59:51.7253417Z ##[debug]INPUT_SEARCHPATTERN: '**/bin/**/*.pdb'
2017-12-29T23:59:51.7253417Z ##[debug]Entering Find-VstsMatch.
2017-12-29T23:59:51.7253417Z ##[debug]DefaultRoot: 'C:\agent\_work\1\s'
2017-12-29T23:59:51.7409657Z ##[debug]FindOptions.FollowSpecifiedSymbolicLink: 'True'
2017-12-29T23:59:51.7409657Z ##[debug]FindOptions.FollowSymbolicLinks: 'True'
2017-12-29T23:59:51.7409657Z ##[debug]MatchOptions.Dot: 'True'
2017-12-29T23:59:51.7565917Z ##[debug]MatchOptions.FlipNegate: 'False'
2017-12-29T23:59:51.7565917Z ##[debug]MatchOptions.MatchBase: 'False'
2017-12-29T23:59:51.7565917Z ##[debug]MatchOptions.NoBrace: 'True'
2017-12-29T23:59:51.7565917Z ##[debug]MatchOptions.NoCase: 'True'
2017-12-29T23:59:51.7565917Z ##[debug]MatchOptions.NoComment: 'False'
2017-12-29T23:59:51.7565917Z ##[debug]MatchOptions.NoExt: 'False'
2017-12-29T23:59:51.7565917Z ##[debug]MatchOptions.NoGlobStar: 'False'
2017-12-29T23:59:51.7722161Z ##[debug]MatchOptions.NoNegate: 'False'
2017-12-29T23:59:51.7722161Z ##[debug]MatchOptions.NoNull: 'False'
2017-12-29T23:59:51.8503408Z ##[debug]Pattern: '**/bin/**/*.pdb'
2017-12-29T23:59:51.8503408Z ##[debug]FindPath: 'C:\agent\_work\1\s'
2017-12-29T23:59:51.8815909Z ##[debug]StatOnly: 'False'
2017-12-29T23:59:52.6316084Z ##[debug]Found 307 paths.
2017-12-29T23:59:52.6316084Z ##[debug]Applying include pattern.
2017-12-29T23:59:52.6472164Z ##[debug]AdjustedPattern: 'C:\agent\_work\1\s\**/bin/**/*.pdb'
2017-12-29T23:59:52.6628414Z ##[debug]1 matches
2017-12-29T23:59:52.6628414Z ##[debug]1 final results
2017-12-29T23:59:52.6628414Z ##[debug]Leaving Find-VstsMatch.
2017-12-29T23:59:52.6784663Z Found 1 files.
2017-12-29T23:59:52.6784663Z ##[debug]Loading module from path 'C:\agent\_work\_tasks\PublishSymbols_0675668a-7bba-4ccb-901d-5ad6554ca653\2.0.3\IndexHelpers\IndexHelpers.psm1'.
2017-12-29T23:59:52.6940920Z ##[debug]Adding dbghelp native wrappers.
2017-12-29T23:59:52.8972171Z ##[debug]Exporting function 'Invoke-IndexSources'.
2017-12-29T23:59:52.8972171Z ##[debug]Importing function 'Invoke-IndexSources'.
2017-12-29T23:59:52.9284673Z ##[debug]Entering Invoke-IndexSources.
2017-12-29T23:59:52.9284673Z ##[debug] TreatNotIndexedAsWarning: 'False'
2017-12-29T23:59:52.9597159Z ##[debug]AGENT_HOMEDIRECTORY: 'C:\agent'
2017-12-29T23:59:52.9597159Z ##[debug]Asserting leaf path exists: 'C:\agent\externals\pdbstr\pdbstr.exe'
2017-12-29T23:59:52.9597159Z ##[debug]Entering Add-DbghelpLibrary.
2017-12-29T23:59:52.9597159Z ##[debug]AGENT_HOMEDIRECTORY: 'C:\agent'
2017-12-29T23:59:52.9753410Z ##[debug]Asserting leaf path exists: 'C:\agent\externals\symstore\dbghelp.dll'
2017-12-29T23:59:53.0378500Z ##[debug]Entering Invoke-LoadLibrary.
2017-12-29T23:59:53.0378500Z ##[debug] LiteralPath: 'C:\agent\externals\symstore\dbghelp.dll'
2017-12-29T23:59:53.0378500Z ##[debug]Leaving Invoke-LoadLibrary.
2017-12-29T23:59:53.0378500Z ##[debug]Leaving Add-DbghelpLibrary.
2017-12-29T23:59:53.0378500Z ##[debug]Entering Get-SourceProvider.
2017-12-29T23:59:53.0378500Z ##[debug]BUILD_REPOSITORY_PROVIDER: 'TfsGit'
2017-12-29T23:59:53.0378500Z ##[debug]BUILD_SOURCESDIRECTORY: 'C:\agent\_work\1\s'
2017-12-29T23:59:53.0378500Z ##[debug]SYSTEM_TEAMPROJECTID: '4e4d2d39-8867-46cd-b24e-dfa598acc9fd'
2017-12-29T23:59:53.0378500Z ##[debug]SYSTEM_TEAMFOUNDATIONCOLLECTIONURI: 'http://tfs.test.local/DefaultCollection/'
2017-12-29T23:59:53.0378500Z ##[debug]BUILD_REPOSITORY_ID: '8d1bbbdd-aa48-41cc-b330-71dea94fbf1e'
2017-12-29T23:59:53.0378500Z ##[debug]BUILD_SOURCEVERSION: 'd70f760dca6a21ad806aee3cab6c9a2b0b016d10'
2017-12-29T23:59:53.0378500Z ##[debug]Leaving Get-SourceProvider.
2017-12-29T23:59:53.0534660Z ##[debug]Skipping: C:\agent\_work\1\s\CoreApp\CoreApp\bin\release\netcoreapp2.0\win10-x64\CoreApp.pdb because it is a Portable PDB
2017-12-29T23:59:53.0690921Z ##[debug]Entering Remove-DbghelpLibrary.
2017-12-29T23:59:53.0690921Z ##[debug] HModule: '140717475758080'
2017-12-29T23:59:53.0690921Z ##[debug]Leaving Remove-DbghelpLibrary.
2017-12-29T23:59:53.0690921Z ##[debug]Leaving Invoke-IndexSources.
2017-12-29T23:59:53.0690921Z ##[debug]INPUT_PUBLISHSYMBOLS: 'true'
2017-12-29T23:59:53.0690921Z ##[debug] Converted to bool: True
2017-12-29T23:59:53.0690921Z ##[debug]Loading module from path 'C:\agent\_work\_tasks\PublishSymbols_0675668a-7bba-4ccb-901d-5ad6554ca653\2.0.3\PublishHelpers\PublishHelpers.psm1'.
2017-12-29T23:59:53.1315913Z ##[debug]Exporting function 'Invoke-PublishSymbols'.
2017-12-29T23:59:53.1315913Z ##[debug]Exporting function 'Invoke-UnpublishSymbols'.
2017-12-29T23:59:53.1315913Z ##[debug]Importing function 'Invoke-PublishSymbols'.
2017-12-29T23:59:53.1315913Z ##[debug]Importing function 'Invoke-UnpublishSymbols'.
2017-12-29T23:59:53.1940933Z ##[debug]Entering Invoke-PublishSymbols.
2017-12-29T23:59:53.1940933Z ##[debug] PdbFiles: 'C:\agent\_work\1\s\CoreApp\CoreApp\bin\release\netcoreapp2.0\win10-x64\CoreApp.pdb'
2017-12-29T23:59:53.1940933Z ##[debug] Share: '\\build\symbols'
2017-12-29T23:59:53.1940933Z ##[debug] Product: 'Build1-ASP.NET Core-CI'
2017-12-29T23:59:53.1940933Z ##[debug] Version: 'master-1.0.9'
2017-12-29T23:59:53.1940933Z ##[debug] MaximumWaitTime: '02:00:00'
2017-12-29T23:59:53.1940933Z ##[debug] ArtifactName: 'Symbols_release'
2017-12-29T23:59:53.1940933Z ##[debug] SemaphoreMessage: 'Machine: BUILD, BuildUri: vstfs:///Build/Build/12, BuildNumber: master-1.0.9, RepositoryName: Build1, RepositoryUri: http://tfs.test.local/DefaultCollection/_git/Build1, Team Project: Build1, CollectionUri: http://tfs.test.local/DefaultCollection/ at 12/29/2017 23:59:53 UTC'
2017-12-29T23:59:53.1940933Z ##[debug]Entering New-ResponseFile.
2017-12-29T23:59:53.1940933Z ##[debug] PdbFiles: 'C:\agent\_work\1\s\CoreApp\CoreApp\bin\release\netcoreapp2.0\win10-x64\CoreApp.pdb'
2017-12-29T23:59:53.2253987Z ##[debug]Leaving New-ResponseFile.
2017-12-29T23:59:53.2253987Z ##[debug]Entering Get-ValidValue.
2017-12-29T23:59:53.2253987Z ##[debug] Current: '02:00:00'
2017-12-29T23:59:53.2253987Z ##[debug] Minimum: '00:01:00'
2017-12-29T23:59:53.2253987Z ##[debug] Maximum: '03:00:00'
2017-12-29T23:59:53.2253987Z ##[debug]Leaving Get-ValidValue.
2017-12-29T23:59:53.2253987Z ##[debug]Entering Lock-Semaphore.
2017-12-29T23:59:53.2253987Z ##[debug] Share: '\\build\symbols'
2017-12-29T23:59:53.2253987Z ##[debug] MaximumWaitTime: '02:00:00'
2017-12-29T23:59:53.2253987Z ##[debug] SemaphoreMessage: 'Machine: BUILD, BuildUri: vstfs:///Build/Build/12, BuildNumber: master-1.0.9, RepositoryName: Build1, RepositoryUri: http://tfs.test.local/DefaultCollection/_git/Build1, Team Project: Build1, CollectionUri: http://tfs.test.local/DefaultCollection/ at 12/29/2017 23:59:53 UTC'
2017-12-29T23:59:53.2565916Z ##[debug]Leaving Lock-Semaphore.
2017-12-29T23:59:53.2565916Z ##[debug]AGENT_HOMEDIRECTORY: 'C:\agent'
2017-12-29T23:59:53.2722165Z ##[debug]Asserting leaf path exists: 'C:\agent\externals\symstore\symstore.exe'
2017-12-29T23:59:53.2722165Z ##[debug]Entering Invoke-VstsTool.
2017-12-29T23:59:53.2722165Z ##[debug] FileName: 'C:\agent\externals\symstore\symstore.exe'
2017-12-29T23:59:53.2722165Z ##[debug] Arguments: 'add /f "@C:\agent\_work\_temp\tmp5546.tmp" /s "\\build\symbols" /t "Build1-ASP.NET Core-CI" /v "master-1.0.9" /o'
2017-12-29T23:59:53.2722165Z ##[debug] WorkingDirectory: 'C:\agent\_work\_temp\'
2017-12-29T23:59:53.2722165Z ##[command]"C:\agent\externals\symstore\symstore.exe" add /f "@C:\agent\_work\_temp\tmp5546.tmp" /s "\\build\symbols" /t "Build1-ASP.NET Core-CI" /v "master-1.0.9" /o
2017-12-29T23:59:53.3503525Z ##[debug]SYMSTORE MESSAGE: 0 alternate indexers registered
2017-12-29T23:59:53.3503525Z ##[debug]SYMSTORE MESSAGE: LastId.txt reported id 8
2017-12-29T23:59:53.3503525Z ##[debug]SYMSTORE MESSAGE: Final id is<PHONE_NUMBER>
2017-12-29T23:59:53.3503525Z ##[debug]SYMSTORE MESSAGE: Skipping C:\agent\_work\1\s\CoreApp\CoreApp\bin\release\netcoreapp2.0\win10-x64\CoreApp.pdb - unsupported format. ErrorLevel is 11.
2017-12-29T23:59:53.3503525Z ##[debug]
2017-12-29T23:59:53.3503525Z ##[debug]SYMSTORE: Number of files stored = 0
2017-12-29T23:59:53.3503525Z ##[debug]SYMSTORE: Number of errors = 0
2017-12-29T23:59:53.3503525Z ##[debug]SYMSTORE: Number of files ignored = 1
2017-12-29T23:59:53.3659660Z ##[debug]Exit code: 0
2017-12-29T23:59:53.3659660Z ##[debug]Leaving Invoke-VstsTool.
2017-12-29T23:59:53.3659660Z ##[debug]Entering Unlock-Semaphore.
2017-12-29T23:59:53.3659660Z ##[debug] Semaphore<EMAIL_ADDRESS>Path=\\build\symbols\_lockfile.sem}'
2017-12-29T23:59:53.3972162Z ##[debug]Leaving Unlock-Semaphore.
2017-12-29T23:59:53.4128433Z ##[debug]Associate artifact: Symbols_release with build: 12 at backend.
2017-12-29T23:59:53.4597162Z ##[debug]Processed: ##vso[artifact.associate TransactionId=0000000008;artifactname=Symbols_release;artifacttype=SymbolStore]\\build\symbols
2017-12-29T23:59:53.4597162Z ##[debug]Leaving Invoke-PublishSymbols.
2017-12-29T23:59:53.4597162Z ##[debug]Leaving C:\agent\_work\_tasks\PublishSymbols_0675668a-7bba-4ccb-901d-5ad6554ca653\2.0.3\PublishSymbols.ps1.
2017-12-29T23:59:53.4597162Z ##[section]Async Command Start: Associate Artifact
2017-12-29T23:59:53.5065975Z Associated artifact 11 with build 12
2017-12-29T23:59:53.5065975Z ##[section]Async Command End: Associate Artifact
2017-12-29T23:59:53.5065975Z ##[section]Finishing: Publish symbols path:

For #1 (it can't index the source because it's a portable library), you can ignore this message.  For Portable PDBs, one should source index the PDBs during the build via SourceLink.
@tmat what's the symstore equivalent for indexing portable PDBs to a share?

I should also mention that the current version of Visual Studio also [doesn't support authenticated source repositories] (https://visualstudio.uservoice.com/forums/121579-visual-studio-ide/suggestions/19107784-debugger-should-support-authentication-with-source). @tmat should also be able to give an update there.

We have a new version of the Index and Publish Symbols task in the repository right now which is rolling on to VSTS. At some point in the future this updated task will get picked up and shipped into a TFS update. This will be more graceful when it arrives. You still need to use SourceLink as @jerickmsft has indicated.

@veccsolutions Windows PDBs (generated by /debug:full compiler option) only work on Windows. While your workaround makes it possible for debuggers on Windows to find the PDBs on the Symbol Server you won't be able to debug your binaries on non-Windows platforms.
To make both x-plat debugging and older tools that do not support Portable PDBs yet (older VSTS, WinDGB, etc) work you can do the following:

Build your projects with /debug:portable (the default setting)
Include Portable PDBs produced by your build in NuGet packages next to the corresponding binaries (if you distribute your components via NuGet) or just upload them to a file server which you can then point x-plat debugger to.
Use Pdb2Pdb command line tool to convert Portable PDBs to Windows PDBs and publish those to VSTS symbol server using Index and Publish Symbols task.

This is an approach we use for our own products such as Roslyn compilers and various Visual Studio components.

@tmat what is the plan for working with .netstandard libs without step 3 - we are currently including pdbs in the main nuget package with embedded source
This makes the consumer side super simple - would like to also have the middle of the road option with respect to package size

Thanks @tmat - with respect to Source Linking with NuGet Packages I'm trying to piece together where we are at today
My understanding is that scenario 2 is fully lit up ... trying to get a handle on what remains to light up scenario 1

SourceLink to embed a reference in the PDB to a remote source to get the code. This is the recommended way and will be built into the tooling by default.

Is there SourceLink support to index .netstandard lib during build if the backing repo is on VSTS
Utilize Index & Publish symbols task to publish SourceLinked PDB to VSTS hosted symbol server (indexing can be turned off because it's done during the build)
VS 2017 will support SourceLink auth with private repos soon

PDB embedded sources, which provides the source code directly in the PDB

Possible today via SourceLink.Embed.AllSourceFiles
dotnet pack does include the PDB embedded sources into the main nupkg
VS 2017 does extract the embedded source and support nupkg debug

Fully embedded PDBs, providing all debugging information and source code in the .dll

A variation on 2 - how is this accomplished?

Including the source files in the .nupkg directly. This method is no longer recommended and will not be supported in the tools.

Methods 1-3 obviate the need to use --include-symbols --include-source?

Hello,
I'm not sure to understand all the nuget mechanisms. I have an on-premise TFS with nuget feeds and a shared folder to store PDB. My build definitions use "Index Sources & Publish Symbols" V2.
Case 1: Library in .Net Framework, I'm able to debug it (VS download PDB which seems to include source files)
Case 2: Library in .Net Standard 2.0, I tried a lot of things but VS does not find the source files.

The shared folder is updated with new folders & pdb files.
The build configuration is set to "debug"
Debugging Information is set to "Full"

Is it possible to debug .Net Standard nuget package with a shared folder to download PDB ?
I spent a lot a days (weeks) on this issue that prevent me to migrate .Net Framework packages to .Net Standard.
Thanks & regards,

I am trying to publish PDB files to a on-premise fileshare (using TFS2018Update3) and am getting "Skipping: .....pdb because it is a Portable PDB".
What are the plans for SymStore.exe to support handling of Portable PDB files when publishing to a fileshare?

Please use the new version of Index and Publish Symbols. Thanks.

I am using the current version of Index and Publish Symbols. Still does not work for standard. Does anyone have an update on this?


location of config yam files

fir this one its better to have config files in
~/.cloudmesh/management

Will look into this one shortly...

currently:
cp etc/accounts/cloudmesh_user.yaml ~/.cloudmesh/cloudmesh_user.yaml
cp etc/accounts/cloudmesh_project.yaml ~/.cloudmesh/cloudmesh_project.yaml
python setup.py install
this should be in ~/.cloudmesh/management
There should also not be a reason to copy these files by hand, as this should be auto generated.
however it needs to be documented how to modify them, so maybe we need
python setup.py yaml
cd ...
edit this and that, make mods here an there
python setup.py install
However if you are satisfied with the current objects you do not have to modify anything and you should just use
python setup.py install

I just saw that you correct used etc/...../accounts but you do not push that dir out into .cloudmesh

Added a small section about the structure of a field description in the yaml file.
Amended the SetupYaml class in setup.py to copy the files from etc/accounts to "~/.cloudmesh/accounts"
Added a ResetYaml class in setup.py to use when the yaml files needs to be replaced
Added a ReGenFile class in setup.py to regenerate the python files base_user.py and base_project.py under cloudmesh_management directory.


wascc-host v0.12.0

Closes https://github.com/wascc/aws-lambda-wascc-runtime/issues/54.

0.12.0 is now in crates.io

Replace with upgrade to v0.14.0.


Cannot switch to loudspeaker (bottom speaker) when running app on iPhone

If headphones are not connected, audio doesn't play through loudspeaker.  It would be wonderful if there is a button to switch to loudspeaker. Or it probably make sense to keep it permanently to loudspeaker whenever headphones are not connected.
Tested In:
iPhone 5S, IOS v9.3.2.
Kurento IOS sdk v0.6.0 and latest source on 9-Dec-2016

you have to wait until you get the remote stream, then you can switch to loudspeaker


change /collection/ to /object/

we had a conversation while you were gone with kate, fernando. fernando thinks it's going to be a pain to maintain handles with /collection/, and urls aren't too important for kate et al

/object route can now handle requests for collections
Redirecting object requests for collection type objects to the collection view controller


How to sort table when using offset and limit ( without any caluse ( where , or etc )

Hi,
I have the following situation.
I have a very large table ( called vehicles )
Now, i can get the vehicles like:
db.vehicles.offset(0).limit(100)  ( and increasing the offset when needed for pagination)
But i want this to be sorted somehow
by using:
db.vehicles.orderBy(column_name).offset(0).limit(100) i get somehow the result i want except that some records are skiped ( as it is stated in documentation)
So, my question is, is there a way to sort a very very large table by the given index and get the result paginated ?
I need to mention that the "vehicle" table contains more than 20.000 records
So far i have only managed to get all data and sort, then slice on the UI ( or where it is needed ) but this is far form optimal
Thank you very much fo any advice !
Br, Sebastian

I'd suggest to fill the gaps in the index so that it covers every record. For example, replace undefined or null with empty string (or other indexable value, such as 0 or an empty array).
db.version(2).upgrade(tx => {
  return tx.vehicles.modify(vh => {
    if (vh.column_name == null) { // checks for null or undefined
      vh.column_name = -Infinity; // Or "" etc.
    }
});

Then you would be able to paginate using the index.

Thank you very much for this information, i will give it a try !


HOUDINI: Intro: Tutorial on Composite Scrolling and Animation (8)

Composite Scrolling and Animation
Create the learn section for Composite Scrolling and Animation (this is an API landing page. Rather, it's in the learn section)
Page 8 of learning section for houdini
Intro (page 1)
CSS Parser API (page 2)
CSS Properties and Values API (page 3)
CSS Typed OM (page 4)
CSS Layout API (page 5)
CSS Paint API (page 6)
Worklets (page 7)
Composite Scrolling and Animation (page 8)
Summary/Conclusion/who knows (page 9)

I'd be a bit wary about putting Houdinni stuff in the Learning Area of MDN. Sure, you are writing tutorials about it, but it's not really a beginner's topic.