Message always sent to 0th partition, if @RetryableTopic has autoCreateTopics="false" and no partition info

When using the autoCreateTopics="false" and not providing partition info in the @RetryableTopic, for an existing Kafka topic, the retry message always gets sent to the 0th partition, even though the existing topic has multiple partitions.
Refer to the test case -
https://github.com/spring-projects/spring-kafka/blob/ff0634e4bde66bf0e216e88acfd878fb3f3d031f/spring-kafka/src/test/java/org/springframework/kafka/retrytopic/ExistingRetryTopicIntegrationTests.java
/*
 * Copyright 2021 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.springframework.kafka.retrytopic;

import org.apache.kafka.clients.admin.AdminClientConfig;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.TaskExecutor;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.annotation.RetryableTopic;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.*;
import org.springframework.kafka.listener.ContainerProperties;
import org.springframework.kafka.support.KafkaHeaders;
import org.springframework.kafka.test.EmbeddedKafkaBroker;
import org.springframework.kafka.test.context.EmbeddedKafka;
import org.springframework.messaging.handler.annotation.Header;
import org.springframework.retry.annotation.Backoff;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;

import java.time.Clock;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.fail;

@SpringJUnitConfig
@DirtiesContext
@EmbeddedKafka(topics = {ExistingRetryTopicIntegrationTests.MAIN_TOPIC_WITH_NO_PARTITION_INFO,
		ExistingRetryTopicIntegrationTests.RETRY_TOPIC_WITH_NO_PARTITION_INFO,
		ExistingRetryTopicIntegrationTests.MAIN_TOPIC_WITH_PARTITION_INFO,
		ExistingRetryTopicIntegrationTests.RETRY_TOPIC_WITH_PARTITION_INFO}, partitions = 4)
@TestPropertySource(properties = "two.attempts=2")
public class ExistingRetryTopicIntegrationTests {

	private static final Logger logger = LoggerFactory.getLogger(ExistingRetryTopicIntegrationTests.class);

	public final static String MAIN_TOPIC_WITH_NO_PARTITION_INFO = "main-topic-1";

	public final static String RETRY_TOPIC_WITH_NO_PARTITION_INFO = "main-topic-1-retry-0";

	public final static String MAIN_TOPIC_WITH_PARTITION_INFO = "main-topic-2";

	public final static String RETRY_TOPIC_WITH_PARTITION_INFO = "main-topic-2-retry-0";

	private final static String MAIN_TOPIC_CONTAINER_FACTORY = "kafkaListenerContainerFactory";

	@Autowired
	private KafkaTemplate<String, String> kafkaTemplate;

	@Autowired
	CountByPartitionContainer countByPartitionContainerWithoutPartition;

	@Autowired
	CountByPartitionContainer countByPartitionContainerWithPartition;

	@Autowired
	private CountDownLatchContainer latchContainer;

	@Test
	@DisplayName("When a @RetryableTopic listener method, with autoCreateTopic=false and NO PARTITION info called, " +
			"should send messages to be retried across partitions for a retry topic")
	void whenNoPartitionInfoProvided_shouldRetryMainTopicCoveringAllPartitionOfRetryTopic() {

		send10MessagesToPartitionWithKey(0, "foo", MAIN_TOPIC_WITH_NO_PARTITION_INFO, this.countByPartitionContainerWithoutPartition);
		send10MessagesToPartitionWithKey(1, "bar", MAIN_TOPIC_WITH_NO_PARTITION_INFO, this.countByPartitionContainerWithoutPartition);
		send10MessagesToPartitionWithKey(2, "buzz", MAIN_TOPIC_WITH_NO_PARTITION_INFO, this.countByPartitionContainerWithoutPartition);
		send10MessagesToPartitionWithKey(3, "fizz", MAIN_TOPIC_WITH_NO_PARTITION_INFO, this.countByPartitionContainerWithoutPartition);

		assertThat(awaitLatch(latchContainer.countDownLatch1)).isTrue();
		assertThat(countByPartitionContainerWithoutPartition.mainTopicMessageCountByPartition)
				.isEqualTo(countByPartitionContainerWithoutPartition.retryTopicMessageCountByPartition);
	}

	@Test
	@DisplayName("When a @RetryableTopic listener method, with autoCreateTopic=false and WITH PARTITION info called, " +
			"should send messages to be retried across partitions for a retry topic")
	void whenPartitionInfoProvided_shouldRetryMainTopicCoveringAllPartitionOfRetryTopic() {

		send10MessagesToPartitionWithKey(0, "foo", MAIN_TOPIC_WITH_PARTITION_INFO, this.countByPartitionContainerWithPartition);
		send10MessagesToPartitionWithKey(1, "bar", MAIN_TOPIC_WITH_PARTITION_INFO, this.countByPartitionContainerWithPartition);
		send10MessagesToPartitionWithKey(2, "buzz", MAIN_TOPIC_WITH_PARTITION_INFO, this.countByPartitionContainerWithPartition);
		send10MessagesToPartitionWithKey(3, "fizz", MAIN_TOPIC_WITH_PARTITION_INFO, this.countByPartitionContainerWithPartition);

		assertThat(awaitLatch(latchContainer.countDownLatch2)).isTrue();
		assertThat(countByPartitionContainerWithPartition.mainTopicMessageCountByPartition)
				.isEqualTo(countByPartitionContainerWithPartition.retryTopicMessageCountByPartition);
	}

	private void send10MessagesToPartitionWithKey(int partition, String messageKey, String mainTopic, CountByPartitionContainer countByPartitionContainer) {
		IntStream.range(0, 10).forEach(messageNumber -> {
			String data = "Test-partition-" + partition + "-messages-" + messageNumber;
			logger.debug("Sending message number:{} to topic {}. Message: {}", messageNumber, mainTopic, data);
			kafkaTemplate.send(mainTopic, partition, messageKey, data);
			countByPartitionContainer.mainTopicMessageCountByPartition.merge(String.valueOf(partition), 1, Integer::sum);
		});
	}

	private boolean awaitLatch(CountDownLatch latch) {
		try {
			return latch.await(60, TimeUnit.SECONDS);
		} catch (Exception e) {
			fail(e.getMessage());
			throw new RuntimeException(e);
		}
	}

	static class MainTopicListener {

		@Autowired
		CountDownLatchContainer container;

		@Autowired
		CountByPartitionContainer countByPartitionContainerWithoutPartition;

		@RetryableTopic(autoCreateTopics = "false", dltStrategy = DltStrategy.NO_DLT,
				attempts = "${two.attempts}", backoff = @Backoff(0), kafkaTemplate = "kafkaTemplate")
		@KafkaListener(id = "firstTopicId", topics = MAIN_TOPIC_WITH_NO_PARTITION_INFO, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY)
		public void listenFirst(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic,
								@Header(KafkaHeaders.ORIGINAL_PARTITION) String originalPartition,
								@Header(KafkaHeaders.RECEIVED_PARTITION_ID) String receivedPartition) {
			logger.debug("Message {} received in topic {}. originalPartition: {}, receivedPartition: {}",
					message, receivedTopic, originalPartition, receivedPartition);

			if (receivedTopic.contains("-retry")) {
				countByPartitionContainerWithoutPartition.retryTopicMessageCountByPartition.merge(receivedPartition, 1, Integer::sum);
				container.countDownLatch1.countDown();
			}

			throw new RuntimeException("Woooops... in topic " + receivedTopic);
		}

		@Autowired
		CountByPartitionContainer countByPartitionContainerWithPartition;

		@RetryableTopic(autoCreateTopics = "false", numPartitions = "4", dltStrategy = DltStrategy.NO_DLT,
				attempts = "${two.attempts}", backoff = @Backoff(0), kafkaTemplate = "kafkaTemplate")
		@KafkaListener(id = "secondTopicId", topics = MAIN_TOPIC_WITH_PARTITION_INFO, containerFactory = MAIN_TOPIC_CONTAINER_FACTORY)
		public void listenSecond(String message, @Header(KafkaHeaders.RECEIVED_TOPIC) String receivedTopic,
								 @Header(KafkaHeaders.ORIGINAL_PARTITION) String originalPartition,
								 @Header(KafkaHeaders.RECEIVED_PARTITION_ID) String receivedPartition) {
			logger.debug("Message {} received in topic {}. originalPartition: {}, receivedPartition: {}",
					message, receivedTopic, originalPartition, receivedPartition);

			if (receivedTopic.contains("-retry")) {
				countByPartitionContainerWithPartition.retryTopicMessageCountByPartition.merge(receivedPartition, 1, Integer::sum);
				container.countDownLatch2.countDown();
			}

			throw new RuntimeException("Woooops... in topic " + receivedTopic);
		}
	}

	static class CountDownLatchContainer {

		CountDownLatch countDownLatch1 = new CountDownLatch(40);
		CountDownLatch countDownLatch2 = new CountDownLatch(40);
	}

	static class CountByPartitionContainer {

		Map<String, Integer> mainTopicMessageCountByPartition = new HashMap<>();
		Map<String, Integer> retryTopicMessageCountByPartition = new HashMap<>();
	}

	@Configuration
	static class RetryTopicConfigurations {

		@Bean
		public MainTopicListener mainTopicListener() {
			return new MainTopicListener();
		}

		@Bean
		CountDownLatchContainer latchContainer() {
			return new CountDownLatchContainer();
		}

		@Bean
		CountByPartitionContainer countByPartitionContainerWithoutPartition() {
			return new CountByPartitionContainer();
		}

		@Bean
		CountByPartitionContainer countByPartitionContainerWithPartition() {
			return new CountByPartitionContainer();
		}
	}

	@Configuration
	public static class RuntimeConfig {

		@Bean(name = RetryTopicInternalBeanNames.INTERNAL_BACKOFF_CLOCK_BEAN_NAME)
		public Clock clock() {
			return Clock.systemUTC();
		}

		@Bean
		public TaskExecutor taskExecutor() {
			return new ThreadPoolTaskExecutor();
		}

		@Bean(destroyMethod = "destroy")
		public TaskExecutorManager taskExecutorManager(ThreadPoolTaskExecutor taskExecutor) {
			return new TaskExecutorManager(taskExecutor);
		}
	}

	static class TaskExecutorManager {
		private final ThreadPoolTaskExecutor taskExecutor;

		TaskExecutorManager(ThreadPoolTaskExecutor taskExecutor) {
			this.taskExecutor = taskExecutor;
		}

		void destroy() {
			this.taskExecutor.shutdown();
		}
	}

	@Configuration
	public static class KafkaProducerConfig {

		@Autowired
		EmbeddedKafkaBroker broker;

		@Bean
		public ProducerFactory<String, String> producerFactory() {
			Map<String, Object> configProps = new HashMap<>();
			configProps.put(
					ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,
					this.broker.getBrokersAsString());
			configProps.put(
					ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
					StringSerializer.class);
			configProps.put(
					ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
					StringSerializer.class);
			return new DefaultKafkaProducerFactory<>(configProps);
		}

		@Bean
		public KafkaTemplate<String, String> kafkaTemplate() {
			return new KafkaTemplate<>(producerFactory());
		}
	}

	@EnableKafka
	@Configuration
	public static class KafkaConsumerConfig {

		@Autowired
		EmbeddedKafkaBroker broker;

		@Bean
		public KafkaAdmin kafkaAdmin() {
			Map<String, Object> configs = new HashMap<>();
			configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, this.broker.getBrokersAsString());
			return new KafkaAdmin(configs);
		}

		@Bean
		public ConsumerFactory<String, String> consumerFactory() {
			Map<String, Object> props = new HashMap<>();
			props.put(
					ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
					this.broker.getBrokersAsString());
			props.put(
					ConsumerConfig.GROUP_ID_CONFIG,
					"groupId");
			props.put(
					ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
					StringDeserializer.class);
			props.put(
					ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
					StringDeserializer.class);
			props.put(
					ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, false);
			props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

			return new DefaultKafkaConsumerFactory<>(props);
		}

		@Bean
		public ConcurrentKafkaListenerContainerFactory<String, String> retryTopicListenerContainerFactory(
				ConsumerFactory<String, String> consumerFactory) {

			ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
			ContainerProperties props = factory.getContainerProperties();
			props.setIdleEventInterval(100L);
			props.setPollTimeout(50L);
			props.setIdlePartitionEventInterval(100L);
			factory.setConsumerFactory(consumerFactory);
			factory.setConcurrency(1);
			factory.setContainerCustomizer(
					container -> container.getContainerProperties().setIdlePartitionEventInterval(100L));
			return factory;
		}

		@Bean
		public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory(
				ConsumerFactory<String, String> consumerFactory) {

			ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
			factory.setConsumerFactory(consumerFactory);
			factory.setConcurrency(1);
			return factory;
		}
	}

}

Hi @deepesh-verma, thanks a lot for using this feature and providing feedback. You're right, that's the current behavior, it uses the provided number of partitions to determine to which partition it should send the message to. In case the user doesn't provide this info, maybe we could:

Set the partition to -1 to allow the Producer itself to assign a partition to the record
Set the partition to the same partition as the original record - DeadLetterPublishingRecoverer has logic to check whether that partition exists, and if it doesn't it sets -1 as above.

Let's see what @garyrussell thinks about this.
Also, if you want to take a look at the logic and maybe prepare a PR, it's in the DeadLetterPublishingRecovererFactory class line 119 for determining the retry topic partition, and DeadLetterPublishingRecoverer class line 372 (checkPartition method) to verify whether or not the partition exists in that topic.
Thanks again!

Another thing is maybe we could extract the new TopicPartition(..) part in the resolveDestination method in DLPRF to a new protected method, so that users can subclass and override this behavior if needed.

Thanks @tomazfernandes for the reply and explaination. I would love to work on this issue.
I am start with below implementation and will raise a PR :

Set the partition to the same partition as the original record - DeadLetterPublishingRecoverer has logic to check whether that partition exists, and if it doesn't it sets -1
extract the new TopicPartition(..) part in the resolveDestination method in DLPRF to a new protected method

I will also keep an eye on what @garyrussell has to say for this, and incorporate that as well in the PR. Thanks!

I concur with the proposal. Thanks!


TagBot trigger issue

This issue is used to trigger TagBot; feel free to unsubscribe.
If you haven't already, you should update your TagBot.yml to include issue comment triggers.
Please see this post on Discourse for instructions and more details.
If you'd like for me to do this for you, comment TagBot fix on this issue.
I'll open a PR within a few hours, please be patient!

Triggering TagBot for merged registry pull request: https://github.com/JuliaRegistries/General/pull/77373

Triggering TagBot for merged registry pull request
This extra notification is being sent because I expected a tag to exist by now, but it doesn't.
You may want to check your TagBot configuration to ensure that it's running, and if it is, check the logs to make sure that there are no errors.

Triggering TagBot for merged registry pull request
This extra notification is being sent because I expected a tag to exist by now, but it doesn't.
You may want to check your TagBot configuration to ensure that it's running, and if it is, check the logs to make sure that there are no errors.

Triggering TagBot for merged registry pull request: https://github.com/JuliaRegistries/General/pull/78572

Triggering TagBot for merged registry pull request: https://github.com/JuliaRegistries/General/pull/78837

Triggering TagBot for merged registry pull request: https://github.com/JuliaRegistries/General/pull/79407


[CI] Netlify deploy

Trigger Netlify to deploy every time a PR is issued

@warunicorn19 I guess we solved this issue right ?

Yup


customInput lost the focus after entering one or max 2 characters

When a custom input passed to datepicker using customInput attribute then the input tag looses its focus after entering one or two characters. Need to focus in the input multiple times in order to enter the complete date
Steps to reproduce the behavior:
() => {
const [startDate, setStartDate] = useState(new Date());
const ExampleCustomInput = (props) => (
<input {...props}/>
);
return (
<DatePicker
selected={startDate}
onChange={date => setStartDate(date)}
customInput={}
/>
);
};

Click in the input box
Try entering the date values using number keys
Input looses the focus

Expected behavior
Input box should allow to enter complete date value using keyboard number keys

Hi,
Can you try making following prop to true
strictParsing={true}
i tried its working

Hi,
Can you try making following prop to true
strictParsing={true}
i tried its working

This "fixes" the issue in as much as the user is able to type dates, but it removes functionality (for example, date guessing doesn't work, it just waits until you've completed  date to do anything).
Does anyone know of a way to get the same functionality available with the default input, but using a custom one?

I had this problem in my code and after some investigation it looks like it works like this because of unique combination of React.cloneElement and ref in date picker code for custom inputs which leads to all sorts of unexpected behavior.
So looks like for now there are only two viable alternatives:

Use custom input for non-input elements only (like a button in custom input example)
Or don't use custom input if you need manual input to work properly - if you need custom input styling (like me) you could use combination of CSS and className / wrapperClassName props to style it as you wish

Hope this helps the next person who comes here with this problem

The bug seems to have to do with "autoFocus". I found a work-around that supports date guessing!

Create a state variable hasFocus
onFocus set to true
onBlur set to false
add attribute autoFocus={hasFocus} to customInput

Full example:
import React, {useRef, useState, forwardRef} from "react";

import DatePicker from "react-datepicker";
import "react-datepicker/dist/react-datepicker.css";

export default ({onFocus, onBlur, ...props}) => {
	//Ref needed for custom input
	const refCustomInput = useRef();

	//State hooks
	const [stateHasFocus, setStateHasFocus] = useState(false); //Initial value

	//Custom input
	const CustomInput = forwardRef((props, ref) => {
		const {autoFocus, ...remaining} = props;

		return <input type="text" ref={ref} autoFocus={stateHasFocus} {...remaining} />;
	});

	return (
		<DatePicker
			customInput={<CustomInput ref={refCustomInput} />}
			onFocus={evt => {
				setStateHasFocus(true);

				//Notify parent
				if (onFocus) {
					onFocus.call(this, evt);
				}
			}}
			onBlur={evt => {
				setStateHasFocus(false);

				//Notify parent
				if (onBlur) {
					onBlur.call(this, evt);
				}
			}}
			{...props}
		/>
	);
};

Sorry, I know the issue is already closed but I thought this could help someone else, still.
I ran into this same problem and went with koga73's solution above which worked nicely for me also. (Thanks!)
However, based on a colleague's review comment I moved the custom input component outside of the exported date picker component and that seemed to remove the problem altogether. So, using the above solution as a basis, this is what I tried:
import React, { useRef, forwardRef } from 'react';

import ReactDatePicker from 'react-datepicker';
import 'react-datepicker/dist/react-datepicker.css';

//Custom input
const CustomInput = forwardRef((props, ref) => {
    return <input type="text" ref={ref} {...props} />;
});

export default ({ onFocus, onBlur, ...props }) => {
    //Ref needed for custom input
    const refCustomInput = useRef();

    return <ReactDatePicker customInput={<CustomInput ref={refCustomInput} />} {...props} />;
};

It seemed like I had no problems with the focus "escaping" outside of the input component any more.
I know the "custom input" example on the site has the custom input component inside of the functional component but I'm not sure if that's a requirement. (I believe creating components there can actually cause problems with the state in more complex components...)

https://codesandbox.io/s/react-datepicker-custominput-sample-forked-pc39wb?file=/src/App.tsx

@mjviljan  Thank you so much for the solution.


Flaxseed Oil / Wood Stain / Wood Bleach craftable without buckets

Would it be possible to  be able to craft the bleach and dye without a bucket (i.e. using bottles instead of buckets)?  I have a mod in my pack that has wooden buckets in it, and I obviously can't create flaxseed oil in a wooden bucket, so I then can't create bleach or dye until I have a vanilla iron bucket, which is a little later in game for me, than with most packs.

Hello,
Thank you for the suggestion. The short answer is "yes, but the current recipes are placeholders", and the long answer is as follows:
I could have implemented it that way but there would be some problems with adding Flaxseed Oil bottle-consuming recipes. The inbuilt crafting recipes will always leave container items behind for crafting recipes that require them. This means that adding a crafting recipe using the Flaxseed oil bottle as an ingredient will leave behind a bottle, even if it crafts into something else that is contained by a bottle (the Wood Stain and Wood Bleach). There isn't an inbuilt "consume container item" mechanism, to my knowledge. Thus, the workarounds are:

Add empty bottles as an extra ingredient to the recipe(s) to avoid glass bottle duplication
Write a custom IRecipe serializer or some kind of crafting function
Just use Buckets as the input
Implement some kind of special crafting device with its own recipes and such.

I chose to combine the first and third options, deciding that this would be an avenue for future improvement. My apologies for any inconvenience.
My suggestion for the meantime would be to add a recipe that better respects your modpack's balance requirements - at least until I add a proper Flax Oil fluid in one of the upcoming versions. Unfortunately I couldn't foresee the balance requirements of every modpack that might want to include my mod, so I can only really balance against the base game where buckets are pretty easy to come by.
Addendum:
I would have preferred to implement Wood Stain and Wood Bleach as a brewing stand recipe, brewing mushrooms into the Flaxseed Oil bottles directly, but the Brewing Stand has an inbuilt limitation that the bottle slots only accept items with a maximum stack size of one. The workarounds here were to either:

Modify the Brewing Stand with a mixin, or
Make the Flaxseed Oil unstackable

I recognized that the latter solutions would probably annoy players, but the former solution would annoy me, so that idea is temporarily suspended for now.


feat(getObject): support useSignedUrl flow

The adapter should be able to handle useSignedUrl on getObject, as described in https://github.com/hyper63/hyper/issues/519

This adapter is deprecated and will not be receiving any updates. Closing any outstanding issues.


Regular Menu Icons use the wrong color when using dark theme macOS

You're right. The issue is Mac exclusive. Activate the key svg.context-properties.content.enabled


Use nightly docker container from docker hub

Instead of downloading manually using rustup, it is better to use the officially supported container.

My only concern is that this image doesn't let us lock to a specific Rust version; that caused a bunch of issues when I first started to revive Hackagotchi

I'm not sure quite what you mean.. when you download nightly using rustup, doesn't it download the latest version?


[feature](schema change) unified schema change for parquet and orc reader

Proposed changes
Following https://github.com/apache/doris/pull/25138, unified schema change interface for parquet and orc reader, and can be applied to other format readers as well.
Supported Type Changes
More type changes are supported:

ColumnTypeConverter
Unified schema change interface for all format readers:

First, read the data according to the column type of the file into source column;
Second, convert source column to the destination column with type planned by FE

PhysicalToLogicalConverter
Convert parquet physical column to logical column
In parquet document(https://github.com/apache/parquet-format/blob/master/LogicalTypes.md),
Logical or converted type is the data type of column, physical type is the stored type of column chunk. eg, decimal type can be stored as INT32, INT64, BYTE_ARRAY, FIXED_LENGTH_BYTE_ARRAY, so there is a convert process from physical type to logical type. In addition, schema change will bring about a change in logical type.
In previous implementations, physical and logical conversion were mixed together, resulting in severe code complexity and bloating.
PhysicalToLogicalConverter strips away the conversion of logical type, and reuse ColumnTypeConverter to resolve schema change, allowing parquet reader to only focus on the conversion of physical types.
Therefore, tow layers converters are designed:

First, read parquet data with the physical type
Second, convert physical type to logical type
Third, convert logical type to the final type planned by FE(schema change)

Ultimate performance optimization:

If process of (First => Second) is consistent, eg. from BYTE_ARRAY to string, no additional copies and conversions will be introduced;
If process of (Second => Third) is consistent, eg. from decimal(12, 4) to decimal(8, 2), no additional copies and conversions will be introduced;
Null map is share among all processes, no additional copies and conversions will be introduced in null map;
Only create one physical column in physical conversion, and reused in each loop;
Only create one logical column in logical conversion, and reused in each loop;
FIXED_LENGTH_BYTE_ARRAY is read as ColumnUInt8 instead of ColumnString, so the underlying decoder has no process to decode string and use memory copy to read the data as a whole, and the conversion has no need to resolve the Offsets in ColumnString.

Further comments
If this is a relatively large or complex change, kick off the discussion at<EMAIL_ADDRESS>by explaining why you chose the solution you did and what alternatives you considered, etc...

run buildall

run buildall

run buildall

run buildall


Rotation direction of arrows is not  fixed.

-The rotation direction of these arrows hasn't been fixed.

Good catch 👍

-Verified these changes on Dev app v1.4.6(232).
-Now the direction of arrows is fixed.It is moving in anti-clockwise direction.


How to make the humanoid model do certain specific poses without being bothered by the physics

Hi
I'm a student trying to figure out how to give the humanoid a specific position. I get it that we can use keyframes in order to give the humanoid a starting initial pose and then physics takes over and the model falls down.
I actually want to make the humanoid do a few poses (more like an actual movement, like swing the hand or lifting the leg from the ground)
Is this possible to achieve without using Reinforcement learning? I am not bothered about the physics at the moment and just want the humanoid to perform these actions. (There are other steps that I have to do later on in the project which will need the physics hence I am sticking to mujoco and not some other software)

If you have a sequence of poses, say from motion capture, you can just set mjData.qpos to those values, call mj_kinematics and mj_camlight to propagate joints to Cartesian, and render a video.
If you want to track motion capture data in a physically realistic way, I recommend looking at @hartikainen's motion-capture tracking task in the MuJoCo MPC project.


Missing method user.getTopTracks

API method user.getTopTracks is missing.

Oops, actually added the interface for this, just forgot to pop the method in there, on it


Doesn't seem to work with indexed resources

Hello! I'm super excited about this tool, but I haven't been able to find a way for it to work with indexed resources (e.g. aws_security_group.many["foo"]). The error message I get when I try to migrate one of these is
╷
│ Error: Index value required
│
│   on  line 1:
│   (source code not available)
│
│ Index brackets must contain either a literal number or a literal string.
╵

Full replication follows your example in the readme:
git clone https://github.com/minamijoyo/tfmigrate
cd tfmigrate/
docker-compose build
docker-compose run --rm tfmigrate /bin/bash

In the sandbox environment, set up the initial statefile:
# mkdir -p tmp/dir1 && cd tmp/dir1
# terraform init -from-module=../../test-fixtures/backend_s3/
# cat << eof > main.tf
> resource "aws_security_group" "many" {
>   for_each = toset(["foo", "bar"])
>   name = each.key
> }
> eof
# terraform apply

A terraform state list will show two resources:
aws_security_group.many["bar"]
aws_security_group.many["foo"]

Now we set up a second root:
# cd .. && mkdir dir2 && cd dir2
# cat ../cir1/config.tf | sed 's/test\/terraform.tfstate/dir2\/terraform.tfstate/' > config.tf
# touch main.tf
# terraform init

Move a resource out of dir1:
# cd ..
# cat << eof > dir1/main.tf
> resource "aws_security_group" "many" {
>   for_each = toset(["foo"])
>   name = each.key
> }
> eof
# cd dir1 && terraform plan
...
Plan: 0 to add, 0 to change, 1 to destroy.
# cd ..

Define a resource in dir2:
# cat << eof > dir2/main.tf
> resource "aws_security_group" "many" {
>   for_each = toset(["bar"])
>   name = each.key
> }
> eof
# cd dir2 && terraform plan
...
Plan: 1 to add, 0 to change, 0 to destroy.
# cd ..

Write a migration file describing this change:
# cat << eof > tfmigrate_multi_state_test.hcl
> migration "multi_state" "test" {
>   from_dir = "dir1"
>   to_dir = "dir2"
>
>   actions = [
>     "mv aws_security_group.many[\"bar\"] aws_security_group.many[\"bar\"]"
>   ]
> }
> eof

Run tfmigrate plan & apply:
# TFMIGRATE_LOG=DEBUG tfmigrate plan tfmigrate_multi_state_test.hcl
...
failed to run command (exited 1): terraform state mv -state=/tmp/tmp849889331 -state-out=/tmp/tmp278533622 -backup=/dev/null aws_security_group.many[bar] aws_security_group.many[bar]
stdout:

stderr:
╷
│ Error: Index value required
│
│   on  line 1:
│   (source code not available)
│
│ Index brackets must contain either a literal number or a literal string.
╵

╷
│ Error: Index value required
│
│   on  line 1:
│   (source code not available)
│
│ Index brackets must contain either a literal number or a literal string.
╵

I have also attempted the following migration files, all with the same error message as above:
migration "multi_state" "test2" {
  from_dir = "dir1"
  to_dir = "dir2"
  actions = ["mv aws_security_group.many['bar'] aws_security_group.many['bar']"]
}

{
  "migration": {
    "multi_state": {
      "test3": {
        "from_dir": "dir1",
        "to_dir": "dir2",
        "actions": ["mv aws_security_group.many[\"bar\"] aws_security_group.many[\"bar\"]"]
      }
    }
  }
}

{
  "migration": {
    "multi_state": {
      "test4": {
        "from_dir": "dir1",
        "to_dir": "dir2",
        "actions": ["mv aws_security_group.many['bar'] aws_security_group.many['bar']"]
      }
    }
  }
}

If I were writing the state move command manually, I'd have to put
terraform state mv "aws_security_group.many[bar]" "aws_security_group.many[bar]"

In a little while I'll try digging in to the source and see if I can open a PR for this fix.

After more debugging than I care to admit, I got a migration file that does what I want it to do:
migration "multi_state" "test27" {
  from_dir = "dir1"
  to_dir = "dir2"
  actions = ["mv 'aws_security_group.many[\"bar\"]' 'aws_security_group.many[\"bar\"]'"]
}

@spilliams Thank you for reporting the issue.
It looks a frequently asked question, so I added to a tip to README in #52. Thanks!


Allow indenting a task - [ ] with the tab key

Description
When I use a tasklist on GitHub, I often make a single task nested. I'd like to nest a task with pressing the tab key. GitHub doesn't allow that currently.
For example:
- [ ] task1
	- [ ] task2
- [ ] ${Here, I hope indent the `- [ ]` with the tab key, but this is not implemented}
- ${not here} [ ]

The GitHub-indent-comments userscript doesn't fit this requirement.
Screenshot
https://user-images.githubusercontent.com/38400669/208927043-0fc46d9d-22b0-409c-811f-60b44fa20f45.mp4
Example URLs
https://github.com/refined-github/refined-github/issues
If this is duplicated, please close this.

Refined GitHub already supports tabbing: either select the line or move the cursor to the beginning of the line, then press tab

@fregante

either select the line or move the cursor to the beginning of the line, then press tab

Yes, that's true. but I kinda frustrated to move the cursor to the beginning or selecting the line to indent a task.
When creating many tasks, I'd like to indent more smoothly without any actions.

That specific behavior is tracked and described here:

https://github.com/fregante/indent-textarea/issues/3


feature request: not have "never expire" as default

It would be useful for us to not have "never expire" as default selection for new pasts
Usecase:
I have deployed wastebin internally in a small company for people to share information among each other that does not fit inside a chat, or do not belong in a chat, where syntax need to be preserved, or in case one doesnt want to clutter documentation with raw text.
So i see people using it to:

share (semi)sensitive data that should expire within a day or week
share code snippets that also lose relevance very quickly
large raw log-files for troubleshooting that can contain also sensitive data sometimes and should expire until the troubleshooting is completed
store supporting data for documentation and reports probably need to remain available for 1 year and upwards

How do people use it:

paste information
press the go button
copy the link
call it a day

And thus you can see they forget to select an appropriate expiration resulting in:

server cluttering with data noone needs (creating digital-waste)
potential sensitive data being stored longer than needed (you know what endusers are like)

Potential solution 1:
allow the sysadmin to set which is the default pre-selected expiration
Con: this might lead to users pre-maturly losing their content
Pro: Previous con leads to user-education on how to correctly use the tool
Pro: this retains the quick flow of creating a paste, where the user can opt not to click on a expiration option
Potential solution 2:
not set a preselected expiration at all, and not accept sumbission of the paste until the use has done so
Con: this breaks the quick create workflow by introducing 1 additional click
Con: a bit of extra effort in coding the feature since the UX flow need to be changed including writing proper error messages
Pro: user-education without pissing users off for accidentally prematurly losing their content (potential solution 1), or pissing the sysadmins off by cluttering the system with items that exist longer then required.
Personally i like option 2 the most even if it's a bit more effort, however i cannot judge the usecases of other people.

Thanks for the request! I see where you are coming from, I guess opting into solution 2 would be a good compromise.

If WASTEBIN_MAX_PASTE_EXPIRATION is set, the default expiry will be 600 seconds, it is too short for most times, if not set, it will default to never, and it is unrealistic, maybe made a env that can change this default setting?

+1, would love to have a default env setting, separate from max paste expiration

Makes sense + setting the default to whatever was chosen as the maximum expiration in case the default is not set.


Home Page Featured Carousel

https://reuse-home.org.uk/ looks like this pretty please

Implemented....
I see the mobile version has huge text, which needs to be fixed.

Font size fixed,
Also, captions should be styled as per the wireframe, with the brand logo to the left

Hm, the main difference I see now I've uploaded the new theme on the site is that the logo is magenta, not white.

The logo image is huge and lilac, any chance it could be smaller.. and white?

Here's the folder for the icons, inclulding the white logo:
https://drive.google.com/drive/u/0/folders/1MDsqERicpcSRi9iIBVkLa9zzbBqa7ty0

White logo added, configurable in a custom theme setting, and that bug on chrome now fixed.


[WIP]feature: added ssl_protocols_mode to skip detail of ssl_protocols.

fixed https://github.com/iresty/apisix/issues/666

Infrastructure implementation, I don't like this way, we need simple and flexible solutions.
This ssl_protocols_mode  can be done on the dashboard.

Infrastructure implementation, I don't like this way, we need simple and flexible solutions.
This ssl_protocols_mode  can be done on the dashboard.

we can add custom mode later.


500 Server Error on FastCGI

40 FastCGI sent in stderr: "PHP message: Argument 1 passed to Model\Universe\SystemModel::Model\Universe{closure}() must be an instance of Model\Universe\object, instance of stdClass given [/var/www/pathfinder/pathfinder/app/main/model/universe/systemmodel.php:146]
Will check details and update...
Thx,
-Micky

Had to comment the "object" type in the closure definition:
       $data->shattered        = count(array_filter($planetsData, function(/*object*/ $planetData){
            return property_exists($planetData, 'type') &&
                (strpos(strtolower($planetData->type->name), '(shattered)') !== false);
        })) == count($planetsData);

Then it worked. Maybe that helps ...
Thank you !
-Micky

@der-micky Thanks for the report. Type hints for object was introduced in PHP 7.2 https://blog.tekmi.nl/php-7-2-and-object-typehint/
I´ll remove the type and keep 7.1 support for the next release.


"_id" doesn't appear to be a constant field across entries

Migrate to something like the provided SHA1. Add in a catch to silently perform said migration, too, to avoid needing to dump articles.

Since moving to BibTeX, the article key has proven relatively reliable as an ID that doesn't rely on matching titles.
Closing as this is relatively solved, but will revisit if needed.


exp: Drop checkpoints.

https://github.com/iterative/dvc/issues/9221 / https://github.com/iterative/dvc/pull/9271

@daavoo & @dberenbaum  - should this be dropped from docs asap or only after 3.0 release (https://github.com/iterative/dvc/issues/7093) ?

@daavoo @dberenbaum  - marking blocked for now but that's only based on my understanding of you rather waiting for the release or to plan a deprecation notice.
If that's not the case please remove the label and merge

Keeping this blocked for now. People are still using checkpoints and we don't have an alternative at this point, so let's not rush to drop it.


Attempt to fix musl build.

Follow-up to #685 and #700.

Thanks! Looks good to me, but CI is failing?

Fixed the style errors; waiting for Travis again.

It appears to be passing!

@bors: r+

:pushpin: Commit 59904c6 has been approved by alexcrichton

:hourglass: Testing commit 59904c6efe05c7c22ca070563b217c7dd1a7762b with merge 334abbc4f0c52a509fdb5586a6b14a577e2c8168...

@bors: r- retry
er actually, can you verify this works in the rustc repo?

:sunny: Test successful - status-appveyor, status-travis
State: approved= try=False

I made a PR for the rustc repo to test it out; I think that's how this is supposed to work?

@alexcrichton it seems to work in the PR I made, so, I think this is okay to merge?

Er what I mean is to actually test this. For example actually build libstd for these targets and make sure it works.
One error, for example, is that stdbuild is no longer a thing. For example this PR as-is should break linkage on all these targets.

:umbrella: The latest upstream changes (presumably #824) made this pull request unmergeable. Please resolve the merge conflicts.


uwsm: starting apps with uwsm have random paths added

Describe the bug
After enabling uwsm programs that are started with uwsm app -- program have some nix paths added. This leads to running bash instead of bashInteractive in a terminal.
Steps To Reproduce
Steps to reproduce the behavior:

Enable uwsm
Open terminal with uwsm app --foot
Try to run bash
See bash: shopt: progcomp: invalid shell option name and weird rendering

Expected behavior
No new paths added.
Screenshots
With uwsm app -- foot:

Should be:

Metadata

system: "x86_64-linux"
host os: Linux 6.11.8, NixOS, 25.05 (Warbler), 25.05.20241115.5e4fbfb
multi-user?: yes
sandbox: yes
version: nix-env (Lix, like Nix) 2.91.1 System type: x86_64-linux Additional system types: i686-linux, x86_64-v1-linux, x86_64-v2-linux, x86_64-v3-linux Features: gc, signed-caches System configuration file: /etc/nix/nix.conf User configuration files: /home/karun/.config/nix/nix.conf:/nix/store/gqpgq34fbzmxaak73j7a1jkph5v3chav-stylix-kde-config/nix/nix.conf:/etc/xdg/nix/nix.conf:/home/karun/.nix-profile/etc/xdg/nix/nix.conf:/nix/profile/etc/xdg/nix/nix.conf:/home/karun/.local/state/nix/profile/etc/xdg/nix/nix.conf:/etc/profiles/per-user/karun/etc/xdg/nix/nix.conf:/nix/var/nix/profiles/default/etc/xdg/nix/nix.conf:/run/current-system/sw/etc/xdg/nix/nix.conf Store directory: /nix/store State directory: /nix/var/nix Data directory: /nix/store/kqpjnkkpv1drbnvrqbhnyd2ixy9dymzb-lix-2.91.1/share
channels(root): "home-manager, nixos-24.05"
nixpkgs: /nix/store/hfz1qqd0z8amlgn8qwich1dvkmldik36-source

Notify maintainers

@JohnRTitor @kai-tub

Note for maintainers: Please tag this issue in your PR.

Add a :+1: reaction to issues you find important.

Yeah, it's because we are wrapping uwsm with bash and other packages, else it won't work because it won't find the dependencies at runtime (it's just a python script). https://github.com/NixOS/nixpkgs/blob/23e89b7da85c3640bbc2173fe04f4bd114342367/pkgs/by-name/uw/uwsm/package.nix#L66-L83
This however, leaks the dependencies to any of it's subprocesses/processes launched with it.
To be honest I really don't like the wrapping approach, but here's the fix anyway https://github.com/NixOS/nixpkgs/pull/358883


Post-MVP cleanups

Simplify task store data structure
Think about reducers not ever actually being called with undefined actions, but they are called with any action.
Maybe replace locale builder with locale generator functions similar to aString(), i.e. aLocale(), aLocaleWithCode(code), etc.

Selectors
Reorgnanize selectors, by creating the following categories of functions (where not far from this now, it just needs more consistency).
Given that these functions may be composed in many different ways, the hope is that many of them will be shared, the most flexible arrangement is to have each function in its own file, named to match the function name. Given the consistent function naming scheme, I think discovering and reusing existing functions will work well and we should avoid needless duplication of functionality.

selectors: located in /src/selectors/, named selectFoo(), all accept app.Store as their only argument, and return types such as selectors.Task (as opposed to stores.Task). Their implementations consist exclusively of calls to the helper functions below, and they contain no loops or branches --- this is so that they need not be unit tested. Big bonus: we avoid unit tests that work with the entire app store. They are called from mapStateToProps() functions. They are closely coupled to the view, since they build data structures exactly as the views need them to be. Another big bonus: since there is no massaging or other logic in mapStateToProps() functions, we don't need to unit test those either.
pullers: located in /src/selectors/detail/ (which implies that they will only be called from code in /src/selectors/ and /src/selectors/detail/), named pullFoo(), all accept app.Store as their only argument, and return some sub store, such as the task user settings map. These functions are the only ones outside of reducers that know the details of how the store is structured. They are all one-line functions and are not unit tested.
filters: located in /src/selectors/detail/, named filterFooByBar(), they contain the logic for taking one or more collections of store data and returning the subset needed for the view, such as the tasks tagged with a given taxonomy. They accept and return types such as stores.Task (as opposed to selectors.Task. These functions are unit tested.
creators: located in  /src/selectors/detail/, named createFoo() or createFooFromBarAndBaz(), they accept one or more collection of stores data structures such as stores.Task and stores.TaskUserSettings, and return data structures as needed for the view, i.e. collections of selector data structures such as selectors.Task. These functions are unit tested.
To avoid circular dependencies, selector types need to be defined in their own files too, either in /src/selectors/task.ts etc., or in /src/selectors/types/task.ts etc.

This naming scheme, we should be able to list the files in a directory in sorted order and get a nice, readable table of the functions, grouped by subject area.

Here is an approach to handling generic Redux Actions in reducers:
// @types/redux.d.ts
// This makes these types global, but they could be placed anywhere.

declare interface Payload {}

declare interface Action<P extends Payload = Payload> {
    readonly type: string;
    readonly payload: P;
}

Now your reducer can be:
interface MyStoreActionPayload extends Payload {
    /* ... */
}

function myStoreActionCreator(arg1: string, arg2: number, ...): Action<MyStoreActionPayload> {
    return {
        "type": constants.MYSTORE_ACTION,
        "payload": { /* ... */ },
    };
}

function reducer(store: MyStore = buildDefaultMyStore(), action: Action) {
    switch(action.type) {
        case constants.MYSTORE_ACTION: {
            const myStoreActionData = action.payload as MyStoreActionPayload; // This cast is as safe as it's going to get.
            // ...
        }
    }
}

https://scotch.io/amp/tutorials/3-useful-typescript-tips-for-angular contains some useful ideas

https://medium.com/netscape/redux-reducers-functional-approach-69c0f6bacdc2


Add measurement constraints to xmon_device

@maffoo says sets of qubits have to be measured at the same time as each other always.

Hey @maffoo, how's this going?

Pinging @Strilanc too.

Looks like @mafoo is back, so i'll ping him again :)

This is de-facto obsolete now. Related: https://github.com/quantumlib/Cirq/pull/2141


No need to create a credits with microcredits =0

When a user transfers all of their microcredits to the "to" address, in addition to creating a credits record for the "to" address, a credits record with microcredits set to 0 is also created for the "from" address. Generating a large number of credits records with microcredits set to 0 can cause wastage of network resources.
https://github.com/AleoHQ/snarkVM/blob/1485bdd42a6156e09dc09066b262302df9252bf0/synthesizer/program/src/resources/credits.aleo#L792

Need to know the impact of the "wastage."  Still, wouldn't block mainnet for this.
Can you demonstrate the impact of the waste?

These 0 balance records are necessary to preserve privacy. A transfer_private call should always have the same number of inputs and outputs in order to reveal as little information about the amounts/data as possible, even if a record is considered "wasted" due to a balance of 0.
I am closing this due because this is a desired property of our system.


CI: pin Python 3.7.16 for MacOS CI tests

Python 3.7.17 binary on MacOS seems to be binary incompatible with
pandas-1.3.5, the last pandas release supporting Python 3.7.
Addresses #6147

Can we remove Python 3.7 support since it's EOL is in a week?

Can we remove Python 3.7 support since it's EOL is in a week?

Fine with me, but I guess this needs a bit more discussion.
Perhaps we should do so in the next minor release?
The problem appears to be in a broken Python 3.7.17 in GitHub runner image for MacOS.
For now I am going to disable the failing test in a separate PR.

Withdrawing - replace by #6151.  Keeping the branch around for the sake of https://github.com/actions/runner-images/issues/7764


Task description newline characters removed by Jinja templating, affecting generated requests and performance

What is the problem?
The code changes from PR #1553 improperly remove trailing newline character \n from task description when provided, resulting in unintended changes to the generated request and potential performance implications..
For example, the subtask temporal sequence from bbh_cot_fewshot includes a task description with a trailing newline:
Task description: Answer questions about which times certain events could have occurred.\n\n.
Ideally, the generated request should preserve the description content as it is. However, after calling function apply_template , the trailing newline characters are removed, altering the content to:
Task description: Answer questions about which times certain events could have occurred.\n.
Why?
This behavior is due to how Jinja2 handles whitespace control and newline characters in templates. By default, Jinja2 removes trailing newline characters from templates. This is done to allow better control over whitespace in the rendered output, especially when working with HTML templates where extra newlines can affect the rendering and formatting of the content.
How to fix?
We have two main options to address the issue:

Revert back to previous implementation
We could remove apply_template function to make the description filed unmodified. This is a low-risk solution as we simply revert back the original implementation.
Modify the apply_template function to properly handle whitespaces and newlines
If we want to keep the templating feature for description, then we need to modify the function to handle whitespaces and newline characters carefully. One candidate solution be to use Jinja's trim_blocks and lstrip_blocks options to control whitespace trimming. However, the risk is unclear since this code change might affect the rendering behavior in other ways.

env = Environment(loader=BaseLoader,
                  undefined=StrictUndefined,
                  lstrip_blocks=True,
                  trim_blocks=True)
env.filters["regex_replace"] = regex_replace

def apply_template(template: str, doc: dict) -> str:
    rtemplate = env.from_string(template)
    return rtemplate.render(**doc)

Thank you for tracking this down!
I think the preferred fix would be:

revert back to previous implementation (description does not have apply_template applied)
have tasks which required this simply generate a description column using process_docs and use that (along with whatever desired trailing whitespace) for custom per-document description fields.

Hi, looks like we are still using "utils.apply_template" to generate the description column. Is someone working on it?


TweetInvi.AuthFlow.InitAuthentication  returns null

Our company has a web app that integrates with twitter . Well, it used to. Our implementation was working up until 06-18-2018. Our implementation was created using the examples and TweetInvi. It's hard to tell, but it looks like twitter is making some changes. I've added "real" callback URLs at apps.twitter.com - that seemed to be most people's fix. Didn't work for me. Can't find a way to contact a developer at twitter - anyone have any suggestions?

Hi @vinceweeks
Since we don't see your code, it can be difficult to help you but I'm going to try
You need to make sure that you give the full URL where Twitter will send back its values to be able to authenticate your users again. You also need to specify the port if you have one in your endpoint.
The following works fine in my case:

As my webapp listen to the /Twitter/Validate endpoint, I had to give this path to Twitter.

Thanks for getting back to me. here's an example of one of our callback URLs: https://trackmypos.com/Twitter/Confirmation	- No port number - this URL had been working but now it does not. Any other suggestions?

I can show code if you need to see it - my point is, our code was working, hadn't been changed in probably six months and now does not work.

It was working before because Twitter asked for a url without really verifying it, it changed recently and you now need to give the exact urls used by your webapp to validate the authentication request. If I'm not wrong you also need to take care to write the http and https if both are accessible on your website.
In my case, the authentication flow goes like this:
1. /Home/Index
2. Clicking on login button -> redirect to /Twitter/Auth which redirect the user to Twitter
3. <twitter login> -> redirect to /Twitter/Validate
4. /Twitter/Validate set some cookies then redirect to /Home/Menu

So I had to put https://<my website>/Twitter/Validate has a callback url to tell Twitter to allow this URL for the logging function.

Hi Lynn – I couldn’t make it work – I changed our approach to just make the required REST calls. Thanks for getting back to me.
From: Lyyn<EMAIL_ADDRESS>Sent: Wednesday, June 27, 2018 12:58 AM
To: linvi/tweetinvi<EMAIL_ADDRESS>Cc: Vince Weeks<EMAIL_ADDRESS>Mention<EMAIL_ADDRESS>Subject: Re: [linvi/tweetinvi] TweetInvi.AuthFlow.InitAuthentication returns null (#724)
It was working before because Twitter asked for a url without really verifying it, it changed recently and you now need to give the exact urls used by your webapp to validate the authentication request. If I'm not wrong you also need to take care to write the http and https if both are accessible on your website.
In my case, the authentication flow goes like this:

/Home/Index

Clicking on login button -> redirect to /Twitter/Auth which redirect the user to Twitter

 -> redirect to /Twitter/Validate

/Twitter/Validate set some cookies then redirect to /Home/Menu

So I had to put https:///Twitter/Validate has a callback url to tell Twitter to allow this URL for the logging function.
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHubhttps://github.com/linvi/tweetinvi/issues/724#issuecomment-400578796, or mute the threadhttps://github.com/notifications/unsubscribe-auth/AIIWzGo439_5gCurVnkzAwbVcSi-K90oks5uAzr8gaJpZM4U29IE.

Hello,
If you have question for the twitter team I would suggest you go on the developer forum https://twittercommunity.com. They usually reply.
I would suggest that you put the real url in your callbacks and try to reset your app credentials.
If this is not working try the same thing with a different app just in case your application got blocked by Twitter bot detection.
Let me know if this helped in any way.

You can also have a look here : https://github.com/linvi/tweetinvi/issues/718
They seemed to be experiencing the same problem.


Sidebar Implementation

Creating a Sidebar component using Prime faces.
Sample Output

Created sidebar tree  as much same from drawio sidebar .
Sample :

Concern :

But there is issue that websight do not have sidebar space, while we are using maincontainer also and layouts, still there is
space in left side of the page.
If we use layouts, we could not drop another layouts on it.

We also created a discussion for the same in websight/starter repo.
Please refer to the following link:
https://github.com/websight-io/starter/discussions/113


How to separate js bundle

How to separate js bundle to frameworkBundle and appBundle?
js bundle is more 500k+, but business code is less then 10%

@satya164

Wow, I just wanna fix this issue, and have tried https://github.com/mjohnston/react-native-webpack-server/ but failed.
Also I tried to get a dev version bundle and copy the framework code as framework.jsbundle, but when I package the business code, I cannot analysis the correct dependencies.
Looking forward this solutions~~~~

@Richard-Cao  IOS also have this issue~~

@Creemli Looking forward……

@Richard-Cao  Help to remove the [Android] tag and then more people can search this issue.

@Creemli ok

What's the purpose of splitting the bundle? Is it just to make incremental app updates lighter weight, or is there some runtime goal you're trying to accomplish?

both @corbt

To help me understand the scope of this, what are the use cases for this beyond just making lighter-weight app updates (which I get)?

@corbt  Yeah, this can make user download less code when updating.
And if we can separate them, we maybe not have to bundle the whole codes, we can combine any business project with the framework code, it's lighter, I think.

I think @martinbigio was looking into this.

Yeah, we were looking into this mostly to help on TTI but decided to pursue another approach, namely unbundling. To make code downloading faster it's not actually necessary to have multiple bundles but to use a "more clever" protocol (i.e.: one that could send diffs only).

@martinbigio  cool

If the main use case for this is just to have smaller updates, then I don't think bundle-splitting is the best solution. There should just be some library that manages updating the JS bundle file using versioned diffs. Such a solution would perform strictly better than bundle-splitting in the size of the updates it would need to send, and would also require less user intervention (in deciding where to split the bundle) to get right.
This would probably make more sense in something like Code Push than in React Native core.

Another possible use is to lazy load parts of a (bigger) app at runtime to reduce startup time. Or is that unlikely to make a difference?

I can give an example of what is being talked about here.
I have a tabbed UI app that already has a bunch of native code already written (iOS and Android). React Native tabs exist side by side with other webview and native tabs. For this scenario, I am having to load all of the app code (core + tab 1 + tab 2 + tab 3) for each tab. Splitting the bundle would mean that:

the app kicks off tabs faster (TTI?)
some functionality that is rarely used can be directly loaded from a server without having to be pre-cached all the time
updating a tab doesn't mean I have to send few megs down the network each time (I understand some diffing protocol was suggested up in the thread)

Also, what is unbundling?

ref #570

any  solution?

any updates?

Ah, try this: https://github.com/ctripcorp/moles-packer

any progress?

looking forward to updates

Looking forward...and...what is unbundling？

@Creemli  moles-packer only resolve the propblem separate js bundle, but how to load them in native module?

Hi @martinbigio, @corbt,
Our guys have been develop APPs with RN Integration for a long time. And we think bundle-splitting is also usefull in these scenarios (maybe not limited to Integration).

Decrease initial time (Run base codes previously)
If we could split some initial and definition codes to a base.jsbundle and run it in JSContext previously before users really enter React Native partition, it can decrease user's waiting time .

Update partially
Just like @Creemli said. If we just use a lock version RN(or update infrequently) , the most of changes and updates is about our business codes. We can also separate some framework code (i.e. React, RN Components, APIs, and even with some other framework) into a base bundle, but business codes in business bundle. So we can fetch business bundle standalone from network and get smaller size with download to hot update. Even without diff it's not too bad. I mean, maybe it's easier to implement than integrate a diff algorithm to the entire work flow.

Share common modules between multiple bundle
If we had the base.jsbundle like the above mentioned, we also can make multiple bundle share it as a common library (it's Just like DllPlugin usecase in webpack). Suppose we have two ReactRootView in our App, currently we should bundle out two jsbundle files both independent and overall with base codes, it includes many duplicated.

Let me generalize my opinion through this picture:
(In order to save space, I omit the --platform, .ios, .android here)

This simple way seems like the DllPlugin in webpack:

Compile-time
Build base.jsbundle. Meanwhile records it's dependecies list and save it, for example,  named by base.manifest.json .
Build index1.jsbundle with the base.manifest.json passed through cli. When resolve modules, use their id from base.manifest.json and skip their definition if they has recorded in the manifest. So a clear business bundle has done!
Build antoher business bundle like the above (We can set a prefix to module id to prevent override).

Run-time
Run base.jsbundle in JSContext at a good time (e.g. Native App launch done, enter to a parent Activity/Controller ).
Run index1.jsbundle when user click/open and will enter to this React Native partition.
Run index2.jsbundle  or other when user will enter to another React Native partition.
Update business bundle from CDN if needed.

How do you think ? 😄

any updates?

android can try unbundle. this will separate mian.jsbundle into pieces of js files。

@njafei unbundle is not a good design, too many small files in output bundles.
it's very easy to split main.jsbundle to base.bundle.js and business.bundle.js, but you need modify the native code for RN to load different parts.

@mc-zone Amazing!

@jimzhao2012  I don't is a good idea to modify RN code.　Mybe could eval business.bundle.js at base.bundle.js?

@mc-zone can you explain how to 【run base.jsbundle in JSContext at a good time】,don't get it, thanks

I hava package react-native code using webpack,https://github.com/yoution/rn-pack; and split code using commonChunk plugin,alse can fix module id in chunk; but I can not know how to load the splited bundle

@jimzhao2012 is there any resource to look deeper into it?
Also, how could we use and pack with unbundle? I can produce the unbundle files, but I don't know yet how to install and run the unbundle result and I don't find many resources on how to do it.

For those who wants to try out unbundle, there's a PR documenting it:
https://github.com/facebook/react-native/pull/15317/files

I'm working on an existing app rewrite, to separate js bundles and hot reload are two important feature for me to investigate, reasons are:

We are running agile approach, so would like delivery our artifact for each sprint, but deliver through app store/google play too often definitely is not good for commercial app. So we would like to deliver major version by store, minor version by hot deploy.
splitting bundle is because inside the app, there are different modules developed under different project, and some function is not used quite often, for example help & service. to have separate bundle will reduce the dependency and decrease project risk, since we can deliver and replace modules separately

Beyond that, we are also interested in bundle encryption/decryption, and file patching, which make is more useful for commercial app.

+1 mark

Also that can speed up ci/server-side builds. Most of builds has same versions of libs, only code of app changes. We can --reset-cache for app bundle only.
For example: we have project with 1200 npm packages. android ci build takes about 3m, ios 5. 80% of build time takes building of bunde.js
Also we can bundle it with two package processes to speed up.

Would this be the equivalent to webpack DDL plugin? I made the following question in stackoverflow about the unbundle speed for rebuilds and I am interested in finding a guide to follow the same strategy for react-native.
Maybe it won't be possible for code-push updates or faster app initialization, but I am interested in improving the DX.

Was anybody able to use unbundling in combination with CodePush?


Unable to toggle notifications on Mac

Great app - thanks!
Just one problem - I don't get notifications on Mac.  I tried to go into the settings and toggle the notifications option (which is turned on) and I get a message at the bottom left that says

Notifications for this site are blocked on your browser

I have used the website both in safari and chrome and have notifications enabled and do get notifications there... I am not sure what else to try.
Any help is appreciated!

Hmm - actually it appears notifications ARE working... There just is not any notification if the app window has focus.  Which makes sense.
That said - it would still be nice/ideal if one could toggle notifications (On another one of my Macs I don't want notifications) - but when I try to toggle I do in fact get the message mentioned above.
Thanks again for the app!

Hmm - actually it appears notifications ARE working... There just is not any notification if the app window has focus. Which makes sense.
That said - it would still be nice/ideal if one could toggle notifications (On another one of my Macs I don't want notifications) - but when I try to toggle I do in fact get the message mentioned above.
Thanks again for the app!

Yeh just realised as well ha

Verified same issue in Windows (probably expected).


Add references to the BinaryOnly OME-TIFF fileset sample

This PR should add hyperlinks to the new OME-TIFF samples added in the context of https://github.com/openmicroscopy/bioformats/pull/2314 to the format specification page.

Looks alright (as far as I understand it), files are properly linked, too. http://www.openmicroscopy.org/site/support/ome-model-staging/ome-tiff/specification.html
:+1:


added getnewargs() to str.rs

Added getnewargs() to str.rs and have tested on test_unicode.py(test_getnewargs())

#3807 was recently merged, which changes how CI runs the test suites. Can you rebase your branch on a fresh copy of main? Once you've done that, feel free to run the following command:
cargo run --release --features ssl,jit -- -m test -j 1 -u all --slowest --fail-env-changed -v test_enum test_pickle test_pickletools test_unicode

(Of course, everything after the -v flag can be changed out with whatever tests you want. Or none at all, if you have plenty of time to spend.)

You mean that delete the test 'functions' which I deleted expectedFailure annotation?

Any lines that now say
def test_function(self):
    super().test_function()

are just redundant at this point if they're not annotated with a @unittest.expectedFailure or a @unittest.skip decorator. They also make it harder to upgrade the test suite in the future.
If a test function has something else other than a call to super(), you can leave it be.

Is this right?

Thank you, too!!

According to the CI, it seems you may have missed a test in test_pickle.py. Are you able to replicate what CI is showing?

@zer0who please let me know if you don't care about email address. I don't mind to merge it as it is if you are ok.

@zer0who please let me know if you don't care about email address. I don't mind to merge it as it is if you are ok.

Here is GitHub's documentation to set your commit email address.
In order to fix these commits, you can run the following commands:
git fetch upstream
git rebase --exec 'git commit --amend --reset-author' -i upstream/main

@zer0who please let me know if you don't care about email address. I don't mind to merge it as it is if you are ok.

I don't know exatly that email you mentioned, but I did it as I understanded. Is it right?

check git log in your branch. the email of your commit is not your actual email.
(I don't think your email is<EMAIL_ADDRESS>

@zer0who please let me know if you don't care about email address. I don't mind to merge it as it is if you are ok.

Here is GitHub's documentation to set your commit email address.
In order to fix these commits, you can run the following commands:
git fetch upstream
git rebase --exec 'git commit --amend --reset-author' -i upstream/main

I forgot to add git push --force to that list.

@zer0who Yeah It seems good now 😊

Am I right??

Here's how you can tell:

On GitHub, you can look to see if your avatar is displayed next to each of your commits.
Locally, running git log will tell you who authored a given commit. There will normally be a line that says Author: Your Name<EMAIL_ADDRESS>If the email address in that commit matches one that GitHub recognizes (e.g. the one linked to your GitHub account), then GitHub will display your avatar next to each of your commits.

Am I right??

Here's how you can tell:

On GitHub, you can look to see if your avatar is displayed next to each of your commits.
Locally, running git log will tell you who authored a given commit. There will normally be a line that says Author: Your Name<EMAIL_ADDRESS>If the email address in that commit matches one that GitHub recognizes (e.g. the one linked to your GitHub account), then GitHub will display your avatar next to each of your commits.

Thank you so much :)


Add Ripe cloud mode docs

Proposed changes
Added documentation for the Ripe cloud mode integration and updated the documentation for the device mode integration accordingly. See the cloud mode PR for reference: https://github.com/segmentio/action-destinations/pull/952
The device mode integration has been renamed to Ripe Web Destination (Actions) for clarity, and the new cloud mode one is called Ripe Cloud Destination (Actions).
Merge timing

ASAP once approved

Related issues (optional)
https://github.com/segmentio/action-destinations/pull/952

@simpajj I'll merge this in once the source is available in the catalog

Moved this to #4118


JS error thrown in console on all pages: TypeError: setting getter-only property "name"

Gitea version (or commit ref): 1.15.0+dev-184-g6ea6e2b4e
Git version: unknown, the one on try.gitea.io
Operating system: try.gitea.io
Database (use [x]): try.gitea.io instance
Can you reproduce the bug at https://try.gitea.io:

[x] Yes (provide example URL) https://try.gitea.io/wolfbeast/pale-moon-localization/issues
[ ] No

Log gist:

Description
Browser: Pale Moon 29.1.1 x64
Error thrown in console:
16:15:37.848 error { target: SharedWorker, isTrusted: true, message: "TypeError: setting getter-only property "name"", filename: "https://try.gitea.io/js/eventsource.sharedworker.js", lineno: 1, colno: 35, srcElement: SharedWorker, currentTarget: SharedWorker, eventPhase: 2, bubbles: false, cancelable: true } 1 index.js:150:2396
	Jh/</< https://try.gitea.io/js/index.js:150:2396

The code in question is:
self.name="eventsource.sharedworker.js";

I'm not sure what self in this context is, but it obviously doesn't seem to have a setter defined.
Note: it doesn't seem to affect functionality in a major way that I know of; it is also present on our production system running 1.12.6

It should be https://developer.mozilla.org/en-US/docs/Web/API/SharedWorkerGlobalScope/name which is indeed read-only. I'm not sure what the purpose of the assignment exactly is.

I'm assuming it would cause loss of functionality if a TypeError is thrown there, even if not critical. It'd abort the script.

I'm not sure why I put it in - I'm sure I was copying from some other code that did set it - but clearly it's incorrect so let's just remove it.

FWIW, we already give that worker a name via initialization, so there's certainly no reason to override it.
new SharedWorker(`${__webpack_public_path__}js/eventsource.sharedworker.js`, 'notification-worker')


[AIRFLOW-2048] Fix task instance failure string formatting

JIRA

[X] My PR addresses the following issues and references them in the PR

AIRFLOW-2048

Description

[X] Here are some details about my PR, including screenshots of any UI changes:
It seems that this commit inadvertently introduced a logging bug: https://github.com/apache/incubator-airflow/commit/a7a518902dcf1e7fd4bf477cf57cee691f181b29#diff-54a57ccc2c8e73d12c812798bf79ccb2R2285. This PR is a simple fix for that bug.

Tests

[X] My PR adds the following unit tests OR does not need testing for this extremely good reason:

Commits

[X] My commits all reference JIRA issues in their subject lines, and I have squashed multiple commits if they address the same issue. In addition, my commits follow the guidelines from "How to write a good git commit message":

Subject is separated from body by a blank line
Subject is limited to 50 characters
Subject does not end with a period
Subject uses the imperative mood ("add", not "adding")
Body wraps at 72 characters
Body explains "what" and "why", not "how"

[X] Passes git diff upstream/master -u -- "*.py" | flake8 --diff

@Fokko I believe this is ready to be merged now

Lgtm


Update subscription channels

chore: improve script output
fix: lock TAS channel to 1.0
fix: downgrade to OSP 4.14

/lgtm


Android: Add setLaunchOptions to ReactRootView

Motivation
When ReactRootView is used in a Android layout file, its properties is expected to be changeable at run time like other native views. iOS RCTRootView has a method named setAppProperties, but no such method found on Android.
Test plan (required)
Create a Android project for TicTacToe example.
Modify TicTacToeApp.js in Examples/TicTacToe:
Change from
<Text style={styles.title}>EXTREME T3</Text>
to
<Text style={styles.title}>EXTREME T3 {this.props.title}</Text>
Implement a TicTacToeActivity:
public class TicTacToeActivity extends ReactActivity {
  private String mTitle;

  @Override
  protected String getMainComponentName() {
    return "TicTacToeApp";
  }

  @Nullable
  @Override
  protected Bundle getLaunchOptions() {
    Bundle args = new Bundle();
    args.putString("title", mTitle);
    return args;
  }

  @Override
  public boolean onKeyUp(int keyCode, KeyEvent event) {
    if (keyCode == KeyEvent.KEYCODE_1 || keyCode == KeyEvent.KEYCODE_2) {
      mTitle = keyCode == KeyEvent.KEYCODE_1 ? "1" : "2";
      notifyLaunchOptionsChanged();
    }
    return super.onKeyUp(keyCode, event);
  }
}

Launch TicTacToe app in an emulator, press 1 or 2, you can see the title is changed.

By analyzing the blame information on this pull request, we identified @cjhopman and @foghina to be potential reviewers.

It's been a while since the last commit was reviewed and the labels show this pull request needs review. Based on the blame information for the files in this pull request we identified @cjhopman as a potential reviewer. Could you take a look please or cc someone with more context?

cc @foghina

AFAICT this requires restarting the app when calling setLaunchOptions? This is highly inefficient and I'd rather have a placeholder in XML that gets replaced with a ReactRootView in onCreate with the correct launch options.

I'd rather not add this API for the reasons stated above. Feel free to reopen if I misunderstood something.


Update faq-product.rst

Added support for search highlighting on mobile after testing.

Summary

Ticket Link

@esethna Can you please review the chart for any changes you are aware of?

Thanks Katie, looks accurate


Add benchmark for climatology

Closes #1549
This benchmark doesn't even run at the small scale yet.

This should be good for a review now. (I expect that we will iterate on this once we have improved on some things within Xarray/Dask.)


How can I make a cropped face cover a larger area?

Testing the node on different photographs, and on most of them, even with different aspect rates, the result is almost always cut off by one of the following:

Ears
Hair
Chin

Is it possible to add some kind of parameter that will say to capture not only the center of the face, but a little more pixels around?
I tried changing the scale_factor, but for some reason it doesn’t affect the result in any way..
This would be very useful for ComfyUI workflows.

"I tried changing the scale_factor, but for some reason it doesn’t affect the result in any way.."
That should be a bug. Let me check real quick!!

Fixed. Please give it another try. Thanks!

from fast checking: looks like now it is working.
Thank you for fast reaction.


investigate env supporting by symfony config

https://symfony.com/blog/new-in-symfony-3-2-runtime-environment-variables

will not do


[DBC Question] Does dbc format support "2 parents" Extended Multiplexing?

Sorry, this is a not an issue related to cantools.It is a question about dbc file format.
It is about how to use DBC SG_MUL_VAL_ key word to define Extended Multiplexing.
I am hoping a dbc description file like below to work. Which supports an Extended Multiplexing by an AND logic Multiplexing of SERVICE and PID. I test with cantools, it seems cantools doesn't support it. And I am not sure if DBC specification supports such kind of usage.
BO_ 2024 RX: 64 ECU
 SG_ RXLEN : 0|8@1+ (1,0) [0|0] "" TOOL
 SG_ SERVICE M : 8|4@1+ (1,0) [0|0] "" TOOL
 SG_ RXACK : 12|4@1+ (1,0) [0|0] "" TOOL
 SG_ PID M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S01_PID00_SupportedIDs_01_20 m1m0 : 24|32@1+ (1,0) [0|0] "" TOOL
 SG_ S02_PID00_SupportedIDs_01_20 m2m0 : 24|32@1+ (1,0) [0|0] "" TOOL

SG_MUL_VAL_ 2024 S01_PID00_SupportedIDs_01_20 SERVICE 1-1;
SG_MUL_VAL_ 2024 S01_PID00_SupportedIDs_01_20 PID 0-0;
SG_MUL_VAL_ 2024 S02_PID00_SupportedIDs_01_20 SERVICE 2-2;
SG_MUL_VAL_ 2024 S02_PID00_SupportedIDs_01_20 PID 0-0;

My current workaround is like below, which renames PID to different names to force creating a multiplexing route, this makes the file very tedious and break things, it leads PID filed to be parsed as a different name, in logic the PID field should be parsed as the same name with different value.
BO_ 2024 RX: 64 ECU
 SG_ RXLEN : 0|8@1+ (1,0) [0|0] "" TOOL
 SG_ SERVICE M : 8|4@1+ (1,0) [0|0] "" TOOL
 SG_ RXACK : 12|4@1+ (1,0) [0|0] "" TOOL
 SG_ S01_PID m1M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S02_PID m2M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S03_PID m3M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S04_PID m4M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S05_PID m5M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S06_PID m6M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S07_PID m7M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S08_PID m8M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S09_PID m9M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S0A_PID m10M : 16|8@1+ (1,0) [0|0] "" TOOL
 SG_ S01_PID00_SupportedIDs_01_20 m0 : 24|32@1+ (1,0) [0|0] "" TOOL
 SG_ S02_PID00_SupportedIDs_01_20 m0 : 24|32@1+ (1,0) [0|0] "" TOOL

SG_MUL_VAL_ 2024 S01_PID SERVICE 1-1;
SG_MUL_VAL_ 2024 S02_PID SERVICE 2-2;
SG_MUL_VAL_ 2024 S03_PID SERVICE 3-3;
SG_MUL_VAL_ 2024 S04_PID SERVICE 4-4;
SG_MUL_VAL_ 2024 S05_PID SERVICE 5-5;
SG_MUL_VAL_ 2024 S06_PID SERVICE 6-6;
SG_MUL_VAL_ 2024 S07_PID SERVICE 7-7;
SG_MUL_VAL_ 2024 S08_PID SERVICE 8-8;
SG_MUL_VAL_ 2024 S09_PID SERVICE 9-9;
SG_MUL_VAL_ 2024 S0A_PID SERVICE 10-10;
SG_MUL_VAL_ 2024 S01_PID00_SupportedIDs_01_20 S01_PID 0-0;
SG_MUL_VAL_ 2024 S02_PID00_SupportedIDs_01_20 S02_PID 0-0;

PS: This question is come from a dbc project what I am doing. Creating a dbc file to fully support obdii (iso15031-5 and J1979DA). Repo is here https://github.com/jiapengLi/dbc

I guess you mean nested multiplexers? Whilst they are supported by both the DBC file format and cantools, they are -- as you've already noticed -- a pretty messy affair and the code in cantools is not very battle-hardened.
If you want to create a description of OBD-II for usage over ISO-TP, I would not recommend trying to do this using DBC because it is a file format specialized for regular signals-based communication. Instead, I'd like to recommend to use a file format adapted to the needs diagnostic descriptions. (I might be biased, though: I'm also involved in odxtools, an analog to cantools for diagnostics using the ODX / ISO22901-1 specifications.)

Hi @andlaus, I think what I need is a 2 same level multiplexors ( service and pid). I want to keep the name PID for different services. To do so, application layer just need check signals named SERVICE and PID. Ease the sw design

I think what I need is a 2 same level multiplexors ( service and pid).

in the context of cantools, that is a nested multiplexer. UDS and OBD-II telegrams look something like this:
+-------+-------+--------+--------+--------+----------+
| sid   | pid   | param1 | param2 | ...    |  param_n |
+-------+-------+--------+--------+--------+----------+

where the parameters depend on th PID and SID. while it is not completely impossible to specify this using nested multiplexers, such structures are very inconvenient to specify using DBC (or any other format supported by cantools). you have been warned ;)

@andlaus Thank you for your prompt reply. Agree with you, sid and pid are actually at the same level, but dbc (not sure it is specification related or implemetation related) seems not support it.
My current solution looks like below, have to rename or create a new pid variable, because same name means same param structure.
 +-------+-----------+--------+--------+--------+----------+
 | sid   |  s01_pid  | param1 | param2 | ...    |  param_n |
 +-------+-----------+--------+--------+--------+----------+
 +-------+-----------+--------+--------+--------+----------+
 | sid   |  s02_pid  | param1 | param2 | ...    |  param_n |
 +-------+-----------+--------+--------+--------+----------+

Another option maybe to merge pid and sid, but it is also inconvenient. ;(
BTW, I take a look at the odxtools roughly, is very interesting. Good job! I may go deep into it after I finish odbii dbc library.


CSV File Splitter To Log

A simple file reader that splits the row  and the columns of a csv file. All done in XML DSL.

Hey Oscar.
I suspect it is the csv file which will not have an apache license as per the other headers. Any idea on how to handle that?
Regards
Neil

https://github.com/apache/camel-quarkus/commit/91773d5306ad4fe070a689887d887e9e34785516
This should be fixed through the commit above. Can you please rebase?

Thanks


archive/zip: load file data descriptor on demand

Read file data descriptor only when the file is open.
Previously, during zip.Reader.init, each new file had its data descriptor
read during the file discovery loop. This caused extra reads for those who
want just to list the files.
Fixes #48374

Looks like this was already fixed by https://go-review.googlesource.com/c/go/+/357489 around the same time. Please don't feel discouraged that your change didn't get reviewed and merged - with such a large open source project, duplicated work happens at times.


Add support for compiling nim scripts

It would be neat to have built-in support for compiling nim scripts. The configuration would be:
{
"cmd": "nim",
"args": [ "compile", "{FILE_ACTIVE}" ],
"sh": false,
"cwd": "{FILE_ACTIVE_PATH}"
}
It would probably be easy to add support for a few other popular languages, such as go or rust (I don't mention C or C++ because atom-build already has makefile support).

I'd love support for more languages/tools natively. Since this support is just a massive if-else-if statement right now I think it needs to be refactored and more plugin-based so it can easily be extended first. I'm all for ideas!

I've made a first attempt at modularizing the build tools in 6efd5f3.
I though I could add support for nim scripts using this new approach.
What would determine if nim can run or not? Is it the extension of the current file? Is it a specific build file (such as Makefile, or package.json)?


Catch invalid JSON errors in schemas when watching

Summary
When mocking in watch mode, if the schema is not a valid JSON, the whole process exits with code 1 because of an uncaught error: unexpected end of JSON input
Steps to reproduce
Mock an invalid JSON schema. For example:
{"openapi": "3.0.0"

Actual result
The whole process (and the docker container) exit with an error code 1.
Expected result
The process should catch the error, show a controlled error message and keep watching changes until schema is a valid JSON.
More info
This happens if you edit the schema manually while watching, and save the file before you're done editing (this happens a lot when you have a continuoues-save-addiction in you).
Algo happens to me when syncing schemas from AWS S3 to my local machine (I guess it reads the file in the middle of the write operation).

This should be fixed in 5d19e0deb51a12ba6eebd5f0134f14a8cfff0403


nnn: init at 1.2

Motivation for this change
And another small and fast TUI file browser which was forked from noice.
Things done

[x] Tested using sandboxing
(nix.useSandbox on NixOS,
or option build-use-sandbox in nix.conf
on non-NixOS)
Built on platform(s)

[x] NixOS
[ ] macOS
[ ] Linux

[ ] Tested via one or more NixOS test(s) if existing and applicable for the change (look inside nixos/tests)
[ ] Tested compilation of all pkgs that depend on this change using nix-shell -p nox --run "nox-review wip"
[x] Tested execution of all binary files (usually in ./result/bin/)
[x] Fits CONTRIBUTING.md.

Thanks!


ECDSA signature malleability

Purpose of this issue
A security vulnerability was discovered in the affected version of @openzeppelin/contracts and  @openzeppelin/contracts-upgradeable >= 4.1.0  with Patched versions released on < 4.7.3.
As described the Impact includes:
The functions ECDSA.recover and ECDSA.tryRecover are vulnerable to a kind of signature malleability due to accepting EIP-2098 compact signatures in addition to the traditional 65-byte signature format. This is only an issue for the functions that take a single bytes argument and not the functions that take r, v, s or  r, vs as separate arguments.
The potentially affected contracts are those that implement signature reuse or replay protection by marking the signature itself as used rather than the signed message or a nonce included in it. A user may take a signature that has already been submitted, submit it again in a different form, and bypass this protection.
The following contract implements might be subject to this vulnerability:

LSP0ERC725AccountCore and
LSP6KeyManagerCore

The above is extracted from ECDSA signature malleability with more details.
@YamenMerhi tried helping but still having some issues and no chance to deeply look into it.

Thanks for the contribution! Will do a PR 🚀


 We were being too clever about weak
references and so the observed object could get deallocated before the
observer. Ideally, we'd just nuke the listener in that case, but we
can't trigger an event just before the observed object gets
deallocated (at least without swizzling, which seems excessive).
So instead, we update the pattern to explicitly retain the observer.
This is typically how lifetimes work anyway, so it shouldn't be an
issue.
Also uses a new way to get a unique id for the association with better guarantees about addresses not changing.
Unfortunately, I couldn't actually reproduce the related crash, but it's
clear from the stack trace that this is the underlying issue. The test I
added crashes on the old implementation.
JIRA: https://openedx.atlassian.net/browse/MA-2136

PR Cover Letter

[ ] A detailed description of the changes in the body of the PR
[ ] If available, a link to the relevant JIRA ticket in the body of the PR

[PROJ-XXXX](https://openedx.atlassian.net/browse/PROJ-XXXX)

Sandbox

[ ] Build a sandbox for your branch and add a link here

Testing

[ ] i18n
[ ] RTL
[ ] Unit, integration, acceptance tests as appropriate
[ ] Analytics
[ ] Performance
[ ] Database migrations are backwards-compatible

Reviewers
If you've been tagged for review, please check your corresponding box once you've given the :+1:.

[ ] Code review #1
[ ] Code review #2
[ ] Docs review (required for UI strings/error messages)
[ ] UX review
[ ] Accessibility review
[ ] Product review

Documentation considerations

[ ] Are there user visible changes? If so, list who is affected, and include a pointer to the relevant code.
[ ] Should this be mentioned in the release notes? If so, make sure that you included a description.

DevOps assistance
If it's needed, or even if you suspect it might be needed, do any/all these things as early as possible. Don't surprise devops right before merging! For example, touching envs/aws.py or or envs/common.py are dead giveaways that you'll need assistance when pushing your changes out. Things that require devops attention include, but are not limited to:

configuration changes
feature flags
migrations
Ways to get devops attention, pick-and-choose whichever is most appropriate for your specific PR:
[ ] Tag devops on your PR (@edx/devops), either for review (more specific to devops = better) or just a general FYI
[ ] File a support ticket

FYI: Tag anyone who might be interested in this PR here.
Post-review

[ ] Squash commits

@mikekatz please review

Current coverage is 48.33%

Merging #685 into master will increase coverage by +0.01% as of 96486ef

@@            master    #685   diff @@
======================================
  Files          424     424
  Stmts        28166   28177    +11
  Branches         0       0
  Methods          0       0
======================================
+ Hit          13612   13619     +7
  Partial          0       0
- Missed       14554   14558     +4

Review entire Coverage Diff as of 96486ef

Powered by Codecov. Updated on successful CI builds.

yikes. are you sure this isn't still too clever? :+1:

It's probably still too clever, but otherwise KVO is a morass of bad API decisions, so at least with this we only have to get it right once?


K8SPS-158: delete-ssl finalizer

Problem:
We delete issuer and certificates, but the secret is left and if you re-create the cluster the cert manager will not issue new certificates.
Cause:
When PS is deleted, we delete CM resources (issuers and certificates) with it, but not the secrets. After a restart, since the secrets are present we will use those, but will not create new issuers and certificates.
Solution:
Add delete-ssl finalizer which will remove ssl secrets and CM resources, but without it we don't delete anything.
Note: This PR handles https://jira.percona.com/browse/K8SPS-170 as well.
CHECKLIST
Jira

[x] Is the Jira ticket created and referenced properly?
[x] Does the Jira ticket have the proper statuses for documentation (Needs Doc) and QA (Needs QA)?
[x] Does the Jira ticket link to the proper milestone (Fix Version field)?

Tests

[x] Is an E2E test/test case added for the new feature/change?
[x] Are unit tests added where appropriate?

Config/Logging/Testability

[x] Are all needed new/changed options added to default YAML files?
[x] Are the manifests (crd/bundle) regenerated if needed?
[x] Did we add proper logging messages for operator actions?
[x] Did we ensure compatibility with the previous version or cluster upgrade process?
[x] Does the change support oldest and newest supported PS version?
[x] Does the change support oldest and newest supported Kubernetes version?

@inelpandzic please fix it https://github.com/percona/percona-server-mysql-operator/pull/314/checks?check_run_id=11962430310 or we should ignore testdata dir

Yeah, I'll ignore it.

@inelpandzic please fix it https://github.com/percona/percona-server-mysql-operator/pull/314/checks?check_run_id=11962430310 or we should ignore testdata dir

Yeah, I'll ignore it.

Or no, I just fixed the spelling.

Test name
Status

async-ignore-annotations
passed

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
passed

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/1d6007d0211b51a6b2790cf6fd10f4993a24b9a9
image: perconalab/percona-server-mysql-operator:PR-314-1d6007d

Test name
Status

async-ignore-annotations
passed

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
failure

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/ff4f892af6bda9bea2a830b17e6e809302a4e249
image: perconalab/percona-server-mysql-operator:PR-314-ff4f892

Test name
Status

async-ignore-annotations
passed

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
passed

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/a6b8c3b1fc0a6f004792b9f5a2bac729a4e2b4ef
image: perconalab/percona-server-mysql-operator:PR-314-a6b8c3b

Test name
Status

async-ignore-annotations
passed

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
passed

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/726ca034feeb93047a0bf7c6e2b69344dcd6094c
image: perconalab/percona-server-mysql-operator:PR-314-726ca03

Test name
Status

async-ignore-annotations
failure

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
failure

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/efde2b796679e63137eb2431b762929912a33a3b
image: perconalab/percona-server-mysql-operator:PR-314-efde2b7

Test name
Status

async-ignore-annotations
failure

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
passed

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/efde2b796679e63137eb2431b762929912a33a3b
image: perconalab/percona-server-mysql-operator:PR-314-efde2b7

Test name
Status

async-ignore-annotations
failure

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
passed

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/efde2b796679e63137eb2431b762929912a33a3b
image: perconalab/percona-server-mysql-operator:PR-314-efde2b7

Test name
Status

async-ignore-annotations
failure

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
passed

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/efde2b796679e63137eb2431b762929912a33a3b
image: perconalab/percona-server-mysql-operator:PR-314-efde2b7

Test name
Status

async-ignore-annotations
failure

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
passed

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/efde2b796679e63137eb2431b762929912a33a3b
image: perconalab/percona-server-mysql-operator:PR-314-efde2b7

Test name
Status

async-ignore-annotations
passed

auto-config
passed

config
passed

config-router
passed

demand-backup
passed

gr-demand-backup
passed

gr-ignore-annotations
passed

gr-init-deploy
passed

gr-scaling
passed

gr-tls-cert-manager
passed

haproxy
passed

init-deploy
passed

limits
passed

monitoring
passed

one-pod
passed

scaling
passed

semi-sync
passed

service-per-pod
passed

sidecars
passed

tls-cert-manager
passed

users
passed

version-service
passed

We run 22 out of 22

commit: https://github.com/percona/percona-server-mysql-operator/pull/314/commits/efde2b796679e63137eb2431b762929912a33a3b
image: perconalab/percona-server-mysql-operator:PR-314-efde2b7


Create .goreleaser.yml

I'll close this PR as stale. I'll take a look at the branch though as it looks like it relates to #74.