TASK [Create a file to store CSP name] *****************************************
2019-11-20T05:21:41.776199 (delta: 1.895444)         elapsed: 43.109247 *******
changed: [<IP_ADDRESS>] => (item={'stderr_lines': [], u'cmd': u"kubectl get csp -l openebs.io/storage-pool-claim=cstor-block-disk-pool -o jsonpath='{.items[*].metadata.name}'", u'end': u'2019-11-20 05:21:41.662249', u'stdout': u'cstor-block-disk-pool-7dpf cstor-block-disk-pool-bj2k cstor-block-disk-pool-f8hm cstor-block-disk-pool-i9vf cstor-block-disk-pool-lp8w', u'changed': True, 'failed': False, u'delta': u'0:00:01.403937', u'stderr': u'', u'rc': 0, 'stdout_lines': [u'cstor-block-disk-pool-7dpf cstor-block-disk-pool-bj2k cstor-block-disk-pool-f8hm cstor-block-disk-pool-i9vf cstor-block-disk-pool-lp8w'], u'start': u'2019-11-20 05:21:40.258312'}) => {"backup": "", "changed": true, "item": {"changed": true, "cmd": "kubectl get csp -l openebs.io/storage-pool-claim=cstor-block-disk-pool -o jsonpath='{.items[*].metadata.name}'", "delta": "0:00:01.403937", "end": "2019-11-20 05:21:41.662249", "failed": false, "rc": 0, "start": "2019-11-20 05:21:40.258312", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-7dpf cstor-block-disk-pool-bj2k cstor-block-disk-pool-f8hm cstor-block-disk-pool-i9vf cstor-block-disk-pool-lp8w", "stdout_lines": ["cstor-block-disk-pool-7dpf cstor-block-disk-pool-bj2k cstor-block-disk-pool-f8hm cstor-block-disk-pool-i9vf cstor-block-disk-pool-lp8w"]}, "msg": "line added and ownership, perms or SE linux context changed"}

TASK [Getting the list of csp on which cvr is created] *************************
2019-11-20T05:21:42.535732 (delta: 0.759387)         elapsed: 43.86878 ********
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get cvr -n openebs -l cstorvolume.openebs.io/name=pvc-3dd89244-a004-423f-b540-f6f7330720ea -o jsonpath='{.items[*].metadata.labels.cstorpool\\.openebs\\.io/name}'", "delta": "0:00:01.439173", "end": "2019-11-20 05:21:44.376170", "rc": 0, "start": "2019-11-20 05:21:42.936997", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-bj2k cstor-block-disk-pool-lp8w", "stdout_lines": ["cstor-block-disk-pool-bj2k cstor-block-disk-pool-lp8w"]}

TASK [Create a file to store CSP name] *****************************************
2019-11-20T05:21:44.475813 (delta: 1.939938)         elapsed: 45.808861 *******
changed: [<IP_ADDRESS>] => (item={'stderr_lines': [], u'cmd': u"kubectl get cvr -n openebs -l cstorvolume.openebs.io/name=pvc-3dd89244-a004-423f-b540-f6f7330720ea -o jsonpath='{.items[*].metadata.labels.cstorpool\\.openebs\\.io/name}'", u'end': u'2019-11-20 05:21:44.376170', u'stdout': u'cstor-block-disk-pool-bj2k cstor-block-disk-pool-lp8w', u'changed': True, 'failed': False, u'delta': u'0:00:01.439173', u'stderr': u'', u'rc': 0, 'stdout_lines': [u'cstor-block-disk-pool-bj2k cstor-block-disk-pool-lp8w'], u'start': u'2019-11-20 05:21:42.936997'}) => {"backup": "", "changed": true, "item": {"changed": true, "cmd": "kubectl get cvr -n openebs -l cstorvolume.openebs.io/name=pvc-3dd89244-a004-423f-b540-f6f7330720ea -o jsonpath='{.items[*].metadata.labels.cstorpool\\.openebs\\.io/name}'", "delta": "0:00:01.439173", "end": "2019-11-20 05:21:44.376170", "failed": false, "rc": 0, "start": "2019-11-20 05:21:42.936997", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-bj2k cstor-block-disk-pool-lp8w", "stdout_lines": ["cstor-block-disk-pool-bj2k cstor-block-disk-pool-lp8w"]}, "msg": "line added and ownership, perms or SE linux context changed"}

TASK [Getting the csp where no cvr is created] *********************************
2019-11-20T05:21:44.942465 (delta: 0.466555)         elapsed: 46.275513 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "cat csp_all.tmp | tr \" \" \"\\n\" > csp_all\n cat csp_cvr.tmp | tr \" \" \"\\n\" > csp_cvr\n comm -3 <(sort csp_all) <(sort csp_cvr)", "delta": "0:00:01.241092", "end": "2019-11-20 05:21:46.499992", "rc": 0, "start": "2019-11-20 05:21:45.258900", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-7dpf\ncstor-block-disk-pool-f8hm\ncstor-block-disk-pool-i9vf", "stdout_lines": ["cstor-block-disk-pool-7dpf", "cstor-block-disk-pool-f8hm", "cstor-block-disk-pool-i9vf"]}

TASK [Getting the target IP from cvr] ******************************************
2019-11-20T05:21:46.583597 (delta: 1.641017)         elapsed: 47.916645 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get cvr -n openebs -l cstorvolume.openebs.io/name=pvc-3dd89244-a004-423f-b540-f6f7330720ea -o jsonpath='{.items[0].spec.targetIP}'", "delta": "0:00:01.675442", "end": "2019-11-20 05:21:48.601002", "rc": 0, "start": "2019-11-20 05:21:46.925560", "stderr": "", "stderr_lines": [], "stdout": "<IP_ADDRESS>", "stdout_lines": ["<IP_ADDRESS>"]}

TASK [Getting the Node name from CSP where CVR need to create] *****************
2019-11-20T05:21:48.724661 (delta: 2.140966)         elapsed: 50.057709 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get csp cstor-block-disk-pool-7dpf -o jsonpath='{.metadata.labels.kubernetes\\.io/hostname}'", "delta": "0:00:01.407643", "end": "2019-11-20 05:21:50.501603", "rc": 0, "start": "2019-11-20 05:21:49.093960", "stderr": "", "stderr_lines": [], "stdout": "d2iq-node3.mayalabs.io", "stdout_lines": ["d2iq-node3.mayalabs.io"]}

TASK [Getting csp uid] *********************************************************
2019-11-20T05:21:50.619801 (delta: 1.895018)         elapsed: 51.952849 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get csp cstor-block-disk-pool-7dpf -o jsonpath='{.metadata.uid}'", "delta": "0:00:01.388248", "end": "2019-11-20 05:21:52.513732", "rc": 0, "start": "2019-11-20 05:21:51.125484", "stderr": "", "stderr_lines": [], "stdout": "4673d316-9165-462c-a9f8-e7f750cb1b36", "stdout_lines": ["4673d316-9165-462c-a9f8-e7f750cb1b36"]}

TASK [Replacing the storage class in cvr yml] **********************************
2019-11-20T05:21:52.614459 (delta: 1.994503)         elapsed: 53.947507 *******
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "1 replacements made"}

TASK [Replacing the Node name in cvr yml] **************************************
2019-11-20T05:21:53.452644 (delta: 0.83806)         elapsed: 54.785692 ********
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "1 replacements made"}

TASK [Replacing the CSP name in cvr yml] ***************************************
2019-11-20T05:21:53.859874 (delta: 0.407116)         elapsed: 55.192922 *******
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "2 replacements made"}

TASK [Replacing the uid in cvr yml] ********************************************
2019-11-20T05:21:54.361671 (delta: 0.501655)         elapsed: 55.694719 *******
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "1 replacements made"}

TASK [Replacing the volume name in cvr yml] ************************************
2019-11-20T05:21:54.719184 (delta: 0.357418)         elapsed: 56.052232 *******
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "3 replacements made"}

TASK [Replacing the volume capacity in cvr yml] ********************************
2019-11-20T05:21:55.181025 (delta: 0.461718)         elapsed: 56.514073 *******
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "1 replacements made"}

TASK [Replacing OpenEBS version in cvr yml] ************************************
2019-11-20T05:21:55.542115 (delta: 0.360973)         elapsed: 56.875163 *******
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "2 replacements made"}

TASK [Replacing the target IP in cvr yml] **************************************
2019-11-20T05:21:56.126786 (delta: 0.584582)         elapsed: 57.459834 *******
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "1 replacements made"}

TASK [Replacing the DRF in patch_cstorvolume yml] ******************************
2019-11-20T05:21:56.593146 (delta: 0.466193)         elapsed: 57.926194 *******
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "1 replacements made"}

TASK [Patching the cstorvolume's desired rep count] ****************************
2019-11-20T05:21:56.974391 (delta: 0.38114)         elapsed: 58.307439 ********
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl patch cstorvolume pvc-3dd89244-a004-423f-b540-f6f7330720ea -n openebs --type merge --patch \"$(cat patch_cstorvolume.yml)\"", "delta": "0:00:01.406789", "end": "2019-11-20 05:21:58.706683", "failed_when_result": false, "rc": 0, "start": "2019-11-20 05:21:57.299894", "stderr": "", "stderr_lines": [], "stdout": "cstorvolume.openebs.io/pvc-3dd89244-a004-423f-b540-f6f7330720ea patched", "stdout_lines": ["cstorvolume.openebs.io/pvc-3dd89244-a004-423f-b540-f6f7330720ea patched"]}

TASK [Creating new cvr] ********************************************************
2019-11-20T05:21:58.822717 (delta: 1.848209)         elapsed: 60.155765 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl apply -f cvr.yml", "delta": "0:00:02.002006", "end": "2019-11-20 05:22:01.162769", "rc": 0, "start": "2019-11-20 05:21:59.160763", "stderr": "", "stderr_lines": [], "stdout": "cstorvolumereplica.openebs.io/pvc-3dd89244-a004-423f-b540-f6f7330720ea-cstor-block-disk-pool-7dpf created", "stdout_lines": ["cstorvolumereplica.openebs.io/pvc-3dd89244-a004-423f-b540-f6f7330720ea-cstor-block-disk-pool-7dpf created"]}

TASK [Getting the uid of newly created cvr] ************************************
2019-11-20T05:22:01.252655 (delta: 2.429818)         elapsed: 62.585703 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get cvr -n openebs -l cstorpool.openebs.io/name=cstor-block-disk-pool-7dpf,cstorvolume.openebs.io/name=pvc-3dd89244-a004-423f-b540-f6f7330720ea -o jsonpath='{.items[0].metadata.uid}'", "delta": "0:00:01.530506", "end": "2019-11-20 05:22:03.079023", "rc": 0, "start": "2019-11-20 05:22:01.548517", "stderr": "", "stderr_lines": [], "stdout": "5b88274f-ef59-467d-b144-ae49055f6b87", "stdout_lines": ["5b88274f-ef59-467d-b144-ae49055f6b87"]}

TASK [Creating MD5SUM of new-uid] **********************************************
2019-11-20T05:22:03.199163 (delta: 1.946427)         elapsed: 64.532211 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "echo 5b88274f-ef59-467d-b144-ae49055f6b87 | md5sum | awk '{print toupper($1)}'", "delta": "0:00:01.274480", "end": "2019-11-20 05:22:04.831112", "rc": 0, "start": "2019-11-20 05:22:03.556632", "stderr": "", "stderr_lines": [], "stdout": "7A2AC5370C7F5C609621DFDAE56D98E9", "stdout_lines": ["7A2AC5370C7F5C609621DFDAE56D98E9"]}

TASK [Replacing the md5sum value in patch yml] *********************************
2019-11-20T05:22:04.931171 (delta: 1.731913)         elapsed: 66.264219 *******
changed: [<IP_ADDRESS>] => {"changed": true, "msg": "1 replacements made"}

TASK [Patching the cvr's replica id with md5sum] *******************************
2019-11-20T05:22:05.347302 (delta: 0.416012)         elapsed: 66.68035 ********
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl patch cvr pvc-3dd89244-a004-423f-b540-f6f7330720ea-cstor-block-disk-pool-7dpf -n openebs --type merge --patch \"$(cat patch_cvr.yml)\"", "delta": "0:00:01.611881", "end": "2019-11-20 05:22:07.222540", "failed_when_result": false, "rc": 0, "start": "2019-11-20 05:22:05.610659", "stderr": "", "stderr_lines": [], "stdout": "cstorvolumereplica.openebs.io/pvc-3dd89244-a004-423f-b540-f6f7330720ea-cstor-block-disk-pool-7dpf patched", "stdout_lines": ["cstorvolumereplica.openebs.io/pvc-3dd89244-a004-423f-b540-f6f7330720ea-cstor-block-disk-pool-7dpf patched"]}

TASK [Getting the cstor pool pod name] *****************************************
2019-11-20T05:22:07.370297 (delta: 2.022899)         elapsed: 68.703345 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get pod -n openebs -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-7dpf -o jsonpath='{.items[0].metadata.name}'", "delta": "0:00:01.464296", "end": "2019-11-20 05:22:09.244158", "rc": 0, "start": "2019-11-20 05:22:07.779862", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-7dpf-695bf4d66b-x9swp", "stdout_lines": ["cstor-block-disk-pool-7dpf-695bf4d66b-x9swp"]}

TASK [Restarting the cstor-pool-mgmt container] ********************************
2019-11-20T05:22:09.354840 (delta: 1.984359)         elapsed: 70.687888 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-7dpf-695bf4d66b-x9swp -n openebs -c cstor-pool-mgmt -- pkill -f /usr/local/bin/", "delta": "0:00:02.268609", "end": "2019-11-20 05:22:11.954298", "rc": 0, "start": "2019-11-20 05:22:09.685689", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [Checking for the pool pod to be in running state] ************************
2019-11-20T05:22:12.058967 (delta: 2.704026)         elapsed: 73.392015 *******
changed: [<IP_ADDRESS>] => {"attempts": 1, "changed": true, "cmd": "kubectl get pod -n openebs -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-7dpf -o jsonpath='{.items[0].status.phase}'", "delta": "0:00:01.515743", "end": "2019-11-20 05:22:13.934945", "rc": 0, "start": "2019-11-20 05:22:12.419202", "stderr": "", "stderr_lines": [], "stdout": "Running", "stdout_lines": ["Running"]}

TASK [Verify newly Created replicaID exists in cstorVolume or not] *************
2019-11-20T05:22:14.106643 (delta: 2.047576)         elapsed: 75.439691 *******
FAILED - RETRYING: Verify newly Created replicaID exists in cstorVolume or not (100 retries left).
FAILED - RETRYING: Verify newly Created replicaID exists in cstorVolume or not (99 retries left).
FAILED - RETRYING: Verify newly Created replicaID exists in cstorVolume or not (98 retries left).
FAILED - RETRYING: Verify newly Created replicaID exists in cstorVolume or not (97 retries left).
FAILED - RETRYING: Verify newly Created replicaID exists in cstorVolume or not (96 retries left).
FAILED - RETRYING: Verify newly Created replicaID exists in cstorVolume or not (95 retries left).
FAILED - RETRYING: Verify newly Created replicaID exists in cstorVolume or not (94 retries left).
FAILED - RETRYING: Verify newly Created replicaID exists in cstorVolume or not (93 retries left).
FAILED - RETRYING: Verify newly Created replicaID exists in cstorVolume or not (92 retries left).
changed: [<IP_ADDRESS>] => {"attempts": 10, "changed": true, "cmd": "kubectl get cstorvolume pvc-3dd89244-a004-423f-b540-f6f7330720ea -n openebs -o jsonpath='{.status.replicaDetails.knownReplicas.7A2AC5370C7F5C609621DFDAE56D98E9}'", "delta": "0:00:01.407836", "end": "2019-11-20 05:24:02.337336", "rc": 0, "start": "2019-11-20 05:24:00.929500", "stderr": "", "stderr_lines": [], "stdout": "7431218314525885559", "stdout_lines": ["7431218314525885559"]}

TASK [Checking for the desired replication factor in cstorvolume] **************
2019-11-20T05:24:02.481852 (delta: 108.375056)         elapsed: 183.8149 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get cstorvolume pvc-3dd89244-a004-423f-b540-f6f7330720ea -n openebs -o jsonpath='{.spec.replicationFactor}'", "delta": "0:00:01.397852", "end": "2019-11-20 05:24:04.190253", "failed_when_result": false, "rc": 0, "start": "2019-11-20 05:24:02.792401", "stderr": "", "stderr_lines": [], "stdout": "3", "stdout_lines": ["3"]}

TASK [Checking for the consistancy factor in cstorvolume] **********************
2019-11-20T05:24:04.331582 (delta: 1.849587)         elapsed: 185.66463 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get cstorvolume pvc-3dd89244-a004-423f-b540-f6f7330720ea -n openebs -o jsonpath='{.spec.consistencyFactor}'", "delta": "0:00:01.397310", "end": "2019-11-20 05:24:06.114013", "failed_when_result": false, "rc": 0, "start": "2019-11-20 05:24:04.716703", "stderr": "", "stderr_lines": [], "stdout": "2", "stdout_lines": ["2"]}

TASK [Verify cvr health status] ************************************************
2019-11-20T05:24:06.272094 (delta: 1.940401)         elapsed: 187.605142 ******
FAILED - RETRYING: Verify cvr health status (100 retries left).
changed: [<IP_ADDRESS>] => {"attempts": 2, "changed": true, "cmd": "kubectl get cvr -n openebs -l openebs.io/persistent-volume=pvc-3dd89244-a004-423f-b540-f6f7330720ea -o jsonpath='{.items[*].status.phase}'", "delta": "0:00:01.427058", "end": "2019-11-20 05:24:19.940822", "rc": 0, "start": "2019-11-20 05:24:18.513764", "stderr": "", "stderr_lines": [], "stdout": "Healthy Healthy Healthy", "stdout_lines": ["Healthy Healthy Healthy"]}

TASK [Get istgt target pod details] ********************************************
2019-11-20T05:24:20.072516 (delta: 13.8003)         elapsed: 201.405564 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get pods -l openebs.io/persistent-volume=pvc-3dd89244-a004-423f-b540-f6f7330720ea -n openebs -o custom-columns=:metadata.name --no-headers", "delta": "0:00:01.435288", "end": "2019-11-20 05:24:21.881173", "failed_when_result": false, "rc": 0, "start": "2019-11-20 05:24:20.445885", "stderr": "", "stderr_lines": [], "stdout": "pvc-3dd89244-a004-423f-b540-f6f7330720ea-target-6c8d78c779bnfld", "stdout_lines": ["pvc-3dd89244-a004-423f-b540-f6f7330720ea-target-6c8d78c779bnfld"]}

TASK [Create an empty list of replica pods.] ***********************************
2019-11-20T05:24:22.031218 (delta: 1.958576)         elapsed: 203.364266 ******
ok: [<IP_ADDRESS>] => {"ansible_facts": {"cstor_replicas_pod": []}, "changed": false}

TASK [Obtaining the pool deployments from cvr] *********************************
2019-11-20T05:24:22.230390 (delta: 0.199062)         elapsed: 203.563438 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get cvr -n openebs -l openebs.io/persistent-volume=pvc-3dd89244-a004-423f-b540-f6f7330720ea --no-headers -o=jsonpath='{range .items[*]}{.metadata.labels.cstorpool\\.openebs\\.io\\/name}{\"\\n\"}{end}'", "delta": "0:00:01.431352", "end": "2019-11-20 05:24:24.075560", "failed_when_result": false, "rc": 0, "start": "2019-11-20 05:24:22.644208", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-7dpf\ncstor-block-disk-pool-bj2k\ncstor-block-disk-pool-lp8w", "stdout_lines": ["cstor-block-disk-pool-7dpf", "cstor-block-disk-pool-bj2k", "cstor-block-disk-pool-lp8w"]}

TASK [Obtaining the pool pods] *************************************************
2019-11-20T05:24:24.229707 (delta: 1.999173)         elapsed: 205.562755 ******
changed: [<IP_ADDRESS>] => (item=cstor-block-disk-pool-7dpf) => {"changed": true, "cmd": "kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-7dpf -n openebs --no-headers -o custom-columns=:.metadata.name", "delta": "0:00:01.427703", "end": "2019-11-20 05:24:25.973666", "failed_when_result": false, "item": "cstor-block-disk-pool-7dpf", "rc": 0, "start": "2019-11-20 05:24:24.545963", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-7dpf-695bf4d66b-x9swp", "stdout_lines": ["cstor-block-disk-pool-7dpf-695bf4d66b-x9swp"]}
changed: [<IP_ADDRESS>] => (item=cstor-block-disk-pool-bj2k) => {"changed": true, "cmd": "kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-bj2k -n openebs --no-headers -o custom-columns=:.metadata.name", "delta": "0:00:01.466935", "end": "2019-11-20 05:24:27.754574", "failed_when_result": false, "item": "cstor-block-disk-pool-bj2k", "rc": 0, "start": "2019-11-20 05:24:26.287639", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn", "stdout_lines": ["cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn"]}
changed: [<IP_ADDRESS>] => (item=cstor-block-disk-pool-lp8w) => {"changed": true, "cmd": "kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-lp8w -n openebs --no-headers -o custom-columns=:.metadata.name", "delta": "0:00:01.536623", "end": "2019-11-20 05:24:29.769655", "failed_when_result": false, "item": "cstor-block-disk-pool-lp8w", "rc": 0, "start": "2019-11-20 05:24:28.233032", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-lp8w-89df46b4c-mwcxx", "stdout_lines": ["cstor-block-disk-pool-lp8w-89df46b4c-mwcxx"]}

TASK [Build a list of replica pods] ********************************************
2019-11-20T05:24:29.879498 (delta: 5.64968)         elapsed: 211.212546 *******
ok: [<IP_ADDRESS>] => (item={'_ansible_parsed': True, 'stderr_lines': [], u'changed': True, u'stdout': u'cstor-block-disk-pool-7dpf-695bf4d66b-x9swp', '_ansible_item_result': True, u'delta': u'0:00:01.427703', 'stdout_lines': [u'cstor-block-disk-pool-7dpf-695bf4d66b-x9swp'], 'failed_when_result': False, '_ansible_item_label': u'cstor-block-disk-pool-7dpf', u'end': u'2019-11-20 05:24:25.973666', '_ansible_no_log': False, 'failed': False, u'cmd': u'kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-7dpf -n openebs --no-headers -o custom-columns=:.metadata.name', 'item': u'cstor-block-disk-pool-7dpf', u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'_raw_params': u'kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-7dpf -n openebs --no-headers -o custom-columns=:.metadata.name', u'removes': None, u'argv': None, u'creates': None, u'chdir': None, u'stdin': None}}, u'start': u'2019-11-20 05:24:24.545963', '_ansible_ignore_errors': None}) => {"ansible_facts": {"cstor_replicas_pod": ["cstor-block-disk-pool-7dpf-695bf4d66b-x9swp"]}, "changed": false, "item": {"changed": true, "cmd": "kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-7dpf -n openebs --no-headers -o custom-columns=:.metadata.name", "delta": "0:00:01.427703", "end": "2019-11-20 05:24:25.973666", "failed": false, "failed_when_result": false, "invocation": {"module_args": {"_raw_params": "kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-7dpf -n openebs --no-headers -o custom-columns=:.metadata.name", "_uses_shell": true, "argv": null, "chdir": null, "creates": null, "executable": null, "removes": null, "stdin": null, "warn": true}}, "item": "cstor-block-disk-pool-7dpf", "rc": 0, "start": "2019-11-20 05:24:24.545963", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-7dpf-695bf4d66b-x9swp", "stdout_lines": ["cstor-block-disk-pool-7dpf-695bf4d66b-x9swp"]}}
ok: [<IP_ADDRESS>] => (item={'_ansible_parsed': True, 'stderr_lines': [], u'changed': True, u'stdout': u'cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn', '_ansible_item_result': True, u'delta': u'0:00:01.466935', 'stdout_lines': [u'cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn'], 'failed_when_result': False, '_ansible_item_label': u'cstor-block-disk-pool-bj2k', u'end': u'2019-11-20 05:24:27.754574', '_ansible_no_log': False, 'failed': False, u'cmd': u'kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-bj2k -n openebs --no-headers -o custom-columns=:.metadata.name', 'item': u'cstor-block-disk-pool-bj2k', u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'_raw_params': u'kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-bj2k -n openebs --no-headers -o custom-columns=:.metadata.name', u'removes': None, u'argv': None, u'creates': None, u'chdir': None, u'stdin': None}}, u'start': u'2019-11-20 05:24:26.287639', '_ansible_ignore_errors': None}) => {"ansible_facts": {"cstor_replicas_pod": ["cstor-block-disk-pool-7dpf-695bf4d66b-x9swp", "cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn"]}, "changed": false, "item": {"changed": true, "cmd": "kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-bj2k -n openebs --no-headers -o custom-columns=:.metadata.name", "delta": "0:00:01.466935", "end": "2019-11-20 05:24:27.754574", "failed": false, "failed_when_result": false, "invocation": {"module_args": {"_raw_params": "kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-bj2k -n openebs --no-headers -o custom-columns=:.metadata.name", "_uses_shell": true, "argv": null, "chdir": null, "creates": null, "executable": null, "removes": null, "stdin": null, "warn": true}}, "item": "cstor-block-disk-pool-bj2k", "rc": 0, "start": "2019-11-20 05:24:26.287639", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn", "stdout_lines": ["cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn"]}}
ok: [<IP_ADDRESS>] => (item={'_ansible_parsed': True, 'stderr_lines': [], u'changed': True, u'stdout': u'cstor-block-disk-pool-lp8w-89df46b4c-mwcxx', '_ansible_item_result': True, u'delta': u'0:00:01.536623', 'stdout_lines': [u'cstor-block-disk-pool-lp8w-89df46b4c-mwcxx'], 'failed_when_result': False, '_ansible_item_label': u'cstor-block-disk-pool-lp8w', u'end': u'2019-11-20 05:24:29.769655', '_ansible_no_log': False, 'failed': False, u'cmd': u'kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-lp8w -n openebs --no-headers -o custom-columns=:.metadata.name', 'item': u'cstor-block-disk-pool-lp8w', u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'_raw_params': u'kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-lp8w -n openebs --no-headers -o custom-columns=:.metadata.name', u'removes': None, u'argv': None, u'creates': None, u'chdir': None, u'stdin': None}}, u'start': u'2019-11-20 05:24:28.233032', '_ansible_ignore_errors': None}) => {"ansible_facts": {"cstor_replicas_pod": ["cstor-block-disk-pool-7dpf-695bf4d66b-x9swp", "cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn", "cstor-block-disk-pool-lp8w-89df46b4c-mwcxx"]}, "changed": false, "item": {"changed": true, "cmd": "kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-lp8w -n openebs --no-headers -o custom-columns=:.metadata.name", "delta": "0:00:01.536623", "end": "2019-11-20 05:24:29.769655", "failed": false, "failed_when_result": false, "invocation": {"module_args": {"_raw_params": "kubectl get pod -l app=cstor-pool,openebs.io/cstor-pool=cstor-block-disk-pool-lp8w -n openebs --no-headers -o custom-columns=:.metadata.name", "_uses_shell": true, "argv": null, "chdir": null, "creates": null, "executable": null, "removes": null, "stdin": null, "warn": true}}, "item": "cstor-block-disk-pool-lp8w", "rc": 0, "start": "2019-11-20 05:24:28.233032", "stderr": "", "stderr_lines": [], "stdout": "cstor-block-disk-pool-lp8w-89df46b4c-mwcxx", "stdout_lines": ["cstor-block-disk-pool-lp8w-89df46b4c-mwcxx"]}}

TASK [Generate snapshot name] **************************************************
2019-11-20T05:24:30.097007 (delta: 0.217405)         elapsed: 211.430055 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1", "delta": "0:00:01.293798", "end": "2019-11-20 05:24:31.702284", "rc": 0, "start": "2019-11-20 05:24:30.408486", "stderr": "", "stderr_lines": [], "stdout": "25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg", "stdout_lines": ["25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg"]}

TASK [set_fact] ****************************************************************
2019-11-20T05:24:31.803482 (delta: 1.706383)         elapsed: 213.13653 *******
ok: [<IP_ADDRESS>] => {"ansible_facts": {"snap_name": "25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg"}, "changed": false}

TASK [Take snapshot for data verification] *************************************
2019-11-20T05:24:31.953712 (delta: 0.150135)         elapsed: 213.28676 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it pvc-3dd89244-a004-423f-b540-f6f7330720ea-target-6c8d78c779bnfld -n openebs --container cstor-istgt -- istgtcontrol snapcreate pvc-3dd89244-a004-423f-b540-f6f7330720ea 25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg 30 30", "delta": "0:00:03.185191", "end": "2019-11-20 05:24:35.461387", "rc": 0, "start": "2019-11-20 05:24:32.276196", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "DONE SNAPCREATE command", "stdout_lines": ["DONE SNAPCREATE command"]}

TASK [Install netcat on current host] ******************************************
2019-11-20T05:24:35.672120 (delta: 3.718265)         elapsed: 217.005168 ******
 [WARNING]: Consider using the apt module rather than running apt-get.  If you
need to use command because apt is insufficient you can add warn=False to this
command task or set command_warnings=False in ansible.cfg to get rid of this
message.
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "apt-get update && apt-get -y install netcat iproute2", "delta": "0:00:14.161410", "end": "2019-11-20 05:24:50.154282", "rc": 0, "start": "2019-11-20 05:24:35.992872", "stderr": "", "stderr_lines": [], "stdout": "Hit:1 http://archive.ubuntu.com/ubuntu bionic InRelease\nGet:2 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\nGet:3 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\nGet:4 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\nGet:5 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [29.9 kB]\nGet:6 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [786 kB]\nGet:7 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [1312 kB]\nGet:8 http://archive.ubuntu.com/ubuntu bionic-updates/multiverse amd64 Packages [9578 B]\nGet:9 http://security.ubuntu.com/ubuntu bionic-security/restricted amd64 Packages [16.8 kB]\nGet:10 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [1031 kB]\nGet:11 http://security.ubuntu.com/ubuntu bionic-security/multiverse amd64 Packages [6508 B]\nGet:12 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [735 kB]\nGet:13 http://archive.ubuntu.com/ubuntu bionic-backports/universe amd64 Packages [4235 B]\nFetched 4184 kB in 7s (637 kB/s)\nReading package lists...\nReading package lists...\nBuilding dependency tree...\nReading state information...\niproute2 is already the newest version (4.15.0-2ubuntu1).\nnetcat is already the newest version (1.10-41.1).\n0 upgraded, 0 newly installed, 0 to remove and 37 not upgraded.", "stdout_lines": ["Hit:1 http://archive.ubuntu.com/ubuntu bionic InRelease", "Get:2 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]", "Get:3 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]", "Get:4 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]", "Get:5 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [29.9 kB]", "Get:6 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [786 kB]", "Get:7 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [1312 kB]", "Get:8 http://archive.ubuntu.com/ubuntu bionic-updates/multiverse amd64 Packages [9578 B]", "Get:9 http://security.ubuntu.com/ubuntu bionic-security/restricted amd64 Packages [16.8 kB]", "Get:10 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [1031 kB]", "Get:11 http://security.ubuntu.com/ubuntu bionic-security/multiverse amd64 Packages [6508 B]", "Get:12 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [735 kB]", "Get:13 http://archive.ubuntu.com/ubuntu bionic-backports/universe amd64 Packages [4235 B]", "Fetched 4184 kB in 7s (637 kB/s)", "Reading package lists...", "Reading package lists...", "Building dependency tree...", "Reading state information...", "iproute2 is already the newest version (4.15.0-2ubuntu1).", "netcat is already the newest version (1.10-41.1).", "0 upgraded, 0 newly installed, 0 to remove and 37 not upgraded."]}

TASK [include] *****************************************************************
2019-11-20T05:24:50.271740 (delta: 14.599436)         elapsed: 231.604788 *****
included: /chaoslib/openebs/fetch_data_from_replica.yml for <IP_ADDRESS>
included: /chaoslib/openebs/fetch_data_from_replica.yml for <IP_ADDRESS>
included: /chaoslib/openebs/fetch_data_from_replica.yml for <IP_ADDRESS>

TASK [Get replica id] **********************************************************
2019-11-20T05:24:50.598124 (delta: 0.326257)         elapsed: 231.931172 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get pods cstor-block-disk-pool-7dpf-695bf4d66b-x9swp -n openebs -o custom-columns=:.spec.containers[*].env[0].value --no-headers | cut -d \",\" -f1", "delta": "0:00:01.374224", "end": "2019-11-20 05:24:52.289196", "rc": 0, "start": "2019-11-20 05:24:50.914972", "stderr": "", "stderr_lines": [], "stdout": "4673d316-9165-462c-a9f8-e7f750cb1b36", "stdout_lines": ["4673d316-9165-462c-a9f8-e7f750cb1b36"]}

TASK [Check for dataset name] **************************************************
2019-11-20T05:24:52.413174 (delta: 1.81495)         elapsed: 233.746222 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-7dpf-695bf4d66b-x9swp -n openebs --container cstor-pool -- zfs list cstor-4673d316-9165-462c-a9f8-e7f750cb1b36/pvc-3dd89244-a004-423f-b540-f6f7330720ea", "delta": "0:00:01.906121", "end": "2019-11-20 05:24:54.778742", "rc": 0, "start": "2019-11-20 05:24:52.872621", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "NAME                                                                                USED  AVAIL  REFER  MOUNTPOINT\ncstor-4673d316-9165-462c-a9f8-e7f750cb1b36/pvc-3dd89244-a004-423f-b540-f6f7330720ea  27.0M  48.2G  25.8M  -", "stdout_lines": ["NAME                                                                                USED  AVAIL  REFER  MOUNTPOINT", "cstor-4673d316-9165-462c-a9f8-e7f750cb1b36/pvc-3dd89244-a004-423f-b540-f6f7330720ea  27.0M  48.2G  25.8M  -"]}

TASK [Dump snapshot datastream to a file] **************************************
2019-11-20T05:24:54.929960 (delta: 2.516659)         elapsed: 236.263008 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-7dpf-695bf4d66b-x9swp -n openebs --container cstor-pool -- bash -c \"zfs send cstor-4673d316-9165-462c-a9f8-e7f750cb1b36/pvc-3dd89244-a004-423f-b540-f6f7330720ea@25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg > /snap.data\"", "delta": "0:00:04.744683", "end": "2019-11-20 05:25:00.096143", "rc": 0, "start": "2019-11-20 05:24:55.351460", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [Remove any object dump files from replica pod] ***************************
2019-11-20T05:25:00.204391 (delta: 5.274282)         elapsed: 241.537439 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-7dpf-695bf4d66b-x9swp -n openebs --container cstor-pool -- rm -rf 1.dump 3.dump", "delta": "0:00:02.404902", "end": "2019-11-20 05:25:02.919463", "rc": 0, "start": "2019-11-20 05:25:00.514561", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [Parse snapshot stream and create dump files] *****************************
2019-11-20T05:25:03.042629 (delta: 2.83813)         elapsed: 244.375677 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-7dpf-695bf4d66b-x9swp -n openebs --container cstor-pool -- bash -c \"zstreamdump -w -1 < /snap.data\"", "delta": "0:00:03.259978", "end": "2019-11-20 05:25:06.703737", "rc": 0, "start": "2019-11-20 05:25:03.443759", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "BEGIN record\n\thdrtype = 1\n\tfeatures = 0\n\tmagic = 2f5bacbac\n\tcreation_time = 5dd4ce12\n\ttype = 3\n\tflags = 0x4\n\ttoguid = 6094ae8f50168186\n\tfromguid = 0\n\ttoname = cstor-4673d316-9165-462c-a9f8-e7f750cb1b36/pvc-3dd89244-a004-423f-b540-f6f7330720ea@25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg\nEND checksum = 5a7d6fc61f7eab/d97da72b83dfd2e5/8a621716c5943c28/7a1ba514516fcb86\nSUMMARY:\n\tTotal DRR_BEGIN records = 1\n\tTotal DRR_END records = 1\n\tTotal DRR_OBJECT records = 3\n\tTotal DRR_FREEOBJECTS records = 4\n\tTotal DRR_WRITE records = 45240\n\tTotal DRR_WRITE_BYREF records = 0\n\tTotal DRR_WRITE_EMBEDDED records = 0\n\tTotal DRR_FREE records = 1493\n\tTotal DRR_SPILL records = 0\n\tTotal records = 46742\n\tTotal write size = 185299456 (0xb0b7200)\n\tTotal stream length = 199882960 (0xbe9f8d0)", "stdout_lines": ["BEGIN record", "\thdrtype = 1", "\tfeatures = 0", "\tmagic = 2f5bacbac", "\tcreation_time = 5dd4ce12", "\ttype = 3", "\tflags = 0x4", "\ttoguid = 6094ae8f50168186", "\tfromguid = 0", "\ttoname = cstor-4673d316-9165-462c-a9f8-e7f750cb1b36/pvc-3dd89244-a004-423f-b540-f6f7330720ea@25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg", "END checksum = 5a7d6fc61f7eab/d97da72b83dfd2e5/8a621716c5943c28/7a1ba514516fcb86", "SUMMARY:", "\tTotal DRR_BEGIN records = 1", "\tTotal DRR_END records = 1", "\tTotal DRR_OBJECT records = 3", "\tTotal DRR_FREEOBJECTS records = 4", "\tTotal DRR_WRITE records = 45240", "\tTotal DRR_WRITE_BYREF records = 0", "\tTotal DRR_WRITE_EMBEDDED records = 0", "\tTotal DRR_FREE records = 1493", "\tTotal DRR_SPILL records = 0", "\tTotal records = 46742", "\tTotal write size = 185299456 (0xb0b7200)", "\tTotal stream length = 199882960 (0xbe9f8d0)"]}

TASK [Install netcat on replica pod] *******************************************
2019-11-20T05:25:06.836363 (delta: 3.793627)         elapsed: 248.169411 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-7dpf-695bf4d66b-x9swp -n openebs --container cstor-pool -- bash -c \"apt-get update && apt-get -y install netcat\"", "delta": "0:00:11.341763", "end": "2019-11-20 05:25:18.611655", "rc": 0, "start": "2019-11-20 05:25:07.269892", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file\nCouldn't create tempfiles for splitting up /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/security.ubuntu.com_ubuntu_dists_xenial-security_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-updates_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-backports_InReleaseW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://security.ubuntu.com/ubuntu xenial-security InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-updates InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-backports InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-updates/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-backports/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://security.ubuntu.com/ubuntu/dists/xenial-security/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Some index files failed to download. They have been ignored, or old ones used instead.\ndebconf: unable to initialize frontend: Dialog\ndebconf: (TERM is not set, so the dialog frontend is not usable.)\ndebconf: falling back to frontend: Readline\ndebconf: unable to initialize frontend: Readline\ndebconf: (This frontend requires a controlling tty.)\ndebconf: falling back to frontend: Teletype\ndpkg-preconfigure: unable to re-open stdin: ", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file", "Couldn't create tempfiles for splitting up /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/security.ubuntu.com_ubuntu_dists_xenial-security_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-updates_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-backports_InReleaseW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://security.ubuntu.com/ubuntu xenial-security InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-updates InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-backports InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-updates/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-backports/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://security.ubuntu.com/ubuntu/dists/xenial-security/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Some index files failed to download. They have been ignored, or old ones used instead.", "debconf: unable to initialize frontend: Dialog", "debconf: (TERM is not set, so the dialog frontend is not usable.)", "debconf: falling back to frontend: Readline", "debconf: unable to initialize frontend: Readline", "debconf: (This frontend requires a controlling tty.)", "debconf: falling back to frontend: Teletype", "dpkg-preconfigure: unable to re-open stdin: "], "stdout": "Get:1 http://security.ubuntu.com/ubuntu xenial-security InRelease [109 kB]\nHit:2 http://archive.ubuntu.com/ubuntu xenial InRelease\nErr:2 http://archive.ubuntu.com/ubuntu xenial InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nGet:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease [109 kB]\nErr:1 http://security.ubuntu.com/ubuntu xenial-security InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nErr:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nGet:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease [107 kB]\nErr:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nFetched 325 kB in 1s (178 kB/s)\nReading package lists...\nReading package lists...\nBuilding dependency tree...\nReading state information...\nThe following additional packages will be installed:\n  netcat-traditional\nThe following NEW packages will be installed:\n  netcat netcat-traditional\n0 upgraded, 2 newly installed, 0 to remove and 24 not upgraded.\nNeed to get 64.1 kB of archives.\nAfter this operation, 191 kB of additional disk space will be used.\nGet:1 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat-traditional amd64 1.10-41 [60.7 kB]\nGet:2 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat all 1.10-41 [3438 B]\nFetched 64.1 kB in 0s (66.8 kB/s)\nSelecting previously unselected package netcat-traditional.\r\n(Reading database ... \r(Reading database ... 5%\r(Reading database ... 10%\r(Reading database ... 15%\r(Reading database ... 20%\r(Reading database ... 25%\r(Reading database ... 30%\r(Reading database ... 35%\r(Reading database ... 40%\r(Reading database ... 45%\r(Reading database ... 50%\r(Reading database ... 55%\r(Reading database ... 60%\r(Reading database ... 65%\r(Reading database ... 70%\r(Reading database ... 75%\r(Reading database ... 80%\r(Reading database ... 85%\r(Reading database ... 90%\r(Reading database ... 95%\r(Reading database ... 100%\r(Reading database ... 9575 files and directories currently installed.)\r\nPreparing to unpack .../netcat-traditional_1.10-41_amd64.deb ...\r\nUnpacking netcat-traditional (1.10-41) ...\r\nSelecting previously unselected package netcat.\r\nPreparing to unpack .../netcat_1.10-41_all.deb ...\r\nUnpacking netcat (1.10-41) ...\r\nSetting up netcat-traditional (1.10-41) ...\r\nupdate-alternatives: using /bin/nc.traditional to provide /bin/nc (nc) in auto mode\r\nSetting up netcat (1.10-41) ...", "stdout_lines": ["Get:1 http://security.ubuntu.com/ubuntu xenial-security InRelease [109 kB]", "Hit:2 http://archive.ubuntu.com/ubuntu xenial InRelease", "Err:2 http://archive.ubuntu.com/ubuntu xenial InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Get:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease [109 kB]", "Err:1 http://security.ubuntu.com/ubuntu xenial-security InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Err:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Get:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease [107 kB]", "Err:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Fetched 325 kB in 1s (178 kB/s)", "Reading package lists...", "Reading package lists...", "Building dependency tree...", "Reading state information...", "The following additional packages will be installed:", "  netcat-traditional", "The following NEW packages will be installed:", "  netcat netcat-traditional", "0 upgraded, 2 newly installed, 0 to remove and 24 not upgraded.", "Need to get 64.1 kB of archives.", "After this operation, 191 kB of additional disk space will be used.", "Get:1 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat-traditional amd64 1.10-41 [60.7 kB]", "Get:2 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat all 1.10-41 [3438 B]", "Fetched 64.1 kB in 0s (66.8 kB/s)", "Selecting previously unselected package netcat-traditional.", "(Reading database ...

TASK [Start receiver to receive data object from replica pod] ******************
2019-11-20T05:25:18.728171 (delta: 11.891674)         elapsed: 260.061219 *****
changed: [<IP_ADDRESS>] => {"ansible_job_id": "789677201052.2597", "changed": true, "finished": 0, "results_file": "/root/.ansible_async/789677201052.2597", "started": 1}

TASK [Wait for few seconds for nc receiver] ************************************
2019-11-20T05:25:20.148278 (delta: 1.419986)         elapsed: 261.481326 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "sleep 5", "delta": "0:00:06.216655", "end": "2019-11-20 05:25:26.644897", "rc": 0, "start": "2019-11-20 05:25:20.428242", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Send data object from replica pod] ***************************************
2019-11-20T05:25:26.832761 (delta: 6.684356)         elapsed: 268.165809 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-7dpf-695bf4d66b-x9swp -n openebs --container cstor-pool -- bash -c \"nc -w 3 <IP_ADDRESS> 9090 < /1.dump\"", "delta": "0:00:09.359881", "end": "2019-11-20 05:25:36.546897", "rc": 0, "start": "2019-11-20 05:25:27.187016", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [kill receiver] ***********************************************************
2019-11-20T05:25:36.656774 (delta: 9.82384)         elapsed: 277.989822 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "bash -c \"for i in `ps aux |grep \\\"nc -l -p 9090\\\" |grep -v grep |awk -F ' ' '{print $2}'`; do kill -9 $i; done\"", "delta": "0:00:01.407357", "end": "2019-11-20 05:25:38.579041", "rc": 0, "start": "2019-11-20 05:25:37.171684", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Wait for few seconds for nc receiver] ************************************
2019-11-20T05:25:38.695712 (delta: 2.038821)         elapsed: 280.02876 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "sleep 5", "delta": "0:00:06.255437", "end": "2019-11-20 05:25:45.300524", "rc": 0, "start": "2019-11-20 05:25:39.045087", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Start receiver to receive meta-data object from replica pod] *************
2019-11-20T05:25:45.429129 (delta: 6.73329)         elapsed: 286.762177 *******
changed: [<IP_ADDRESS>] => {"ansible_job_id": "513194976802.2745", "changed": true, "finished": 0, "results_file": "/root/.ansible_async/513194976802.2745", "started": 1}

TASK [Send meta-data object from replica pod] **********************************
2019-11-20T05:25:46.775339 (delta: 1.346103)         elapsed: 288.108387 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-7dpf-695bf4d66b-x9swp -n openebs --container cstor-pool -- bash -c \"nc -w 3 <IP_ADDRESS> 9090 < /3.dump\"", "delta": "0:00:08.078133", "end": "2019-11-20 05:25:55.187852", "rc": 0, "start": "2019-11-20 05:25:47.109719", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [kill receiver] ***********************************************************
2019-11-20T05:25:55.330617 (delta: 8.555131)         elapsed: 296.663665 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "bash -c \"for i in `ps aux |grep \\\"nc -l -p 9090\\\" |grep -v grep |awk -F ' ' '{print $2}'`; do kill -9 $i; done\"", "delta": "0:00:01.264948", "end": "2019-11-20 05:25:56.858869", "rc": 0, "start": "2019-11-20 05:25:55.593921", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Get replica id] **********************************************************
2019-11-20T05:25:56.962098 (delta: 1.631345)         elapsed: 298.295146 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get pods cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn -n openebs -o custom-columns=:.spec.containers[*].env[0].value --no-headers | cut -d \",\" -f1", "delta": "0:00:01.456359", "end": "2019-11-20 05:25:58.786027", "rc": 0, "start": "2019-11-20 05:25:57.329668", "stderr": "", "stderr_lines": [], "stdout": "13fb6066-bd9a-4d7a-bb9f-dd80d08d34aa", "stdout_lines": ["13fb6066-bd9a-4d7a-bb9f-dd80d08d34aa"]}

TASK [Check for dataset name] **************************************************
2019-11-20T05:25:58.908920 (delta: 1.946731)         elapsed: 300.241968 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn -n openebs --container cstor-pool -- zfs list cstor-13fb6066-bd9a-4d7a-bb9f-dd80d08d34aa/pvc-3dd89244-a004-423f-b540-f6f7330720ea", "delta": "0:00:02.269449", "end": "2019-11-20 05:26:01.580200", "rc": 0, "start": "2019-11-20 05:25:59.310751", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "NAME                                                                                USED  AVAIL  REFER  MOUNTPOINT\ncstor-13fb6066-bd9a-4d7a-bb9f-dd80d08d34aa/pvc-3dd89244-a004-423f-b540-f6f7330720ea  29.1M  48.2G  25.7M  -", "stdout_lines": ["NAME                                                                                USED  AVAIL  REFER  MOUNTPOINT", "cstor-13fb6066-bd9a-4d7a-bb9f-dd80d08d34aa/pvc-3dd89244-a004-423f-b540-f6f7330720ea  29.1M  48.2G  25.7M  -"]}

TASK [Dump snapshot datastream to a file] **************************************
2019-11-20T05:26:01.732765 (delta: 2.823716)         elapsed: 303.065813 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn -n openebs --container cstor-pool -- bash -c \"zfs send cstor-13fb6066-bd9a-4d7a-bb9f-dd80d08d34aa/pvc-3dd89244-a004-423f-b540-f6f7330720ea@25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg > /snap.data\"", "delta": "0:00:04.992591", "end": "2019-11-20 05:26:07.001724", "rc": 0, "start": "2019-11-20 05:26:02.009133", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [Remove any object dump files from replica pod] ***************************
2019-11-20T05:26:07.158030 (delta: 5.425155)         elapsed: 308.491078 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn -n openebs --container cstor-pool -- rm -rf 1.dump 3.dump", "delta": "0:00:02.070747", "end": "2019-11-20 05:26:09.541687", "rc": 0, "start": "2019-11-20 05:26:07.470940", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [Parse snapshot stream and create dump files] *****************************
2019-11-20T05:26:09.651851 (delta: 2.493676)         elapsed: 310.984899 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn -n openebs --container cstor-pool -- bash -c \"zstreamdump -w -1 < /snap.data\"", "delta": "0:00:03.062311", "end": "2019-11-20 05:26:13.069321", "rc": 0, "start": "2019-11-20 05:26:10.007010", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "BEGIN record\n\thdrtype = 1\n\tfeatures = 0\n\tmagic = 2f5bacbac\n\tcreation_time = 5dd4ce11\n\ttype = 3\n\tflags = 0x4\n\ttoguid = c629c86c9ce7ffb7\n\tfromguid = 0\n\ttoname = cstor-13fb6066-bd9a-4d7a-bb9f-dd80d08d34aa/pvc-3dd89244-a004-423f-b540-f6f7330720ea@25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg\nEND checksum = 5afdb02765a51a/57b0fc56513f06d9/5e32b424ef42d490/5206a0e49cf195a2\nSUMMARY:\n\tTotal DRR_BEGIN records = 1\n\tTotal DRR_END records = 1\n\tTotal DRR_OBJECT records = 3\n\tTotal DRR_FREEOBJECTS records = 4\n\tTotal DRR_WRITE records = 45240\n\tTotal DRR_WRITE_BYREF records = 0\n\tTotal DRR_WRITE_EMBEDDED records = 0\n\tTotal DRR_FREE records = 1493\n\tTotal DRR_SPILL records = 0\n\tTotal records = 46742\n\tTotal write size = 185299456 (0xb0b7200)\n\tTotal stream length = 199882960 (0xbe9f8d0)", "stdout_lines": ["BEGIN record", "\thdrtype = 1", "\tfeatures = 0", "\tmagic = 2f5bacbac", "\tcreation_time = 5dd4ce11", "\ttype = 3", "\tflags = 0x4", "\ttoguid = c629c86c9ce7ffb7", "\tfromguid = 0", "\ttoname = cstor-13fb6066-bd9a-4d7a-bb9f-dd80d08d34aa/pvc-3dd89244-a004-423f-b540-f6f7330720ea@25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg", "END checksum = 5afdb02765a51a/57b0fc56513f06d9/5e32b424ef42d490/5206a0e49cf195a2", "SUMMARY:", "\tTotal DRR_BEGIN records = 1", "\tTotal DRR_END records = 1", "\tTotal DRR_OBJECT records = 3", "\tTotal DRR_FREEOBJECTS records = 4", "\tTotal DRR_WRITE records = 45240", "\tTotal DRR_WRITE_BYREF records = 0", "\tTotal DRR_WRITE_EMBEDDED records = 0", "\tTotal DRR_FREE records = 1493", "\tTotal DRR_SPILL records = 0", "\tTotal records = 46742", "\tTotal write size = 185299456 (0xb0b7200)", "\tTotal stream length = 199882960 (0xbe9f8d0)"]}

TASK [Install netcat on replica pod] *******************************************
2019-11-20T05:26:13.191884 (delta: 3.539925)         elapsed: 314.524932 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn -n openebs --container cstor-pool -- bash -c \"apt-get update && apt-get -y install netcat\"", "delta": "0:00:11.406625", "end": "2019-11-20 05:26:24.982302", "rc": 0, "start": "2019-11-20 05:26:13.575677", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file\nCouldn't create tempfiles for splitting up /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/security.ubuntu.com_ubuntu_dists_xenial-security_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-updates_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-backports_InReleaseW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://security.ubuntu.com/ubuntu xenial-security InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-updates InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-backports InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-updates/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-backports/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://security.ubuntu.com/ubuntu/dists/xenial-security/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Some index files failed to download. They have been ignored, or old ones used instead.\ndebconf: unable to initialize frontend: Dialog\ndebconf: (TERM is not set, so the dialog frontend is not usable.)\ndebconf: falling back to frontend: Readline\ndebconf: unable to initialize frontend: Readline\ndebconf: (This frontend requires a controlling tty.)\ndebconf: falling back to frontend: Teletype\ndpkg-preconfigure: unable to re-open stdin: ", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file", "Couldn't create tempfiles for splitting up /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/security.ubuntu.com_ubuntu_dists_xenial-security_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-updates_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-backports_InReleaseW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://security.ubuntu.com/ubuntu xenial-security InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-updates InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-backports InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-updates/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-backports/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://security.ubuntu.com/ubuntu/dists/xenial-security/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Some index files failed to download. They have been ignored, or old ones used instead.", "debconf: unable to initialize frontend: Dialog", "debconf: (TERM is not set, so the dialog frontend is not usable.)", "debconf: falling back to frontend: Readline", "debconf: unable to initialize frontend: Readline", "debconf: (This frontend requires a controlling tty.)", "debconf: falling back to frontend: Teletype", "dpkg-preconfigure: unable to re-open stdin: "], "stdout": "Get:1 http://security.ubuntu.com/ubuntu xenial-security InRelease [109 kB]\nHit:2 http://archive.ubuntu.com/ubuntu xenial InRelease\nErr:2 http://archive.ubuntu.com/ubuntu xenial InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nGet:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease [109 kB]\nErr:1 http://security.ubuntu.com/ubuntu xenial-security InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nErr:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nGet:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease [107 kB]\nErr:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nFetched 325 kB in 1s (185 kB/s)\nReading package lists...\nReading package lists...\nBuilding dependency tree...\nReading state information...\nThe following additional packages will be installed:\n  netcat-traditional\nThe following NEW packages will be installed:\n  netcat netcat-traditional\n0 upgraded, 2 newly installed, 0 to remove and 24 not upgraded.\nNeed to get 64.1 kB of archives.\nAfter this operation, 191 kB of additional disk space will be used.\nGet:1 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat-traditional amd64 1.10-41 [60.7 kB]\nGet:2 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat all 1.10-41 [3438 B]\nFetched 64.1 kB in 0s (74.9 kB/s)\nSelecting previously unselected package netcat-traditional.\r\n(Reading database ... \r(Reading database ... 5%\r(Reading database ... 10%\r(Reading database ... 15%\r(Reading database ... 20%\r(Reading database ... 25%\r(Reading database ... 30%\r(Reading database ... 35%\r(Reading database ... 40%\r(Reading database ... 45%\r(Reading database ... 50%\r(Reading database ... 55%\r(Reading database ... 60%\r(Reading database ... 65%\r(Reading database ... 70%\r(Reading database ... 75%\r(Reading database ... 80%\r(Reading database ... 85%\r(Reading database ... 90%\r(Reading database ... 95%\r(Reading database ... 100%\r(Reading database ... 9575 files and directories currently installed.)\r\nPreparing to unpack .../netcat-traditional_1.10-41_amd64.deb ...\r\nUnpacking netcat-traditional (1.10-41) ...\r\nSelecting previously unselected package netcat.\r\nPreparing to unpack .../netcat_1.10-41_all.deb ...\r\nUnpacking netcat (1.10-41) ...\r\nSetting up netcat-traditional (1.10-41) ...\r\nupdate-alternatives: using /bin/nc.traditional to provide /bin/nc (nc) in auto mode\r\nSetting up netcat (1.10-41) ...", "stdout_lines": ["Get:1 http://security.ubuntu.com/ubuntu xenial-security InRelease [109 kB]", "Hit:2 http://archive.ubuntu.com/ubuntu xenial InRelease", "Err:2 http://archive.ubuntu.com/ubuntu xenial InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Get:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease [109 kB]", "Err:1 http://security.ubuntu.com/ubuntu xenial-security InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Err:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Get:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease [107 kB]", "Err:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Fetched 325 kB in 1s (185 kB/s)", "Reading package lists...", "Reading package lists...", "Building dependency tree...", "Reading state information...", "The following additional packages will be installed:", "  netcat-traditional", "The following NEW packages will be installed:", "  netcat netcat-traditional", "0 upgraded, 2 newly installed, 0 to remove and 24 not upgraded.", "Need to get 64.1 kB of archives.", "After this operation, 191 kB of additional disk space will be used.", "Get:1 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat-traditional amd64 1.10-41 [60.7 kB]", "Get:2 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat all 1.10-41 [3438 B]", "Fetched 64.1 kB in 0s (74.9 kB/s)", "Selecting previously unselected package netcat-traditional.", "(Reading database ...

TASK [Start receiver to receive data object from replica pod] ******************
2019-11-20T05:26:25.100572 (delta: 11.908562)         elapsed: 326.43362 ******
changed: [<IP_ADDRESS>] => {"ansible_job_id": "641524660000.3072", "changed": true, "finished": 0, "results_file": "/root/.ansible_async/641524660000.3072", "started": 1}

TASK [Wait for few seconds for nc receiver] ************************************
2019-11-20T05:26:26.402279 (delta: 1.301586)         elapsed: 327.735327 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "sleep 5", "delta": "0:00:06.252730", "end": "2019-11-20 05:26:32.944698", "rc": 0, "start": "2019-11-20 05:26:26.691968", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Send data object from replica pod] ***************************************
2019-11-20T05:26:33.159912 (delta: 6.757539)         elapsed: 334.49296 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn -n openebs --container cstor-pool -- bash -c \"nc -w 3 <IP_ADDRESS> 9090 < /1.dump\"", "delta": "0:00:09.460982", "end": "2019-11-20 05:26:43.023895", "rc": 0, "start": "2019-11-20 05:26:33.562913", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [kill receiver] ***********************************************************
2019-11-20T05:26:43.121342 (delta: 9.961227)         elapsed: 344.45439 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "bash -c \"for i in `ps aux |grep \\\"nc -l -p 9090\\\" |grep -v grep |awk -F ' ' '{print $2}'`; do kill -9 $i; done\"", "delta": "0:00:01.169676", "end": "2019-11-20 05:26:44.641893", "rc": 0, "start": "2019-11-20 05:26:43.472217", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Wait for few seconds for nc receiver] ************************************
2019-11-20T05:26:44.743013 (delta: 1.621547)         elapsed: 346.076061 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "sleep 5", "delta": "0:00:06.314212", "end": "2019-11-20 05:26:51.484055", "rc": 0, "start": "2019-11-20 05:26:45.169843", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Start receiver to receive meta-data object from replica pod] *************
2019-11-20T05:26:51.589997 (delta: 6.846857)         elapsed: 352.923045 ******
changed: [<IP_ADDRESS>] => {"ansible_job_id": "864326230240.3220", "changed": true, "finished": 0, "results_file": "/root/.ansible_async/864326230240.3220", "started": 1}

TASK [Send meta-data object from replica pod] **********************************
2019-11-20T05:26:52.891732 (delta: 1.301642)         elapsed: 354.22478 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn -n openebs --container cstor-pool -- bash -c \"nc -w 3 <IP_ADDRESS> 9090 < /3.dump\"", "delta": "0:00:08.284422", "end": "2019-11-20 05:27:01.444199", "rc": 0, "start": "2019-11-20 05:26:53.159777", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [kill receiver] ***********************************************************
2019-11-20T05:27:01.562439 (delta: 8.670602)         elapsed: 362.895487 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "bash -c \"for i in `ps aux |grep \\\"nc -l -p 9090\\\" |grep -v grep |awk -F ' ' '{print $2}'`; do kill -9 $i; done\"", "delta": "0:00:01.254659", "end": "2019-11-20 05:27:03.192670", "rc": 0, "start": "2019-11-20 05:27:01.938011", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Get replica id] **********************************************************
2019-11-20T05:27:03.279752 (delta: 1.717182)         elapsed: 364.6128 ********
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get pods cstor-block-disk-pool-lp8w-89df46b4c-mwcxx -n openebs -o custom-columns=:.spec.containers[*].env[0].value --no-headers | cut -d \",\" -f1", "delta": "0:00:01.452652", "end": "2019-11-20 05:27:05.022955", "rc": 0, "start": "2019-11-20 05:27:03.570303", "stderr": "", "stderr_lines": [], "stdout": "717eb3fb-86e5-4010-a895-c94846e13fd3", "stdout_lines": ["717eb3fb-86e5-4010-a895-c94846e13fd3"]}

TASK [Check for dataset name] **************************************************
2019-11-20T05:27:05.133036 (delta: 1.853188)         elapsed: 366.466084 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-lp8w-89df46b4c-mwcxx -n openebs --container cstor-pool -- zfs list cstor-717eb3fb-86e5-4010-a895-c94846e13fd3/pvc-3dd89244-a004-423f-b540-f6f7330720ea", "delta": "0:00:02.351210", "end": "2019-11-20 05:27:07.877708", "rc": 0, "start": "2019-11-20 05:27:05.526498", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "NAME                                                                                USED  AVAIL  REFER  MOUNTPOINT\ncstor-717eb3fb-86e5-4010-a895-c94846e13fd3/pvc-3dd89244-a004-423f-b540-f6f7330720ea  31.3M  48.2G  25.8M  -", "stdout_lines": ["NAME                                                                                USED  AVAIL  REFER  MOUNTPOINT", "cstor-717eb3fb-86e5-4010-a895-c94846e13fd3/pvc-3dd89244-a004-423f-b540-f6f7330720ea  31.3M  48.2G  25.8M  -"]}

TASK [Dump snapshot datastream to a file] **************************************
2019-11-20T05:27:07.998148 (delta: 2.864972)         elapsed: 369.331196 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-lp8w-89df46b4c-mwcxx -n openebs --container cstor-pool -- bash -c \"zfs send cstor-717eb3fb-86e5-4010-a895-c94846e13fd3/pvc-3dd89244-a004-423f-b540-f6f7330720ea@25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg > /snap.data\"", "delta": "0:00:05.050805", "end": "2019-11-20 05:27:13.366775", "rc": 0, "start": "2019-11-20 05:27:08.315970", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [Remove any object dump files from replica pod] ***************************
2019-11-20T05:27:13.498992 (delta: 5.500711)         elapsed: 374.83204 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-lp8w-89df46b4c-mwcxx -n openebs --container cstor-pool -- rm -rf 1.dump 3.dump", "delta": "0:00:02.074154", "end": "2019-11-20 05:27:15.951711", "rc": 0, "start": "2019-11-20 05:27:13.877557", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [Parse snapshot stream and create dump files] *****************************
2019-11-20T05:27:16.171137 (delta: 2.67201)         elapsed: 377.504185 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-lp8w-89df46b4c-mwcxx -n openebs --container cstor-pool -- bash -c \"zstreamdump -w -1 < /snap.data\"", "delta": "0:00:03.019455", "end": "2019-11-20 05:27:19.550058", "rc": 0, "start": "2019-11-20 05:27:16.530603", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "BEGIN record\n\thdrtype = 1\n\tfeatures = 0\n\tmagic = 2f5bacbac\n\tcreation_time = 5dd4ce12\n\ttype = 3\n\tflags = 0x4\n\ttoguid = 1057504bfebeeba8\n\tfromguid = 0\n\ttoname = cstor-717eb3fb-86e5-4010-a895-c94846e13fd3/pvc-3dd89244-a004-423f-b540-f6f7330720ea@25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg\nEND checksum = 5ac13990e05fa8/6abd469bbe35d14c/cad6aa319ab01e33/8e676ac0a8bc78d\nSUMMARY:\n\tTotal DRR_BEGIN records = 1\n\tTotal DRR_END records = 1\n\tTotal DRR_OBJECT records = 3\n\tTotal DRR_FREEOBJECTS records = 4\n\tTotal DRR_WRITE records = 45240\n\tTotal DRR_WRITE_BYREF records = 0\n\tTotal DRR_WRITE_EMBEDDED records = 0\n\tTotal DRR_FREE records = 1493\n\tTotal DRR_SPILL records = 0\n\tTotal records = 46742\n\tTotal write size = 185299456 (0xb0b7200)\n\tTotal stream length = 199882960 (0xbe9f8d0)", "stdout_lines": ["BEGIN record", "\thdrtype = 1", "\tfeatures = 0", "\tmagic = 2f5bacbac", "\tcreation_time = 5dd4ce12", "\ttype = 3", "\tflags = 0x4", "\ttoguid = 1057504bfebeeba8", "\tfromguid = 0", "\ttoname = cstor-717eb3fb-86e5-4010-a895-c94846e13fd3/pvc-3dd89244-a004-423f-b540-f6f7330720ea@25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg", "END checksum = 5ac13990e05fa8/6abd469bbe35d14c/cad6aa319ab01e33/8e676ac0a8bc78d", "SUMMARY:", "\tTotal DRR_BEGIN records = 1", "\tTotal DRR_END records = 1", "\tTotal DRR_OBJECT records = 3", "\tTotal DRR_FREEOBJECTS records = 4", "\tTotal DRR_WRITE records = 45240", "\tTotal DRR_WRITE_BYREF records = 0", "\tTotal DRR_WRITE_EMBEDDED records = 0", "\tTotal DRR_FREE records = 1493", "\tTotal DRR_SPILL records = 0", "\tTotal records = 46742", "\tTotal write size = 185299456 (0xb0b7200)", "\tTotal stream length = 199882960 (0xbe9f8d0)"]}

TASK [Install netcat on replica pod] *******************************************
2019-11-20T05:27:19.756871 (delta: 3.585567)         elapsed: 381.089919 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-lp8w-89df46b4c-mwcxx -n openebs --container cstor-pool -- bash -c \"apt-get update && apt-get -y install netcat\"", "delta": "0:00:11.473939", "end": "2019-11-20 05:27:31.689572", "rc": 0, "start": "2019-11-20 05:27:20.215633", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file\nCouldn't create tempfiles for splitting up /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-updates_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/security.ubuntu.com_ubuntu_dists_xenial-security_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-backports_InReleaseW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-updates InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://security.ubuntu.com/ubuntu xenial-security InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-backports InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-updates/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-backports/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Failed to fetch http://security.ubuntu.com/ubuntu/dists/xenial-security/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nW: Some index files failed to download. They have been ignored, or old ones used instead.\ndebconf: unable to initialize frontend: Dialog\ndebconf: (TERM is not set, so the dialog frontend is not usable.)\ndebconf: falling back to frontend: Readline\ndebconf: unable to initialize frontend: Readline\ndebconf: (This frontend requires a controlling tty.)\ndebconf: falling back to frontend: Teletype\ndpkg-preconfigure: unable to re-open stdin: ", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file", "Couldn't create tempfiles for splitting up /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_xenial_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-updates_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/security.ubuntu.com_ubuntu_dists_xenial-security_InReleaseCouldn't create tempfiles for splitting up /var/lib/apt/lists/partial/archive.ubuntu.com_ubuntu_dists_xenial-backports_InReleaseW: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-updates InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://security.ubuntu.com/ubuntu xenial-security InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: http://archive.ubuntu.com/ubuntu xenial-backports InRelease: Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-updates/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/xenial-backports/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Failed to fetch http://security.ubuntu.com/ubuntu/dists/xenial-security/InRelease  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "W: Some index files failed to download. They have been ignored, or old ones used instead.", "debconf: unable to initialize frontend: Dialog", "debconf: (TERM is not set, so the dialog frontend is not usable.)", "debconf: falling back to frontend: Readline", "debconf: unable to initialize frontend: Readline", "debconf: (This frontend requires a controlling tty.)", "debconf: falling back to frontend: Teletype", "dpkg-preconfigure: unable to re-open stdin: "], "stdout": "Hit:1 http://archive.ubuntu.com/ubuntu xenial InRelease\nErr:1 http://archive.ubuntu.com/ubuntu xenial InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nGet:2 http://security.ubuntu.com/ubuntu xenial-security InRelease [109 kB]\nGet:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease [109 kB]\nErr:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nErr:2 http://security.ubuntu.com/ubuntu xenial-security InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nGet:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease [107 kB]\nErr:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease\n  Could not execute 'apt-key' to verify signature (is gnupg installed?)\nFetched 325 kB in 1s (179 kB/s)\nReading package lists...\nReading package lists...\nBuilding dependency tree...\nReading state information...\nThe following additional packages will be installed:\n  netcat-traditional\nThe following NEW packages will be installed:\n  netcat netcat-traditional\n0 upgraded, 2 newly installed, 0 to remove and 24 not upgraded.\nNeed to get 64.1 kB of archives.\nAfter this operation, 191 kB of additional disk space will be used.\nGet:1 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat-traditional amd64 1.10-41 [60.7 kB]\nGet:2 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat all 1.10-41 [3438 B]\nFetched 64.1 kB in 0s (71.4 kB/s)\nSelecting previously unselected package netcat-traditional.\r\n(Reading database ... \r(Reading database ... 5%\r(Reading database ... 10%\r(Reading database ... 15%\r(Reading database ... 20%\r(Reading database ... 25%\r(Reading database ... 30%\r(Reading database ... 35%\r(Reading database ... 40%\r(Reading database ... 45%\r(Reading database ... 50%\r(Reading database ... 55%\r(Reading database ... 60%\r(Reading database ... 65%\r(Reading database ... 70%\r(Reading database ... 75%\r(Reading database ... 80%\r(Reading database ... 85%\r(Reading database ... 90%\r(Reading database ... 95%\r(Reading database ... 100%\r(Reading database ... 9575 files and directories currently installed.)\r\nPreparing to unpack .../netcat-traditional_1.10-41_amd64.deb ...\r\nUnpacking netcat-traditional (1.10-41) ...\r\nSelecting previously unselected package netcat.\r\nPreparing to unpack .../netcat_1.10-41_all.deb ...\r\nUnpacking netcat (1.10-41) ...\r\nSetting up netcat-traditional (1.10-41) ...\r\nupdate-alternatives: using /bin/nc.traditional to provide /bin/nc (nc) in auto mode\r\nSetting up netcat (1.10-41) ...", "stdout_lines": ["Hit:1 http://archive.ubuntu.com/ubuntu xenial InRelease", "Err:1 http://archive.ubuntu.com/ubuntu xenial InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Get:2 http://security.ubuntu.com/ubuntu xenial-security InRelease [109 kB]", "Get:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease [109 kB]", "Err:3 http://archive.ubuntu.com/ubuntu xenial-updates InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Err:2 http://security.ubuntu.com/ubuntu xenial-security InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Get:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease [107 kB]", "Err:4 http://archive.ubuntu.com/ubuntu xenial-backports InRelease", "  Could not execute 'apt-key' to verify signature (is gnupg installed?)", "Fetched 325 kB in 1s (179 kB/s)", "Reading package lists...", "Reading package lists...", "Building dependency tree...", "Reading state information...", "The following additional packages will be installed:", "  netcat-traditional", "The following NEW packages will be installed:", "  netcat netcat-traditional", "0 upgraded, 2 newly installed, 0 to remove and 24 not upgraded.", "Need to get 64.1 kB of archives.", "After this operation, 191 kB of additional disk space will be used.", "Get:1 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat-traditional amd64 1.10-41 [60.7 kB]", "Get:2 http://archive.ubuntu.com/ubuntu xenial/universe amd64 netcat all 1.10-41 [3438 B]", "Fetched 64.1 kB in 0s (71.4 kB/s)", "Selecting previously unselected package netcat-traditional.", "(Reading database ...

TASK [Start receiver to receive data object from replica pod] ******************
2019-11-20T05:27:31.861148 (delta: 12.104134)         elapsed: 393.194196 *****
changed: [<IP_ADDRESS>] => {"ansible_job_id": "269674998744.3549", "changed": true, "finished": 0, "results_file": "/root/.ansible_async/269674998744.3549", "started": 1}

TASK [Wait for few seconds for nc receiver] ************************************
2019-11-20T05:27:33.243933 (delta: 1.382661)         elapsed: 394.576981 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "sleep 5", "delta": "0:00:06.189103", "end": "2019-11-20 05:27:39.703641", "rc": 0, "start": "2019-11-20 05:27:33.514538", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Send data object from replica pod] ***************************************
2019-11-20T05:27:39.883540 (delta: 6.63952)         elapsed: 401.216588 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-lp8w-89df46b4c-mwcxx -n openebs --container cstor-pool -- bash -c \"nc -w 3 <IP_ADDRESS> 9090 < /1.dump\"", "delta": "0:00:09.558148", "end": "2019-11-20 05:27:49.814289", "rc": 0, "start": "2019-11-20 05:27:40.256141", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [kill receiver] ***********************************************************
2019-11-20T05:27:49.961744 (delta: 10.078032)         elapsed: 411.294792 *****
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "bash -c \"for i in `ps aux |grep \\\"nc -l -p 9090\\\" |grep -v grep |awk -F ' ' '{print $2}'`; do kill -9 $i; done\"", "delta": "0:00:01.268209", "end": "2019-11-20 05:27:51.581212", "rc": 0, "start": "2019-11-20 05:27:50.313003", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Wait for few seconds for nc receiver] ************************************
2019-11-20T05:27:51.673548 (delta: 1.711674)         elapsed: 413.006596 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "sleep 5", "delta": "0:00:06.259127", "end": "2019-11-20 05:27:58.291684", "rc": 0, "start": "2019-11-20 05:27:52.032557", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Start receiver to receive meta-data object from replica pod] *************
2019-11-20T05:27:58.484286 (delta: 6.810634)         elapsed: 419.817334 ******
changed: [<IP_ADDRESS>] => {"ansible_job_id": "871236651663.3698", "changed": true, "finished": 0, "results_file": "/root/.ansible_async/871236651663.3698", "started": 1}

TASK [Send meta-data object from replica pod] **********************************
2019-11-20T05:27:59.835840 (delta: 1.351379)         elapsed: 421.168888 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it cstor-block-disk-pool-lp8w-89df46b4c-mwcxx -n openebs --container cstor-pool -- bash -c \"nc -w 3 <IP_ADDRESS> 9090 < /3.dump\"", "delta": "0:00:08.238977", "end": "2019-11-20 05:28:08.453094", "rc": 0, "start": "2019-11-20 05:28:00.214117", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "", "stdout_lines": []}

TASK [kill receiver] ***********************************************************
2019-11-20T05:28:08.557017 (delta: 8.721054)         elapsed: 429.890065 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "bash -c \"for i in `ps aux |grep \\\"nc -l -p 9090\\\" |grep -v grep |awk -F ' ' '{print $2}'`; do kill -9 $i; done\"", "delta": "0:00:01.357041", "end": "2019-11-20 05:28:10.284956", "rc": 0, "start": "2019-11-20 05:28:08.927915", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [set_fact] ****************************************************************
2019-11-20T05:28:10.386008 (delta: 1.828895)         elapsed: 431.719056 ******
ok: [<IP_ADDRESS>] => {"ansible_facts": {"base_replica": "cstor-block-disk-pool-7dpf-695bf4d66b-x9swp"}, "changed": false}

TASK [compare data object] *****************************************************
2019-11-20T05:28:10.546731 (delta: 0.160622)         elapsed: 431.879779 ******
changed: [<IP_ADDRESS>] => (item=cstor-block-disk-pool-7dpf-695bf4d66b-x9swp) => {"changed": true, "cmd": "diff cstor-block-disk-pool-7dpf-695bf4d66b-x9swp.1.dump cstor-block-disk-pool-7dpf-695bf4d66b-x9swp.1.dump", "delta": "0:00:01.216596", "end": "2019-11-20 05:28:12.164669", "rc": 0, "rpod": "cstor-block-disk-pool-7dpf-695bf4d66b-x9swp", "start": "2019-11-20 05:28:10.948073", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
changed: [<IP_ADDRESS>] => (item=cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn) => {"changed": true, "cmd": "diff cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn.1.dump cstor-block-disk-pool-7dpf-695bf4d66b-x9swp.1.dump", "delta": "0:00:01.670753", "end": "2019-11-20 05:28:14.162027", "rc": 0, "rpod": "cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn", "start": "2019-11-20 05:28:12.491274", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
changed: [<IP_ADDRESS>] => (item=cstor-block-disk-pool-lp8w-89df46b4c-mwcxx) => {"changed": true, "cmd": "diff cstor-block-disk-pool-lp8w-89df46b4c-mwcxx.1.dump cstor-block-disk-pool-7dpf-695bf4d66b-x9swp.1.dump", "delta": "0:00:01.659047", "end": "2019-11-20 05:28:16.063586", "rc": 0, "rpod": "cstor-block-disk-pool-lp8w-89df46b4c-mwcxx", "start": "2019-11-20 05:28:14.404539", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [compare meta data object] ************************************************
2019-11-20T05:28:16.170976 (delta: 5.624118)         elapsed: 437.504024 ******
changed: [<IP_ADDRESS>] => (item=cstor-block-disk-pool-7dpf-695bf4d66b-x9swp) => {"changed": true, "cmd": "diff cstor-block-disk-pool-7dpf-695bf4d66b-x9swp.3.dump cstor-block-disk-pool-7dpf-695bf4d66b-x9swp.3.dump", "delta": "0:00:01.224571", "end": "2019-11-20 05:28:17.756368", "rc": 0, "rpod": "cstor-block-disk-pool-7dpf-695bf4d66b-x9swp", "start": "2019-11-20 05:28:16.531797", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
changed: [<IP_ADDRESS>] => (item=cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn) => {"changed": true, "cmd": "diff cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn.3.dump cstor-block-disk-pool-7dpf-695bf4d66b-x9swp.3.dump", "delta": "0:00:01.308337", "end": "2019-11-20 05:28:19.342039", "rc": 0, "rpod": "cstor-block-disk-pool-bj2k-97fd6f84f-hvvtn", "start": "2019-11-20 05:28:18.033702", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}
changed: [<IP_ADDRESS>] => (item=cstor-block-disk-pool-lp8w-89df46b4c-mwcxx) => {"changed": true, "cmd": "diff cstor-block-disk-pool-lp8w-89df46b4c-mwcxx.3.dump cstor-block-disk-pool-7dpf-695bf4d66b-x9swp.3.dump", "delta": "0:00:01.206848", "end": "2019-11-20 05:28:20.833748", "rc": 0, "rpod": "cstor-block-disk-pool-lp8w-89df46b4c-mwcxx", "start": "2019-11-20 05:28:19.626900", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []}

TASK [Destroy snapshot created for data verification] **************************
2019-11-20T05:28:20.957205 (delta: 4.786104)         elapsed: 442.290253 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec -it pvc-3dd89244-a004-423f-b540-f6f7330720ea-target-6c8d78c779bnfld -n openebs --container cstor-istgt -- istgtcontrol snapdestroy pvc-3dd89244-a004-423f-b540-f6f7330720ea 25cHTklZ5Cco2Jgk2JxrX2cQfEVWDUpg 30 30", "delta": "0:00:02.082094", "end": "2019-11-20 05:28:23.360585", "rc": 0, "start": "2019-11-20 05:28:21.278491", "stderr": "Unable to use a TTY - input is not a terminal or the right kind of file", "stderr_lines": ["Unable to use a TTY - input is not a terminal or the right kind of file"], "stdout": "DONE SNAPDESTROY command", "stdout_lines": ["DONE SNAPDESTROY command"]}

TASK [Verify application data persistence] *************************************
2019-11-20T05:28:23.451716 (delta: 2.494411)         elapsed: 444.784764 ******
included: /utils/scm/applications/mysql/mysql_data_persistence.yml for <IP_ADDRESS>

TASK [Create some test data in the mysql database] *****************************
2019-11-20T05:28:23.737790 (delta: 0.285978)         elapsed: 445.070838 ******
skipping: [<IP_ADDRESS>] => (item=mysql -uroot -pk8sDem0 -e 'create database testaman;')  => {"changed": false, "item": "mysql -uroot -pk8sDem0 -e 'create database testaman;'", "skip_reason": "Conditional result was False"}
skipping: [<IP_ADDRESS>] => (item=mysql -uroot -pk8sDem0 -e 'create table ttbl (Data VARCHAR(20));' testaman)  => {"changed": false, "item": "mysql -uroot -pk8sDem0 -e 'create table ttbl (Data VARCHAR(20));' testaman", "skip_reason": "Conditional result was False"}
skipping: [<IP_ADDRESS>] => (item=mysql -uroot -pk8sDem0 -e 'insert into ttbl (Data) VALUES ("tdata");' testaman)  => {"changed": false, "item": "mysql -uroot -pk8sDem0 -e 'insert into ttbl (Data) VALUES (\"tdata\");' testaman", "skip_reason": "Conditional result was False"}

TASK [Kill the application pod] ************************************************
2019-11-20T05:28:23.874202 (delta: 0.1363)         elapsed: 445.20725 *********
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl delete pod percona-7c6969cf78-qwb7x -n pc", "delta": "0:00:10.995645", "end": "2019-11-20 05:28:35.191499", "rc": 0, "start": "2019-11-20 05:28:24.195854", "stderr": "", "stderr_lines": [], "stdout": "pod \"percona-7c6969cf78-qwb7x\" deleted", "stdout_lines": ["pod \"percona-7c6969cf78-qwb7x\" deleted"]}

TASK [Verify if the application pod is deleted] ********************************
2019-11-20T05:28:35.346759 (delta: 11.4724)         elapsed: 456.679807 *******
 [WARNING]: when statements should not include jinja2 templating delimiters
such as {{ }} or {% %}. Found: "{{ pod_name }}" not in podstatus.stdout
changed: [<IP_ADDRESS>] => {"attempts": 1, "changed": true, "cmd": "kubectl get pods -n pc", "delta": "0:00:01.468209", "end": "2019-11-20 05:28:37.169274", "rc": 0, "start": "2019-11-20 05:28:35.701065", "stderr": "", "stderr_lines": [], "stdout": "NAME                       READY   STATUS    RESTARTS   AGE\npercona-7c6969cf78-npx8j   1/1     Running   0          12s", "stdout_lines": ["NAME                       READY   STATUS    RESTARTS   AGE", "percona-7c6969cf78-npx8j   1/1     Running   0          12s"]}

TASK [Obtain the newly created pod name for applicaion] ************************
2019-11-20T05:28:37.295120 (delta: 1.948229)         elapsed: 458.628168 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl get pods -n pc -l name=percona -o jsonpath='{.items[].metadata.name}'", "delta": "0:00:01.496989", "end": "2019-11-20 05:28:39.118755", "rc": 0, "start": "2019-11-20 05:28:37.621766", "stderr": "", "stderr_lines": [], "stdout": "percona-7c6969cf78-npx8j", "stdout_lines": ["percona-7c6969cf78-npx8j"]}

TASK [Checking application pod is in running state] ****************************
2019-11-20T05:28:39.270780 (delta: 1.975558)         elapsed: 460.603828 ******
changed: [<IP_ADDRESS>] => {"attempts": 1, "changed": true, "cmd": "kubectl get pods -n pc -o jsonpath='{.items[?(@.metadata.name==\"percona-7c6969cf78-npx8j\")].status.phase}'", "delta": "0:00:01.499745", "end": "2019-11-20 05:28:41.243654", "rc": 0, "start": "2019-11-20 05:28:39.743909", "stderr": "", "stderr_lines": [], "stdout": "Running", "stdout_lines": ["Running"]}

TASK [Get the container status of application.] ********************************
2019-11-20T05:28:41.422765 (delta: 2.151857)         elapsed: 462.755813 ******
changed: [<IP_ADDRESS>] => {"attempts": 1, "changed": true, "cmd": "kubectl get pods -n pc -o jsonpath='{.items[?(@.metadata.name==\"percona-7c6969cf78-npx8j\")].status.containerStatuses[].state}' | grep running", "delta": "0:00:01.452990", "end": "2019-11-20 05:28:43.331498", "rc": 0, "start": "2019-11-20 05:28:41.878508", "stderr": "", "stderr_lines": [], "stdout": "map[running:map[startedAt:2019-11-20T05:28:29Z]]", "stdout_lines": ["map[running:map[startedAt:2019-11-20T05:28:29Z]]"]}

TASK [Checking for the Corrupted tables] ***************************************
2019-11-20T05:28:43.497154 (delta: 2.074241)         elapsed: 464.830202 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec percona-7c6969cf78-npx8j -n pc -- mysqlcheck -c testaman -uroot -pk8sDem0", "delta": "0:00:02.020466", "end": "2019-11-20 05:28:46.049535", "failed_when_result": false, "rc": 0, "start": "2019-11-20 05:28:44.029069", "stderr": "mysqlcheck: [Warning] Using a password on the command line interface can be insecure.", "stderr_lines": ["mysqlcheck: [Warning] Using a password on the command line interface can be insecure."], "stdout": "testaman.ttbl                                      OK", "stdout_lines": ["testaman.ttbl                                      OK"]}

TASK [Verify mysql data persistence] *******************************************
2019-11-20T05:28:46.198551 (delta: 2.701253)         elapsed: 467.531599 ******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "kubectl exec percona-7c6969cf78-npx8j -n pc -- mysql -uroot -pk8sDem0 -e 'select * from ttbl' testaman;", "delta": "0:00:02.052772", "end": "2019-11-20 05:28:48.716711", "failed_when_result": false, "rc": 0, "start": "2019-11-20 05:28:46.663939", "stderr": "mysql: [Warning] Using a password on the command line interface can be insecure.", "stderr_lines": ["mysql: [Warning] Using a password on the command line interface can be insecure."], "stdout": "Data\ntdata", "stdout_lines": ["Data", "tdata"]}

TASK [Delete/drop MySQL database] **********************************************
2019-11-20T05:28:48.868086 (delta: 2.669402)         elapsed: 470.201134 ******
skipping: [<IP_ADDRESS>] => {"changed": false, "skip_reason": "Conditional result was False"}

TASK [Verify successful db delete] *********************************************
2019-11-20T05:28:48.996375 (delta: 0.128193)         elapsed: 470.329423 ******
skipping: [<IP_ADDRESS>] => {"changed": false, "skip_reason": "Conditional result was False"}

TASK [set_fact] ****************************************************************
2019-11-20T05:28:49.110247 (delta: 0.113744)         elapsed: 470.443295 ******
ok: [<IP_ADDRESS>] => {"ansible_facts": {"flag": "Pass"}, "changed": false}

TASK [include_tasks] ***********************************************************
2019-11-20T05:28:49.311929 (delta: 0.201568)         elapsed: 470.644977 ******
included: /utils/fcm/update_litmus_result_resource.yml for <IP_ADDRESS>

TASK [Generate the litmus result CR to reflect SOT (Start of Test)] ************
2019-11-20T05:28:49.507832 (delta: 0.195792)         elapsed: 470.84088 *******
skipping: [<IP_ADDRESS>] => {"changed": false, "skip_reason": "Conditional result was False"}

TASK [Analyze the cr yaml] *****************************************************
2019-11-20T05:28:49.605217 (delta: 0.097302)         elapsed: 470.938265 ******
skipping: [<IP_ADDRESS>] => {"changed": false, "skip_reason": "Conditional result was False"}

TASK [Apply the litmus result CR] **********************************************
2019-11-20T05:28:49.752541 (delta: 0.14721)         elapsed: 471.085589 *******
skipping: [<IP_ADDRESS>] => {"changed": false, "skip_reason": "Conditional result was False"}

TASK [Generate the litmus result CR to reflect EOT (End of Test)] **************
2019-11-20T05:28:49.929853 (delta: 0.177154)         elapsed: 471.262901 ******
changed: [<IP_ADDRESS>] => {"changed": true, "checksum": "29f47af49a2a3a5aa59a27f68d7e0ae0e21bf497", "dest": "./litmus-result.yaml", "gid": 0, "group": "root", "md5sum": "0c41791561c2cc1d180fe9dc2c7444f6", "mode": "0644", "owner": "root", "size": 418, "src": "/root/.ansible/tmp/ansible-tmp-1574227730.06-25858984824331/source", "state": "file", "uid": 0}

TASK [Analyze the cr yaml] *****************************************************
2019-11-20T05:28:53.227365 (delta: 3.29739)         elapsed: 474.560413 *******
changed: [<IP_ADDRESS>] => {"changed": true, "cmd": "cat litmus-result.yaml", "delta": "0:00:01.311071", "end": "2019-11-20 05:28:54.879607", "rc": 0, "start": "2019-11-20 05:28:53.568536", "stderr": "", "stderr_lines": [], "stdout": "---\napiVersion: litmus.io/v1alpha1\nkind: LitmusResult\nmetadata:\n\n  # name of the litmus testcase\n  name: openebs-cvr-scaleup \nspec:\n\n  # holds information on the testcase\n  testMetadata:\n    app:  \n    chaostype:  \n\n  # holds the state of testcase,  manually updated by json merge patch\n  # result is the useful value today, but anticipate phase use in future \n  testStatus: \n    phase: completed  \n    result: Pass ", "stdout_lines": ["---", "apiVersion: litmus.io/v1alpha1", "kind: LitmusResult", "metadata:", "", "  # name of the litmus testcase", "  name: openebs-cvr-scaleup ", "spec:", "", "  # holds information on the testcase", "  testMetadata:", "    app:  ", "    chaostype:  ", "", "  # holds the state of testcase,  manually updated by json merge patch", "  # result is the useful value today, but anticipate phase use in future ", "  testStatus: ", "    phase: completed  ", "    result: Pass "]}

TASK [Apply the litmus result CR] **********************************************
2019-11-20T05:28:54.989074 (delta: 1.761559)         elapsed: 476.322122 ******
changed: [<IP_ADDRESS>] => {"changed": true, "failed_when_result": false, "method": "patch", "result": {"apiVersion": "litmus.io/v1alpha1", "kind": "LitmusResult", "metadata": {"creationTimestamp": "2019-11-20T05:21:15Z", "generation": 2, "name": "openebs-cvr-scaleup", "resourceVersion": "487691", "selfLink": "/apis/litmus.io/v1alpha1/litmusresults/openebs-cvr-scaleup", "uid": "04542e93-1bac-4611-8ab9-9a790fba6976"}, "spec": {"testMetadata": {}, "testStatus": {"phase": "completed", "result": "Pass"}}}}

PLAY RECAP *********************************************************************
<IP_ADDRESS>                  : ok=117  changed=101  unreachable=0    failed=0

2019-11-20T05:28:56.450161 (delta: 1.460986)         elapsed: 477.783209 ******