Required details

---------------------------------------------------------------------------------------------------------------------------

C:\Program Files\MongoDB\Server\4.4\bin>mongod.exe --dbpath=E:\mongodb

C:\Kafka\kafka_2.12-2.2.0\zookeeper-3.4.14\bin>zkServer.cmd

C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows>kafka-server-start.bat C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\config\server.properties

C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows>kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue

kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue
kafka-console-producer.bat --broker-list localhost:9092 --topic FigiDataLoadQueue
kafka-console-producer.bat --broker-list localhost:9092 --topic StreamingUdlQueue


kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue
kafka-console-consumer --bootstrap-server localhost:9092 --topic topicname --offset 152582 --property print.key=true --property key.separator=|

---------------------------------------------------------------------------------------------------------------------------

mongod.exe --dbpath=E:\mongodb

mongod.exe --config=C:\Program Files\MongoDB\Server\3.6\mongod.conf --dbpath=e:\mongodb


start mongodb : C:\Program Files\MongoDB\Server\3.6\bin>mongod.exe

start zookeeper : C:\Kafka\kafka_2.12-2.2.0\zookeeper-3.4.14\bin>zkServer.cmd

C:\Zookeeper\zookeeper-3.4.14\bin>zkServer.cmd

start kafka broker C:\Kafka\kafka_2.12-2.2.0>C:\Kafka\kafka_2.12-2.2.0\bin\windows\kafka-server-start.bat C:\Kafka\kafka_2.12-2.2.0\config\server.properties


C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin


C:\Kafka\kafka2.4.1>C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows\kafka-server-start.bat C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\config\server.properties

C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows\kafka-server-start.bat C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\config\server.properties

C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows


(master)$ git tag –a 1.4.1 –m "Tagging 1.4.1 release."

(master)$ git push --tags

https://git-scm.com/book/en/v2/Git-Basics-Tagging


start producer console : C:\Kafka\kafka_2.12-2.2.0\bin\windows>kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue

kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue
kafka-console-producer.bat --broker-list localhost:9092 --topic FigiDataLoadQueue
kafka-console-producer.bat --broker-list localhost:9092 --topic StreamingUdlQueue

EnableRaisingExceptions true 16

IP:XXX.168.XXX.180
user/password:hadoop/hadoop
/home/hadoop/irfan/soft/Vinish
to start
cd RCCLoader-null
nohup ./run.sh &
to check logs
tail -100f ../logs/RCCLoader.log

files path
/home/hadoop/loader_files/si_watcher/done
cp application.properties ../../RCCLoader-null/config/
chmod 755 run.sh

hbase shell
to cont--- count 'tbalename'



http://camel.apache.org/tracer.html

XXX.16x.116.XXX udm - pa$$0rd123

Kafka1 XXX.168.116.140
udm/pa$$0rd123

Log location : /home/udm/PROD/SparkLoader/logs


ec.XXXX.eu-west-2.compute.amazonaws.com:8088/


select count(*) from "dvDomainCache".DVDOMAIN where DOMAINSOURCE='trds' or DOMAINSOURCE='figi' or DOMAINSOURCE='esma' or DOMAINSOURCE='idcApex'

select * from "dvDomainCache".DVDOMAIN where DOMAINSOURCE='esma' and DOMAINNAME='currencyMap7' and NORMALIZEDVALUE='MDL'

kafka-console-consumer.bat --bootstrap-server localhost:9092 test --from-beginning

kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic FigiRequestQueue --from-beginning

-DIGNITE_H2_DEBUG_CONSOLE=true




message from UDL for figi
{"isin":"US0455281065","sedol":"BZ6VMW7","cusip":"045528106","tradeCurrencyCode":"USD","exchangeCode":"XNYS","assetType":"5101", "changedIdentifiers":["sedol","isin","cusip","exchangeCode","tradeCurrencyCode","instrumentTypeCode"], "dependentMics":[],"micIndex":-1}


message for ProformaQueue
{"documentId":"5d67e3b396391f46940a9943","uniqueAttributeVsValueMap":{"rduInstrumentId":"RDUNIN000000000181"},"lastModifiedDate":"2019-08-29T14:39:47.44"}


{"documentId":"5e39ac659ef5d070427c8f7b","uniqueAttributeVsValueMap":{"rduInstrumentId":"RDUNIN000000000001"},"lastModifiedDate":"2019-08-29T14:39:47.44"}


{"documentId":"5e39c25b7ca0f951cf9b2e2f","uniqueAttributeVsValueMap":{"rduInstrumentId":"RDUNIN000000000003"},"lastModifiedDate":"2020-01-04T14:39:47.44"}

ObjectId("5e39c25b7ca0f951cf9b2e2f")
ObjectId("5c6d541775ecec105e4243fe")

{"taskName" : "EQUITY_FULL"}

ObjectId("5e39aec87986ed55991af5da")

-Dspark.mongodb.input.uri=mongodb://127.0.0.1/Reuters.proformaIns

C:\Kafka\kafka_2.12-2.2.0\bin\windows>kafka-console-producer.bat --broker-list localhost:9092 --topic ProformaDistributionQueue

kafka-console-producer.bat --broker-list localhost:9092 --topic ProformaQueue

-DIGNITE_H2_DEBUG_CONSOLE=true



./kafka-run-class.sh kafka.admin.ConsumerGroupCommand --bootstrap-server localhost:9092 --group UDL --describe

./kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list ec2-52-56-162-60.eu-west-2.compute.amazonaws.com:9092 --topic ProformaQueue

hostname -a

scp hdpdata1.abc.com:/home/hadoop/SftrRcc/output/full/ESMA/ESMA_FULL.20200317.00004-46092989-b51a-4a70-9fe6-9c100bb09c89-c000.csv.tsv /home/hadoop

C:\Kafka\kafka_2.12-2.2.0\bin\windows>kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic FigiRequestQueue --from-beginning >E:\JIRA\UDM-36972\message.txt


Take message backup from queue::
./kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic UdlQueue --from-beginning > udlMessage.txt


kafka-simple-consumer-shell.cmd --bootstrap-server localhost:9092 --topic test--partition testPartition --offset 0

./kafka-console-consumer.sh --topic StreamingUdlQueue --bootstrap-server localhost:9092 --from-beginning > StreamingUdlQueue.txt &


https://confluence.-stp.com/display/UDM/NgUDM+-+Integration+environment+details

download from server using scp
scp -r udm@XXX.168.160.66:/home/udm/AUTOMATION/git/automation.zip ./

scp -r udm@XXX.168.118.203:/daq01/dis01/ABC/equity/2022/01/ANNA_FULL/20220115_1642276864275.zip /home/udm/sftrRccLoader/input/full/ANNA

scp -r udm@XXX.168.118.203:/daq01/dis01/ABC/equity/2022/01/ESMA_FULL/20220115_1642276804214.zip.zip /home/udm/sftrRccLoader/input/full/ESMA

20220115_1642276804214.zip

wget http://XXX.168.118.93:8080/jenkins/view/For_Developer_only/job/Local_SftrRccLoader/3/artifact/SftrRccLoader/target/SftrRccLoader-1.0-3-dist.zip

db.setProfilingLevel(0)
db.system.profile.drop()
db.setProfilingLevel(2)
db.system.profile.find().pretty()

./kafka-console-producer.sh --broker-list localhost:9092 --property "parse.key=true" --property "key.separator=|" --topic DataEnrichmentQueue < /home/udm/SEN/DataEnrichmentQueueModified.json




kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic XrfQueue --from-beginning --formatter kafka.tools.DefaultMessageFormatter --property print.key=true --property print.value=true --property key.deserialzer=org.apache.kafka.common.serialization.StringDeserializer --property value.deserializer=org.apache.kafka.common.serialization.StringDeserializer