Required details

---------------------------------------------------------------------------------------------------------------------------

C:\Program Files\MongoDB\Server\4.4\bin>mongod.exe --dbpath=E:\mongodb

C:\Kafka\kafka_2.12-2.2.0\zookeeper-3.4.14\bin>zkServer.cmd

C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows>kafka-server-start.bat C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\config\server.properties

C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows>kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue

kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue
kafka-console-producer.bat --broker-list localhost:9092 --topic FigiDataLoadQueue
kafka-console-producer.bat --broker-list localhost:9092 --topic StreamingUdlQueue


kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue
kafka-console-consumer --bootstrap-server localhost:9092 --topic topicname --offset 152582 --property print.key=true --property key.separator=|

---------------------------------------------------------------------------------------------------------------------------

mongod.exe --dbpath=E:\mongodb

mongod.exe --config=C:\Program Files\MongoDB\Server\3.6\mongod.conf --dbpath=e:\mongodb


start mongodb : C:\Program Files\MongoDB\Server\3.6\bin>mongod.exe

start zookeeper : C:\Kafka\kafka_2.12-2.2.0\zookeeper-3.4.14\bin>zkServer.cmd

C:\Zookeeper\zookeeper-3.4.14\bin>zkServer.cmd

start kafka broker C:\Kafka\kafka_2.12-2.2.0>C:\Kafka\kafka_2.12-2.2.0\bin\windows\kafka-server-start.bat C:\Kafka\kafka_2.12-2.2.0\config\server.properties


C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin


C:\Kafka\kafka2.4.1>C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows\kafka-server-start.bat C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\config\server.properties

C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows\kafka-server-start.bat C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\config\server.properties

C:\Kafka\kafka2.4.1\kafka_2.12-2.4.1\bin\windows


(master)$ git tag –a 1.4.1 –m "Tagging 1.4.1 release."

(master)$ git push --tags

https://git-scm.com/book/en/v2/Git-Basics-Tagging


start producer console : C:\Kafka\kafka_2.12-2.2.0\bin\windows>kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue

kafka-console-producer.bat --broker-list localhost:9092 --topic UdlQueue
kafka-console-producer.bat --broker-list localhost:9092 --topic FigiDataLoadQueue
kafka-console-producer.bat --broker-list localhost:9092 --topic StreamingUdlQueue

EnableRaisingExceptions true 16

IP:XXX.168.XXX.180
user/password:hadoop/hadoop
/home/hadoop/irfan/soft/Vinish
to start
cd RCCLoader-null
nohup ./run.sh &
to check logs
tail -100f ../logs/RCCLoader.log

files path
/home/hadoop/loader_files/si_watcher/done
cp application.properties ../../RCCLoader-null/config/
chmod 755 run.sh

hbase shell
to cont--- count 'tbalename'



http://camel.apache.org/tracer.html

XXX.16x.116.XXX udm - pa$$0rd123

Kafka1 XXX.168.116.140
udm/pa$$0rd123

Log location : /home/udm/PROD/SparkLoader/logs


ec.XXXX.eu-west-2.compute.amazonaws.com:8088/


select count(*) from "dvDomainCache".DVDOMAIN where DOMAINSOURCE='trds' or DOMAINSOURCE='figi' or DOMAINSOURCE='esma' or DOMAINSOURCE='idcApex'

select * from "dvDomainCache".DVDOMAIN where DOMAINSOURCE='esma' and DOMAINNAME='currencyMap7' and NORMALIZEDVALUE='MDL'

kafka-console-consumer.bat --bootstrap-server localhost:9092 test --from-beginning

kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic FigiRequestQueue --from-beginning

-DIGNITE_H2_DEBUG_CONSOLE=true




message from UDL for figi
{"isin":"US0455281065","sedol":"BZ6VMW7","cusip":"045528106","tradeCurrencyCode":"USD","exchangeCode":"XNYS","assetType":"5101", "changedIdentifiers":["sedol","isin","cusip","exchangeCode","tradeCurrencyCode","instrumentTypeCode"], "dependentMics":[],"micIndex":-1}


message for ProformaQueue
{"documentId":"5d67e3b396391f46940a9943","uniqueAttributeVsValueMap":{"rduInstrumentId":"RDUNIN000000000181"},"lastModifiedDate":"2019-08-29T14:39:47.44"}


{"documentId":"5e39ac659ef5d070427c8f7b","uniqueAttributeVsValueMap":{"rduInstrumentId":"RDUNIN000000000001"},"lastModifiedDate":"2019-08-29T14:39:47.44"}


{"documentId":"5e39c25b7ca0f951cf9b2e2f","uniqueAttributeVsValueMap":{"rduInstrumentId":"RDUNIN000000000003"},"lastModifiedDate":"2020-01-04T14:39:47.44"}

ObjectId("5e39c25b7ca0f951cf9b2e2f")
ObjectId("5c6d541775ecec105e4243fe")

{"taskName" : "EQUITY_FULL"}

ObjectId("5e39aec87986ed55991af5da")

-Dspark.mongodb.input.uri=mongodb://127.0.0.1/Reuters.proformaIns

C:\Kafka\kafka_2.12-2.2.0\bin\windows>kafka-console-producer.bat --broker-list localhost:9092 --topic ProformaDistributionQueue

kafka-console-producer.bat --broker-list localhost:9092 --topic ProformaQueue

-DIGNITE_H2_DEBUG_CONSOLE=true



./kafka-run-class.sh kafka.admin.ConsumerGroupCommand --bootstrap-server localhost:9092 --group UDL --describe

./kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list ec2-52-56-162-60.eu-west-2.compute.amazonaws.com:9092 --topic ProformaQueue

hostname -a

scp hdpdata1.abc.com:/home/hadoop/SftrRcc/output/full/ESMA/ESMA_FULL.20200317.00004-46092989-b51a-4a70-9fe6-9c100bb09c89-c000.csv.tsv /home/hadoop

C:\Kafka\kafka_2.12-2.2.0\bin\windows>kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic FigiRequestQueue --from-beginning >E:\JIRA\UDM-36972\message.txt


Take message backup from queue::
./kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic UdlQueue --from-beginning > udlMessage.txt


kafka-simple-consumer-shell.cmd --bootstrap-server localhost:9092 --topic test--partition testPartition --offset 0

./kafka-console-consumer.sh --topic StreamingUdlQueue --bootstrap-server localhost:9092 --from-beginning > StreamingUdlQueue.txt &


https://confluence.-stp.com/display/UDM/NgUDM+-+Integration+environment+details

download from server using scp
scp -r udm@XXX.168.160.66:/home/udm/AUTOMATION/git/automation.zip ./

scp -r udm@XXX.168.118.203:/daq01/dis01/ABC/equity/2022/01/ANNA_FULL/20220115_1642276864275.zip /home/udm/sftrRccLoader/input/full/ANNA

scp -r udm@XXX.168.118.203:/daq01/dis01/ABC/equity/2022/01/ESMA_FULL/20220115_1642276804214.zip.zip /home/udm/sftrRccLoader/input/full/ESMA

20220115_1642276804214.zip

wget http://XXX.168.118.93:8080/jenkins/view/For_Developer_only/job/Local_SftrRccLoader/3/artifact/SftrRccLoader/target/SftrRccLoader-1.0-3-dist.zip

db.setProfilingLevel(0)
db.system.profile.drop()
db.setProfilingLevel(2)
db.system.profile.find().pretty()

./kafka-console-producer.sh --broker-list localhost:9092 --property "parse.key=true" --property "key.separator=|" --topic DataEnrichmentQueue < /home/udm/SEN/DataEnrichmentQueueModified.json




kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic XrfQueue --from-beginning --formatter kafka.tools.DefaultMessageFormatter --property print.key=true --property print.value=true --property key.deserialzer=org.apache.kafka.common.serialization.StringDeserializer --property value.deserializer=org.apache.kafka.common.serialization.StringDeserializer

Set environment variable on macOS 11 Big Sur (zsh)

 On macOs 10.15 and later , default terminal is zsh. follow are the steps :

 vi ~/.zshenv_profile

  Add env variable path

JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk-11.0.12.jdk/Contents/Home

export JAVA_HOME

M2_HOME=/opt/apache-maven-3.8.1

export M2_HOME

PATH=$PATH:$JAVA_HOME/bin:$M2_HOME/bin

Export PATH

source ~/.zshenv_profile

sudo: source: command not found

vinish@Vinishs-MacBook-Pro ~ % vi ~/.zshenv_profile       

vinish@Vinishs-MacBook-Pro ~ % source ~/.zshenv_profile 

vinish@Vinishs-MacBook-Pro ~ % mvn -version

Apache Maven 3.8.1 (05c21c65bdfed0f71a2f2ada8b84da59348c4c5d)

Maven home: /opt/apache-maven-3.8.1

Java version: 11.0.12, vendor: Oracle Corporation, runtime: /Library/Java/JavaVirtualMachines/jdk-11.0.12.jdk/Contents/Home

Default locale: en_IN, platform encoding: UTF-8

OS name: "mac os x", version: "11.4", arch: "x86_64", family: "mac"

vinish@Vinishs-MacBook-Pro ~ % echo $JAVA_HOME

/Library/Java/JavaVirtualMachines/jdk-11.0.12.jdk/Contents/Home

vinish@Vinishs-MacBook-Pro ~ % mvn -version

Data Enginner Syllabus

https://www.analyticsvidhya.com/blog/2018/11/data-engineer-comprehensive-list-resources-get-started/

https://www.analyticsvidhya.com/blog/2016/09/comprehensive-introduction-to-apache-spark-rdds-dataframes-using-pyspark/

Migrate to Java 11

https://blog.codefx.org/java/java-11-migration-guide/

https://blog.codefx.org/java/java-11-gems/

https://developer.ibm.com/technologies/java/tutorials/migration-to-java-11-made-easy/

Maven Shaded Plugin

This article is about the what is maven shaded plugin and where we should use it.

maven-shade-plugin is used for the multiple purpose :

•  It can be used to generate the Uber jar and select the content of the Uber jar
•  Relocating the classes (means reassign the package for the given class)
  
• Make the executable jar.

In our day to day activity , we faced a problem that we were using jira-rest-java-client-core-5.2.0 with spark-streaming_2.11-2.3.0 , and both of them carries different version of guava

jira-rest-java-client-core-5.2.0(guava-26-jre) and spark-streaming_2.11-2.3.0(guava-14.0.1), In straight way I can't just exclude the any one of them , we need both of them , because guava 26-jre(some of the class and method is removed which was in 14.0.1 version)  is not compatible with guava-14.0.1.

So to over come with problem , only solution left is to keep both guava version and with the help of maven plugin m we will rename all package name com.google.common with shaded.com.google.common inside  jira-rest-java-client-core-5.2.0, this way we will be able to use both of the guava version and it wont conflict with each other.

Following are the example of one project which does 

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0"
 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 <modelVersion>4.0.0</modelVersion>

 <properties>
 <shade.groupId>com.xyz.util</shade.groupId>
 <shade.artifactId>jira-rest-java-client-core-shaded</shade.artifactId>
 <shade.version>5.2.0-SHADED</shade.version>
 <shade.packaging>jar</shade.packaging>
 <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 </properties>
 
 <groupId>${shade.groupId}</groupId>
 <artifactId>${shade.artifactId}</artifactId>
 <version>${shade.version}</version>
 <packaging>${shade.packaging}</packaging>

 <name>JiraRestJavaClientCoreShaded</name>
 <url>http://maven.apache.org</url>

 <repositories>
  <repository>
   <id>maven-public</id>
   <url>https://repo.maven.apache.org/maven2/</url>
  </repository>
  <repository>
   <id>atlassian-public</id>
   <url>https://packages.atlassian.com/maven/repository/public</url>
  </repository>
 </repositories>

 <dependencies>
  <dependency>
   <groupId>com.atlassian.jira</groupId>
   <artifactId>jira-rest-java-client-core</artifactId>
   <version>5.2.0</version>
  </dependency>
 </dependencies>

 <build>
 <!-- Below plugin will shade the artifacts guava-26.0-jre which is transitive dependency of jira-rest-java-client-core  -->
  <plugins>
   <plugin>
    <groupId>org.apache.maven.plugins</groupId>
    <artifactId>maven-shade-plugin</artifactId>
    <version>3.2.0</version>
    <executions>
     <execution>
      <phase>package</phase>
      <goals>
       <goal>shade</goal>
      </goals>
      <configuration>
      
       <!-- <shadedArtifactAttached>true</shadedArtifactAttached> -->
       <shadedClassifierName>shaded</shadedClassifierName>
       
       <relocations>
        <relocation>
         <pattern>com.google.common</pattern>
         <shadedPattern>shaded.com.google.common</shadedPattern>
        </relocation>
       </relocations>
       
       <transformers>
        <transformer
         implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" />
       </transformers>

       <!-- Exclude below artifacts from the uber jar-->
       <artifactSet>
        <excludes>
         <exclude>org.springframework:*</exclude>
         <exclude>org.apache.commons:*</exclude>
         <exclude>commons-logging:*</exclude>
         <exclude>org.codehaus.jackson:*</exclude>
        </excludes>
       </artifactSet>
      </configuration>
     </execution>
    </executions>
    
    <!-- Below plugin install the artifacts into maven repository so we wont need the scope as system for the above shaded jar  -->
   </plugin>
   
    <plugin>
     <groupId>org.apache.maven.plugins</groupId>
     <artifactId>maven-install-plugin</artifactId>
     <version>2.5.2</version>
     <executions>
      <execution>
       <id>install-external</id>
       <phase>package</phase>
       <configuration>
        <file>${project.build.directory}/${shade.artifactId}-${shade.version}.jar</file>
        <repositoryLayout>default</repositoryLayout>
        <groupId>${shade.groupId}</groupId>
        <artifactId>${shade.artifactId}</artifactId>
        <version>${shade.version}-JAR</version>
        <packaging>${shade.packaging}</packaging>
        <packaging>jar</packaging>
        <generatePom>true</generatePom>
       </configuration>
       <goals>
        <goal>install-file</goal>
       </goals>
      </execution>
     </executions>
    </plugin>
  </plugins>
 </build>
</project>

The above pom will shade it and if you run maven goal mvn clean install then it will install into local repositories.

Then in you main project you can use this shaded jar as below:

<dependency>
    <groupId>com.smartstreamrdu.libraries</groupId>
    <artifactId>jira-rest-java-client-core-shaded</artifactId>
    <version>5.2.0-SHADED-JAR</version>
    <type>jar</type>
 </dependency>

This way we can keep both the guava version and it will work as expected.

Overview of AI Libraries in Java

https://www.baeldung.com/java-ai 

Topic to be covered on coming days

1. Ignite

2. Redis

3. ImportTsv

4. Camel 

5. Spark