SUPPORT-9122:add kafka

This commit is contained in:
adel.kalimullin 2025-04-17 14:11:31 +03:00
parent 63b3e36d69
commit 0657acebf5
10 changed files with 1101 additions and 0 deletions

View file

@ -167,6 +167,14 @@
<groupId>org.postgresql</groupId> <groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
</dependencies> </dependencies>
<build> <build>
<finalName>${project.parent.artifactId}</finalName> <finalName>${project.parent.artifactId}</finalName>

View file

@ -22,8 +22,10 @@ import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.context.annotation.FilterType; import org.springframework.context.annotation.FilterType;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.servlet.config.annotation.EnableWebMvc; import org.springframework.web.servlet.config.annotation.EnableWebMvc;
/** /**
@ -117,4 +119,9 @@ public class AppConfig {
throw new AppInitializeException(e); throw new AppInitializeException(e);
} }
} }
@Bean
public RestTemplate restTemplate() {
return new RestTemplate();
}
} }

View file

@ -0,0 +1,62 @@
package ervu_business_metrics.kafka;
import java.util.HashMap;
import java.util.Map;
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.config.SaslConfigs;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
/**
* @author Adel Kalimullin
*/
@Configuration
@EnableKafka
public class KafkaConfig {
@Value("${kafka.hosts}")
private String bootstrapServers;
@Value("${kafka.auth_sec_proto}")
private String securityProtocol;
@Value("${kafka.auth_sasl_module}")
private String loginModule;
@Value("${kafka.user}")
private String username;
@Value("${kafka.pass}")
private String password;
@Value("${kafka.auth_sasl_mech}")
private String saslMechanism;
@Bean
public ConsumerFactory<String, String> consumerFactory() {
return new DefaultKafkaConsumerFactory<>(consumerConfigs());
}
@Bean
public Map<String, Object> consumerConfigs() {
Map<String, Object> props = new HashMap<>();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol);
props.put(SaslConfigs.SASL_JAAS_CONFIG, loginModule + " required username=\""
+ username + "\" password=\"" + password + "\";");
props.put(SaslConfigs.SASL_MECHANISM, saslMechanism);
return props;
}
@Bean
public ConcurrentKafkaListenerContainerFactory<String, String> kafkaListenerContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(consumerFactory());
return factory;
}
}

View file

@ -0,0 +1,33 @@
package ervu_business_metrics.kafka.listener;
import ervu_business_metrics.service.IdmDirectoriesService;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;
/**
* @author Adel Kalimullin
*/
@Component
public class IdmDirectoriesListener {
private IdmDirectoriesService idmDirectoriesService;
@KafkaListener(id = "${kafka.domain.group.id}", topics = "${kafka.domain.reconciliation}")
public void listenKafkaDomain(String kafkaMessage) {
idmDirectoriesService.upsertKafkaDomainMessage(kafkaMessage);
}
@KafkaListener(id = "${kafka.role.group.id}", topics = "${kafka.role.reconciliation}")
public void listenKafkaRole(String kafkaMessage) {
idmDirectoriesService.upsertKafkaRoleMessage(kafkaMessage);
}
@KafkaListener(id = "${kafka.account.group.id}", topics = "${kafka.account.reconciliation}")
public void listenKafkaAccount(String kafkaMessage) {
idmDirectoriesService.upsertKafkaAccountMessage(kafkaMessage);
}
@KafkaListener(id = "${kafka.person.group.id}", topics = "${kafka.person.reconciliation}")
public void listenKafkaPerson(String kafkaMessage) {
idmDirectoriesService.upsertKafkaPersonMessage(kafkaMessage);
}
}

View file

@ -0,0 +1,184 @@
package ervu_business_metrics.model.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* @author Adel Kalimullin
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class AccountResponse {
private List<Data> data;
public List<Data> getData() {
return data;
}
public void setData(List<Data> data) {
this.data = data;
}
@JsonIgnoreProperties(ignoreUnknown = true)
public static class Data {
private String id;
private int version;
private long modified;
private String schema;
private String start;
private String finish;
private boolean enabled;
private String position;
private String fio;
private String workMail;
private boolean esiaAccount;
private String userDomainId;
private String personId;
@JsonProperty("user-domain")
private ReferenceEntity userDomain;
private ReferenceEntity person;
private List<String> roles;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public int getVersion() {
return version;
}
public void setVersion(int version) {
this.version = version;
}
public long getModified() {
return modified;
}
public void setModified(long modified) {
this.modified = modified;
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
public String getStart() {
return start;
}
public void setStart(String start) {
this.start = start;
}
public String getFinish() {
return finish;
}
public void setFinish(String finish) {
this.finish = finish;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getPosition() {
return position;
}
public void setPosition(String position) {
this.position = position;
}
public String getFio() {
return fio;
}
public void setFio(String fio) {
this.fio = fio;
}
public String getWorkMail() {
return workMail;
}
public void setWorkMail(String workMail) {
this.workMail = workMail;
}
public boolean isEsiaAccount() {
return esiaAccount;
}
public void setEsiaAccount(boolean esiaAccount) {
this.esiaAccount = esiaAccount;
}
public String getUserDomainId() {
return userDomainId;
}
public void setUserDomainId(String userDomainId) {
this.userDomainId = userDomainId;
}
public String getPersonId() {
return personId;
}
public void setPersonId(String personId) {
this.personId = personId;
}
public ReferenceEntity getUserDomain() {
return userDomain;
}
public void setUserDomain(ReferenceEntity userDomain) {
this.userDomain = userDomain;
}
public ReferenceEntity getPerson() {
return person;
}
public void setPerson(ReferenceEntity person) {
this.person = person;
}
public List<String> getRoles() {
return roles;
}
public void setRoles(List<String> roles) {
this.roles = roles;
}
}
@JsonIgnoreProperties(ignoreUnknown = true)
public static class ReferenceEntity {
private String id;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}
}

View file

@ -0,0 +1,474 @@
package ervu_business_metrics.model.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* @author Eduard Tihomirov
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class DomainResponse{
private List<Data> data;
public List<Data> getData() {
return data;
}
public void setData(List<Data> data) {
this.data = data;
}
@JsonIgnoreProperties(ignoreUnknown = true)
public static class Data {
private String id;
private int version;
private long modified;
private String schema;
private String name;
private String shortname;
private String fullname;
private String dns;
private String email;
private String phone;
private String address;
private String postalAddress;
private String addressId;
private String postalAddressId;
private String militaryCode;
private String timezone;
private boolean reportsEnabled;
private String inn;
private String leg;
private String ogrn;
private String region;
private String epguId;
private String type;
private boolean esiaEmployeeAuthorization;
private String defaultS3Bucket;
private String opf;
private String kpp;
private String checkingAccount;
private String bik;
private String bankName;
private String bankCorrespondentAccount;
private String oktmo;
private String okato;
private String govRegistrationDate;
private String govOrganizationType;
private String aliasKey;
private String passKey;
private String certificate;
private String accountNumberTOFK;
private String bikTOFK;
private String correspondentBankAccountTOFK;
private String nameTOFK;
private String nsiOrganizationId;
private String docHandle;
private String divisionType;
private String tnsDepartmentId;
private boolean enabled;
private String parent;
private String regionId;
private String managed;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public int getVersion() {
return version;
}
public void setVersion(int version) {
this.version = version;
}
public long getModified() {
return modified;
}
public void setModified(long modified) {
this.modified = modified;
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getShortname() {
return shortname;
}
public void setShortname(String shortname) {
this.shortname = shortname;
}
public String getFullname() {
return fullname;
}
public void setFullname(String fullname) {
this.fullname = fullname;
}
public String getDns() {
return dns;
}
public void setDns(String dns) {
this.dns = dns;
}
public String getEmail() {
return email;
}
public void setEmail(String email) {
this.email = email;
}
public String getPhone() {
return phone;
}
public void setPhone(String phone) {
this.phone = phone;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public String getPostalAddress() {
return postalAddress;
}
public void setPostalAddress(String postalAddress) {
this.postalAddress = postalAddress;
}
public String getAddressId() {
return addressId;
}
public void setAddressId(String addressId) {
this.addressId = addressId;
}
public String getPostalAddressId() {
return postalAddressId;
}
public void setPostalAddressId(String postalAddressId) {
this.postalAddressId = postalAddressId;
}
public String getMilitaryCode() {
return militaryCode;
}
public void setMilitaryCode(String militaryCode) {
this.militaryCode = militaryCode;
}
public String getTimezone() {
return timezone;
}
public void setTimezone(String timezone) {
this.timezone = timezone;
}
public boolean isReportsEnabled() {
return reportsEnabled;
}
public void setReportsEnabled(boolean reportsEnabled) {
this.reportsEnabled = reportsEnabled;
}
public String getInn() {
return inn;
}
public void setInn(String inn) {
this.inn = inn;
}
public String getLeg() {
return leg;
}
public void setLeg(String leg) {
this.leg = leg;
}
public String getOgrn() {
return ogrn;
}
public void setOgrn(String ogrn) {
this.ogrn = ogrn;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
public String getEpguId() {
return epguId;
}
public void setEpguId(String epguId) {
this.epguId = epguId;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public boolean isEsiaEmployeeAuthorization() {
return esiaEmployeeAuthorization;
}
public void setEsiaEmployeeAuthorization(boolean esiaEmployeeAuthorization) {
this.esiaEmployeeAuthorization = esiaEmployeeAuthorization;
}
public String getDefaultS3Bucket() {
return defaultS3Bucket;
}
public void setDefaultS3Bucket(String defaultS3Bucket) {
this.defaultS3Bucket = defaultS3Bucket;
}
public String getOpf() {
return opf;
}
public void setOpf(String opf) {
this.opf = opf;
}
public String getKpp() {
return kpp;
}
public void setKpp(String kpp) {
this.kpp = kpp;
}
public String getCheckingAccount() {
return checkingAccount;
}
public void setCheckingAccount(String checkingAccount) {
this.checkingAccount = checkingAccount;
}
public String getBik() {
return bik;
}
public void setBik(String bik) {
this.bik = bik;
}
public String getBankName() {
return bankName;
}
public void setBankName(String bankName) {
this.bankName = bankName;
}
public String getBankCorrespondentAccount() {
return bankCorrespondentAccount;
}
public void setBankCorrespondentAccount(String bankCorrespondentAccount) {
this.bankCorrespondentAccount = bankCorrespondentAccount;
}
public String getOktmo() {
return oktmo;
}
public void setOktmo(String oktmo) {
this.oktmo = oktmo;
}
public String getOkato() {
return okato;
}
public void setOkato(String okato) {
this.okato = okato;
}
public String getGovRegistrationDate() {
return govRegistrationDate;
}
public void setGovRegistrationDate(String govRegistrationDate) {
this.govRegistrationDate = govRegistrationDate;
}
public String getGovOrganizationType() {
return govOrganizationType;
}
public void setGovOrganizationType(String govOrganizationType) {
this.govOrganizationType = govOrganizationType;
}
public String getAliasKey() {
return aliasKey;
}
public void setAliasKey(String aliasKey) {
this.aliasKey = aliasKey;
}
public String getPassKey() {
return passKey;
}
public void setPassKey(String passKey) {
this.passKey = passKey;
}
public String getCertificate() {
return certificate;
}
public void setCertificate(String certificate) {
this.certificate = certificate;
}
public String getAccountNumberTOFK() {
return accountNumberTOFK;
}
public void setAccountNumberTOFK(String accountNumberTOFK) {
this.accountNumberTOFK = accountNumberTOFK;
}
public String getBikTOFK() {
return bikTOFK;
}
public void setBikTOFK(String bikTOFK) {
this.bikTOFK = bikTOFK;
}
public String getCorrespondentBankAccountTOFK() {
return correspondentBankAccountTOFK;
}
public void setCorrespondentBankAccountTOFK(String correspondentBankAccountTOFK) {
this.correspondentBankAccountTOFK = correspondentBankAccountTOFK;
}
public String getNameTOFK() {
return nameTOFK;
}
public void setNameTOFK(String nameTOFK) {
this.nameTOFK = nameTOFK;
}
public String getNsiOrganizationId() {
return nsiOrganizationId;
}
public void setNsiOrganizationId(String nsiOrganizationId) {
this.nsiOrganizationId = nsiOrganizationId;
}
public String getDocHandle() {
return docHandle;
}
public void setDocHandle(String docHandle) {
this.docHandle = docHandle;
}
public String getDivisionType() {
return divisionType;
}
public void setDivisionType(String divisionType) {
this.divisionType = divisionType;
}
public String getTnsDepartmentId() {
return tnsDepartmentId;
}
public void setTnsDepartmentId(String tnsDepartmentId) {
this.tnsDepartmentId = tnsDepartmentId;
}
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getParent() {
return parent;
}
public void setParent(String parent) {
this.parent = parent;
}
public String getRegionId() {
return regionId;
}
public void setRegionId(String regionId) {
this.regionId = regionId;
}
public String getManaged() {
return managed;
}
public void setManaged(String managed) {
this.managed = managed;
}
}
}

View file

@ -0,0 +1,169 @@
package ervu_business_metrics.model.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* @author Adel Kalimullin
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class PersonResponse {
private List<Data> data;
public List<Data> getData() {
return data;
}
public void setData(List<Data> data) {
this.data = data;
}
@JsonIgnoreProperties(ignoreUnknown = true)
public static class Data {
private String id;
private int version;
private long modified;
private String schema;
private String birthdate;
private String firstname;
private String middlename;
private String surname;
private String sex;
private String email;
private String photo;
private String phone;
private String snils;
private boolean secondFactorEnabled;
private List<String> ipAddresses;
private String fio;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public int getVersion() {
return version;
}
public void setVersion(int version) {
this.version = version;
}
public long getModified() {
return modified;
}
public void setModified(long modified) {
this.modified = modified;
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
public String getBirthdate() {
return birthdate;
}
public void setBirthdate(String birthdate) {
this.birthdate = birthdate;
}
public String getFirstname() {
return firstname;
}
public void setFirstname(String firstname) {
this.firstname = firstname;
}
public String getMiddlename() {
return middlename;
}
public void setMiddlename(String middlename) {
this.middlename = middlename;
}
public String getSurname() {
return surname;
}
public void setSurname(String surname) {
this.surname = surname;
}
public String getSex() {
return sex;
}
public void setSex(String sex) {
this.sex = sex;
}
public String getEmail() {
return email;
}
public void setEmail(String email) {
this.email = email;
}
public String getPhoto() {
return photo;
}
public void setPhoto(String photo) {
this.photo = photo;
}
public String getPhone() {
return phone;
}
public void setPhone(String phone) {
this.phone = phone;
}
public String getSnils() {
return snils;
}
public void setSnils(String snils) {
this.snils = snils;
}
public boolean isSecondFactorEnabled() {
return secondFactorEnabled;
}
public void setSecondFactorEnabled(boolean secondFactorEnabled) {
this.secondFactorEnabled = secondFactorEnabled;
}
public List<String> getIpAddresses() {
return ipAddresses;
}
public void setIpAddresses(List<String> ipAddresses) {
this.ipAddresses = ipAddresses;
}
public String getFio() {
return fio;
}
public void setFio(String fio) {
this.fio = fio;
}
}
}

View file

@ -0,0 +1,124 @@
package ervu_business_metrics.model.dto;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
/**
* @author Eduard Tihomirov
*/
@JsonIgnoreProperties(ignoreUnknown = true)
public class RoleResponse {
private List<Data> data;
public List<Data> getData() {
return data;
}
public void setData(List<Data> data) {
this.data = data;
}
@JsonIgnoreProperties(ignoreUnknown = true)
public static class Data {
private String id;
private int version;
private long modified;
private String schema;
private String name;
private String shortname;
private String displayName;
private int sessionsLimit;
private boolean ervuRole;
private int imported;
private String description;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public int getVersion() {
return version;
}
public void setVersion(int version) {
this.version = version;
}
public long getModified() {
return modified;
}
public void setModified(long modified) {
this.modified = modified;
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getShortname() {
return shortname;
}
public void setShortname(String shortname) {
this.shortname = shortname;
}
public String getDisplayName() {
return displayName;
}
public void setDisplayName(String displayName) {
this.displayName = displayName;
}
public int getSessionsLimit() {
return sessionsLimit;
}
public void setSessionsLimit(int sessionsLimit) {
this.sessionsLimit = sessionsLimit;
}
public boolean isErvuRole() {
return ervuRole;
}
public void setErvuRole(boolean ervuRole) {
this.ervuRole = ervuRole;
}
public int getImported() {
return imported;
}
public void setImported(int imported) {
this.imported = imported;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}
}

View file

@ -5,3 +5,20 @@ DB_APP_PASSWORD=ervu_business_metrics
DB_APP_HOST=10.10.31.119 DB_APP_HOST=10.10.31.119
DB_APP_PORT=5432 DB_APP_PORT=5432
DB_APP_NAME=ervu-dashboard-recr DB_APP_NAME=ervu-dashboard-recr
#Kafka
KAFKA_HOSTS=10.10.31.11:32609
KAFKA_AUTH_SEC_PROTO=SASL_PLAINTEXT
KAFKA_AUTH_SASL_MECH=SCRAM-SHA-256
KAFKA_AUTH_SASL_MODULE=org.apache.kafka.common.security.scram.ScramLoginModule
KAFKA_USER=user1
KAFKA_PASS=Blfi9d2OFG
KAFKA_DOMAIN_GROUP_ID=ervu-business-metrics-backend-domain
KAFKA_ROLE_GROUP_ID=ervu-business-metrics-backend-role
KAFKA_ACCOUNT_GROUP_ID=ervu-business-metrics-backend-account
KAFKA_ROLE_RECONCILIATION=idmv2.role.reconciliation
KAFKA_DOMAIN_RECONCILIATION=idmv2.domain.reconciliation
KAFKA_ACCOUNT_RECONCILIATION=idmv2.account.reconciliation
KAFKA_PERSON_RECONCILIATION=idmv2.person.reconciliation
KAFKA_CREDENTIAL_RECONCILIATION=idmv2.credential.reconciliation
IDM_URL=http://idm

23
pom.xml
View file

@ -15,6 +15,7 @@
<connection>scm:git:git://gitserver/webbpm/webbpm-components.git</connection> <connection>scm:git:git://gitserver/webbpm/webbpm-components.git</connection>
</scm> </scm>
<properties> <properties>
<spring-kafka.version>2.9.13</spring-kafka.version>
<org.bouncycastle.version>1.60</org.bouncycastle.version> <org.bouncycastle.version>1.60</org.bouncycastle.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<enable.version.in.url>false</enable.version.in.url> <enable.version.in.url>false</enable.version.in.url>
@ -285,6 +286,28 @@
<artifactId>log4j-web</artifactId> <artifactId>log4j-web</artifactId>
<version>2.23.1</version> <version>2.23.1</version>
</dependency> </dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
<version>${spring-kafka.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>3.9.0</version>
<exclusions>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies> </dependencies>
</dependencyManagement> </dependencyManagement>
<repositories> <repositories>