diff --git a/Tribler/Test/Community/Market/Reputation/test_reputation_base.py b/Tribler/Test/Community/Market/Reputation/test_reputation_base.py index 2c69b1cc030..7a0f4adec2d 100644 --- a/Tribler/Test/Community/Market/Reputation/test_reputation_base.py +++ b/Tribler/Test/Community/Market/Reputation/test_reputation_base.py @@ -2,6 +2,7 @@ from Tribler.Test.test_as_server import AbstractServer from Tribler.community.market.database import MarketDB +from Tribler.community.market.reputation.temporal_pagerank_manager import TemporalPagerankReputationManager from Tribler.community.market.tradechain.block import TradeChainBlock @@ -16,18 +17,24 @@ def setUp(self, annotate=True): os.mkdir(os.path.join(self.session_base_dir, 'sqlite')) self.market_db = MarketDB(self.session_base_dir, 'market') - def insert_transaction(self, pubkey1, pubkey2, asset1_type, asset1_amount, asset2_type, asset2_amount): - latest_block = self.market_db.get_latest(pubkey1) - - block = TradeChainBlock() - block.public_key = pubkey1 - if latest_block: - block.sequence_number = latest_block.sequence_number + 1 - - block.link_public_key = pubkey2 - - transaction = {"asset1_type": asset1_type, "asset1_amount": asset1_amount, - "asset2_type": asset2_type, "asset2_amount": asset2_amount} - block.transaction = transaction + def insert_transaction(self, pubkey1, pubkey2, quantity, price): + transaction = { + "tx": { + "quantity_type": quantity.wallet_id, + "quantity": float(quantity), + "price_type": price.wallet_id, + "price": float(price) + } + } + block = TradeChainBlock.create(transaction, self.market_db, pubkey1, link=None, link_pk=pubkey2) + link_block = TradeChainBlock.create(transaction, self.market_db, pubkey2, link=block, link_pk=pubkey1) self.market_db.add_block(block) + self.market_db.add_block(link_block) + + def compute_reputations(self): + blocks = self.market_db.get_all_blocks() + rep_manager = TemporalPagerankReputationManager(blocks) + rep = rep_manager.compute(own_public_key='a') + self.assertIsInstance(rep, dict) + return rep diff --git a/Tribler/Test/Community/Market/Reputation/test_reputation_pagerank.py b/Tribler/Test/Community/Market/Reputation/test_reputation_pagerank.py index d1e6ec97260..84bf50b0b00 100644 --- a/Tribler/Test/Community/Market/Reputation/test_reputation_pagerank.py +++ b/Tribler/Test/Community/Market/Reputation/test_reputation_pagerank.py @@ -1,5 +1,6 @@ from Tribler.Test.Community.Market.Reputation.test_reputation_base import TestReputationBase -from Tribler.community.market.reputation.pagerank_manager import PagerankReputationManager +from Tribler.community.market.core.price import Price +from Tribler.community.market.core.quantity import Quantity class TestReputationPagerank(TestReputationBase): @@ -8,13 +9,57 @@ class TestReputationPagerank(TestReputationBase): """ def test_pagerank_1(self): - self.insert_transaction('a', 'b', 1, 20, 2, 20) - self.insert_transaction('a', 'b', 1, 20, 2, 20) - self.insert_transaction('b', 'c', 1, 20, 2, 20) - self.insert_transaction('b', 'd', 1, 20, 2, 20) - self.insert_transaction('d', 'e', 1, 10, 2, 10) + """ + Test a very simple Temporal Pagerank computation + """ + self.insert_transaction('a', 'b', Quantity(1, 'BTC'), Price(1, 'MC')) + rep_dict = self.compute_reputations() + self.assertTrue('a' in rep_dict) + self.assertTrue('b' in rep_dict) + self.assertGreater(rep_dict['a'], 0) + self.assertGreater(rep_dict['b'], 0) - blocks = self.market_db.get_all_blocks() - rep_manager = PagerankReputationManager(blocks) - rep = rep_manager.compute(own_public_key='a') - self.assertIsInstance(rep, dict) + def test_pagerank_2(self): + """ + Test isolated nodes during a Temporal Pagerank computation + """ + self.insert_transaction('a', 'b', Quantity(1, 'BTC'), Price(1, 'MC')) + self.insert_transaction('c', 'd', Quantity(1, 'BTC'), Price(1, 'MC')) + rep_dict = self.compute_reputations() + self.assertTrue('c' in rep_dict) + self.assertTrue('d' in rep_dict) + + def test_pagerank_3(self): + """ + Test a more involved example of a Temporal Pagerank computation + """ + self.insert_transaction('a', 'b', Quantity(1, 'BTC'), Price(1, 'MC')) + self.insert_transaction('b', 'c', Quantity(100, 'BTC'), Price(100, 'MC')) + self.insert_transaction('b', 'd', Quantity(100, 'BTC'), Price(100, 'MC')) + self.insert_transaction('b', 'e', Quantity(100, 'BTC'), Price(100, 'MC')) + rep_dict = self.compute_reputations() + self.assertEqual(len(rep_dict.keys()), 5) + for _, rep in rep_dict.iteritems(): + self.assertGreater(rep, 0) + + def test_pagerank_4(self): + """ + Test an empty pagerank computation + """ + rep_dict = self.compute_reputations() + self.assertDictEqual(rep_dict, {}) + + def test_pagerank_5(self): + """ + Test a Temporal Pagerank computation + """ + self.insert_transaction('a', 'b', Quantity(1, 'BTC'), Price(1, 'MC')) + self.insert_transaction('a', 'c', Quantity(1, 'BTC'), Price(2, 'MC')) + self.insert_transaction('a', 'd', Quantity(1, 'BTC'), Price(3, 'MC')) + self.insert_transaction('a', 'e', Quantity(1, 'BTC'), Price(4, 'MC')) + self.insert_transaction('a', 'f', Quantity(1, 'BTC'), Price(5, 'MC')) + self.insert_transaction('a', 'g', Quantity(1, 'BTC'), Price(6, 'MC')) + self.insert_transaction('a', 'h', Quantity(1, 'BTC'), Price(7, 'MC')) + rep_dict = self.compute_reputations() + self.assertTrue('c' in rep_dict) + self.assertTrue('d' in rep_dict) diff --git a/Tribler/Test/Community/Market/test_community.py b/Tribler/Test/Community/Market/test_community.py index 4a4841bccfe..ea2f5c9fc97 100644 --- a/Tribler/Test/Community/Market/test_community.py +++ b/Tribler/Test/Community/Market/test_community.py @@ -382,9 +382,9 @@ def test_compute_reputation(self): """ Test the compute_reputation method """ - self.market_community.tradechain_community = MockObject() - self.market_community.tradechain_community.persistence = MockObject() - self.market_community.tradechain_community.persistence.get_all_blocks = lambda: [] + self.market_community.persistence = MockObject() + self.market_community.persistence.get_all_blocks = lambda: [] + self.market_community.persistence.close = lambda: None self.market_community.compute_reputation() self.assertFalse(self.market_community.reputation_dict) diff --git a/Tribler/community/market/community.py b/Tribler/community/market/community.py index e56e1aa1ac2..68455b6de67 100644 --- a/Tribler/community/market/community.py +++ b/Tribler/community/market/community.py @@ -33,7 +33,7 @@ from Tribler.community.market.payload import TradePayload, DeclinedTradePayload,\ StartTransactionPayload, WalletInfoPayload, PaymentPayload, MatchPayload, AcceptMatchPayload, DeclineMatchPayload, \ InfoPayload, OrderStatusRequestPayload, OrderStatusResponsePayload -from Tribler.community.market.reputation.pagerank_manager import PagerankReputationManager +from Tribler.community.market.reputation.temporal_pagerank_manager import TemporalPagerankReputationManager from Tribler.community.market.tradechain.block import TradeChainBlock from Tribler.community.market.wallet.tc_wallet import TrustchainWallet from Tribler.community.trustchain.community import TrustChainCommunity, HALF_BLOCK_BROADCAST, BLOCK_PAIR, \ @@ -93,24 +93,24 @@ class MarketCommunity(TrustChainCommunity): @classmethod def get_master_members(cls, dispersy): - # generated: Tue Mar 22 23:29:40 2016 - # curve: NID_sect571r1 + # generated: Thu Sep 21 09:18:42 2017 + # curve: None # len: 571 bits ~ 144 bytes signature - # pub: 170 3081a7301006072a8648ce3d020106052b8104002703819200040159af0c0925034bba3b4ea26661828e09247236059c773 - # dac29ac9fb84d50fa6bd8acc035127a6f5c11873915f9b9a460e116ecccccfc5db1b5d8ba86bd701886ea45d8dbbb634906989395d36 - # 6888d008f4119ad0e7f45b9dab7fb3d78a0065c5f7a866b78cb8e59b9a7d048cc0d650c5a86bdfdabb434396d23945d1239f88de4935 - # 467424c7cc02b6579e45f63ee - # pub-sha1 dda25d128ebabe6b588384d05b8ff46153f98c78 + # pub: 170 3081a7301006072a8648ce3d020106052b81040027038192000407c78a0f74058fe70e6101709389ba198cd7f41826f160b + # 546e3b9ad1c634e041cbd1da82849968f9736ef1d4ccc6c7ebd41ad4c29e6988e99f0f597925dd956b1f7b809fe13fe7702a34f95662 + # 699ad6e4756b169fd6cf61b2658818e0d0e9e144827c1b5c476603b8ba3b21a735f206e9c58a361c53f133de14b9552dfc317627e101 + # 4c901030bca5d88cea35c7489 + # pub-sha1 6317befc5c61e41341ed2a9590980be9ea77eb3b # -----BEGIN PUBLIC KEY----- - # MIGnMBAGByqGSM49AgEGBSuBBAAnA4GSAAQBWa8MCSUDS7o7TqJmYYKOCSRyNgWc - # dz2sKayfuE1Q+mvYrMA1EnpvXBGHORX5uaRg4RbszMz8XbG12LqGvXAYhupF2Nu7 - # Y0kGmJOV02aIjQCPQRmtDn9Fudq3+z14oAZcX3qGa3jLjlm5p9BIzA1lDFqGvf2r - # tDQ5bSOUXRI5+I3kk1RnQkx8wCtleeRfY+4= + # MIGnMBAGByqGSM49AgEGBSuBBAAnA4GSAAQHx4oPdAWP5w5hAXCTiboZjNf0GCbx + # YLVG47mtHGNOBBy9HagoSZaPlzbvHUzMbH69Qa1MKeaYjpnw9ZeSXdlWsfe4Cf4T + # /ncCo0+VZiaZrW5HVrFp/Wz2GyZYgY4NDp4USCfBtcR2YDuLo7Iac18gbpxYo2HF + # PxM94UuVUt/DF2J+EBTJAQMLyl2IzqNcdIk= # -----END PUBLIC KEY----- - master_key = "3081a7301006072a8648ce3d020106052b81040027038192000403e6f247258f60430f570cb02f5d830426fefaec" \ - "76a506db6e806ea0f10ee6061996f54fe6960e19978b32a0c92ece60dc0b85deaa07b7fd13fa6e54205154f78c1a" \ - "294effb43801045fb17124a85e42a338275d109da989942337dbc6c3b06dc2c4c62d0c2b64f2cdfe02aad5c058be" \ - "23027e4b99fc7271a94d176f020543e06da7a371f9794240dae44e9bc130a1a6".decode('hex') + master_key = "3081a7301006072a8648ce3d020106052b81040027038192000407c78a0f74058fe70e6101709389ba198cd7f41826f" \ + "160b546e3b9ad1c634e041cbd1da82849968f9736ef1d4ccc6c7ebd41ad4c29e6988e99f0f597925dd956b1f7b809fe" \ + "13fe7702a34f95662699ad6e4756b169fd6cf61b2658818e0d0e9e144827c1b5c476603b8ba3b21a735f206e9c58a36" \ + "1c53f133de14b9552dfc317627e1014c901030bca5d88cea35c7489".decode('hex') master = dispersy.get_member(public_key=master_key) return [master] @@ -124,7 +124,6 @@ def __init__(self, *args, **kwargs): self.matching_engine = None self.incoming_match_messages = {} # Map of TraderId -> Message (we save all incoming matches) self.tribler_session = None - self.tradechain_community = None self.wallets = None self.transaction_manager = None self.reputation_dict = {} @@ -160,7 +159,6 @@ def initialize(self, tribler_session=None, tradechain_community=None, wallets=No self.order_manager = OrderManager(order_repository) self.tribler_session = tribler_session - self.tradechain_community = tradechain_community self.wallets = wallets or {} self.transaction_manager = TransactionManager(transaction_repository) @@ -916,9 +914,11 @@ def on_match_message(self, messages): if order.status != "open" or order.available_quantity == Quantity(0, order.available_quantity.wallet_id): # Send a declined trade back + decline_reason = DeclineMatchReason.ORDER_COMPLETED if order.status != "open" \ + else DeclineMatchReason.OTHER self.send_decline_match_message(message.payload.match_id, message.payload.matchmaker_trader_id, - DeclineMatchReason.ORDER_COMPLETED) + decline_reason) continue propose_quantity = Quantity(min(float(order.available_quantity), float(message.payload.match_quantity)), @@ -1671,7 +1671,7 @@ def send_transaction_completed(self, transaction, block): del self.incoming_match_messages[transaction.match_id] candidate = Candidate(self.lookup_ip(match_message.payload.matchmaker_trader_id), False) - linked_block = self.market_database.get_linked(block) + linked_block = self.market_database.get_linked(block) or block self.send_block_pair(block, linked_block, candidate) def on_transaction_completed_message(self, block1, block2): @@ -1712,5 +1712,5 @@ def compute_reputation(self): """ Compute the reputation of peers in the community """ - rep_manager = PagerankReputationManager(self.tradechain_community.persistence.get_all_blocks()) + rep_manager = TemporalPagerankReputationManager(self.persistence.get_all_blocks()) self.reputation_dict = rep_manager.compute(self.my_member.public_key) diff --git a/Tribler/community/market/reputation/pagerank_manager.py b/Tribler/community/market/reputation/pagerank_manager.py deleted file mode 100644 index 59d66c8c6a0..00000000000 --- a/Tribler/community/market/reputation/pagerank_manager.py +++ /dev/null @@ -1,29 +0,0 @@ -import networkx as nx - -from Tribler.community.market.reputation.reputation_manager import ReputationManager - - -class PagerankReputationManager(ReputationManager): - - def compute(self, own_public_key): - """ - Compute the reputation based on the data in the TradeChain database using the PageRank algorithm. - """ - - nodes = set() - G = nx.Graph() - for block in self.blocks: - nodes.add(block.public_key) - nodes.add(block.link_public_key) - - G.add_edge(block.public_key, block.link_public_key, - attr_dict={'weight': block.transaction["asset1_amount"]}) - G.add_edge(block.link_public_key, block.public_key, - attr_dict={'weight': block.transaction["asset2_amount"]}) - - personalization_vector = {} - for node in nodes: - personalization_vector[node] = 0 - personalization_vector[own_public_key] = 1 # You trust yourself the most - - return nx.pagerank_scipy(G, personalization=personalization_vector) diff --git a/Tribler/community/market/reputation/reputation_manager.py b/Tribler/community/market/reputation/reputation_manager.py index 65b221bb5e0..390cc6a3095 100644 --- a/Tribler/community/market/reputation/reputation_manager.py +++ b/Tribler/community/market/reputation/reputation_manager.py @@ -1,6 +1,10 @@ +import logging + + class ReputationManager(object): def __init__(self, blocks): + self._logger = logging.getLogger(self.__class__.__name__) self.blocks = blocks def compute(self, own_public_key): diff --git a/Tribler/community/market/reputation/temporal_pagerank_manager.py b/Tribler/community/market/reputation/temporal_pagerank_manager.py new file mode 100644 index 00000000000..e3c77c753be --- /dev/null +++ b/Tribler/community/market/reputation/temporal_pagerank_manager.py @@ -0,0 +1,64 @@ +import networkx as nx + +from Tribler.community.market.reputation.reputation_manager import ReputationManager +from Tribler.community.trustchain.block import UNKNOWN_SEQ + + +class TemporalPagerankReputationManager(ReputationManager): + + def compute(self, own_public_key): + """ + Compute the reputation based on the data in the TradeChain database using the Temporal PageRank algorithm. + """ + + nodes = set() + G = nx.DiGraph() + + for block in self.blocks: + if block.link_sequence_number == UNKNOWN_SEQ: + continue # Don't consider half interactions + + pubkey_requester = block.link_public_key + pubkey_responder = block.public_key + + sequence_number_requester = block.link_sequence_number + sequence_number_responder = block.sequence_number + + # In our market, we consider the amount of Bitcoin that have been transferred from A -> B. + # For now, we assume that the value from B -> A is of equal worth. + + is_price_btc = block.transaction["tx"]["quantity_type"] == "BTC" + value_exchange = block.transaction["tx"]["quantity"] if is_price_btc else block.transaction["tx"]["price"] + + G.add_edge((pubkey_requester, sequence_number_requester), (pubkey_requester, sequence_number_requester + 1), + contribution=value_exchange) + G.add_edge((pubkey_requester, sequence_number_requester), (pubkey_responder, sequence_number_responder + 1), + contribution=value_exchange) + + G.add_edge((pubkey_responder, sequence_number_responder), (pubkey_responder, sequence_number_responder + 1), + contribution=value_exchange) + G.add_edge((pubkey_responder, sequence_number_responder), (pubkey_requester, sequence_number_requester + 1), + contribution=value_exchange) + + nodes.add(pubkey_requester) + nodes.add(pubkey_responder) + + personal_nodes = [node1 for node1 in G.nodes() if node1[0] == own_public_key] + number_of_nodes = len(personal_nodes) + if number_of_nodes == 0: + return {} + personalisation = {node_name: 1.0 / number_of_nodes if node_name in personal_nodes else 0 + for node_name in G.nodes()} + + try: + result = nx.pagerank_scipy(G, personalization=personalisation, weight='contribution') + except nx.NetworkXException: + self._logger.info("Empty Temporal PageRank, returning empty scores") + return {} + + sums = {} + + for interaction in result.keys(): + sums[interaction[0]] = sums.get(interaction[0], 0) + result[interaction] + + return sums diff --git a/doc/development/development_on_linux.rst b/doc/development/development_on_linux.rst index d894646eb6a..2dff3f1f7d2 100644 --- a/doc/development/development_on_linux.rst +++ b/doc/development/development_on_linux.rst @@ -7,7 +7,7 @@ First, install the required dependencies by executing the following command in y .. code-block:: none - sudo apt-get install libav-tools libsodium18 libx11-6 python-apsw python-cherrypy3 python-cryptography python-decorator python-dnspython python-ecdsa python-feedparser python-jsonrpclib python-keyring python-keyrings.alt python-leveldb python-libtorrent python-matplotlib python-meliae python-m2crypto python-netifaces python-pbkdf2 python-pil python-protobuf python-pyasn1 python-pysocks python-requests python-twisted python2.7 vlc python-chardet python-configobj python-pyqt5 python-pyqt5.qtsvg + sudo apt-get install libav-tools libsodium18 libx11-6 python-apsw python-cherrypy3 python-cryptography python-decorator python-dnspython python-ecdsa python-feedparser python-jsonrpclib python-keyring python-keyrings.alt python-leveldb python-libtorrent python-matplotlib python-meliae python-m2crypto python-netifaces python-pbkdf2 python-pil python-protobuf python-pyasn1 python-pysocks python-requests python-scipy python-twisted python2.7 vlc python-chardet python-configobj python-pyqt5 python-pyqt5.qtsvg Next, download the latest .deb file from `here `_. diff --git a/doc/development/development_on_osx.rst b/doc/development/development_on_osx.rst index 79d7c8ee15f..58ac4518196 100644 --- a/doc/development/development_on_osx.rst +++ b/doc/development/development_on_osx.rst @@ -139,7 +139,7 @@ There are a bunch of other packages that can easily be installed using pip and b brew install homebrew/python/pillow gmp mpfr libmpc libsodium sudo easy_install pip pip install --user cython # Needs to be installed first for meliae - pip install --user cherrypy cffi chardet configobj cryptography decorator dnspython ecdsa feedparser gmpy2 jsonrpclib idna keyring leveldb meliae netifaces numpy pbkdf2 pillow protobuf pyasn1 pysocks pycparser requests twisted service_identity + pip install --user cherrypy cffi chardet configobj cryptography decorator dnspython ecdsa feedparser gmpy2 jsonrpclib idna keyring leveldb meliae netifaces numpy pbkdf2 pillow protobuf pyasn1 pysocks pycparser requests scipy twisted service_identity If you encounter any error during the installation of Pillow, make sure that libjpeg and zlib are installed. They can be installed using: